diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-07-19 17:16:28 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-07-19 17:16:28 +0300 |
commit | e4384360a16dd9a19d4d2d25d0ef1f2b862ed2a6 (patch) | |
tree | 2fcdfa7dcdb9db8f5208b2562f4b4e803d671243 /lib | |
parent | ffda4e7bcac36987f936b4ba515995a6698698f0 (diff) |
Add latest changes from gitlab-org/gitlab@16-2-stable-eev16.2.0-rc42
Diffstat (limited to 'lib')
334 files changed, 11271 insertions, 3518 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 090fbaa7f93..7da5f21b21f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -316,6 +316,7 @@ module API mount ::API::UsageDataQueries mount ::API::Users mount ::API::UserCounts + mount ::API::UserRunners mount ::API::Wikis add_open_api_documentation! diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index f7a39db7249..aa7468723b7 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -82,7 +82,9 @@ module API unauthorized! unless award.user == current_user || current_user&.can_admin_all_resources? - destroy_conditionally!(award) + destroy_conditionally!(award) do + AwardEmojis::DestroyService.new(awardable, award.name, award.user).execute + end end end end diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 84c9f780a53..62eecdbd5e5 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -22,7 +22,9 @@ module API %w[group project].each do |source_type| params do - requires :id, type: String, desc: "The ID of a #{source_type}" + requires :id, + type: String, + desc: "The ID or URL-encoded path of the #{source_type} owned by the authenticated user." end resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Gets a list of #{source_type} badges viewable by the authenticated user." do diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb index e27ec24fb44..1606d5ba649 100644 --- a/lib/api/ci/pipeline_schedules.rb +++ b/lib/api/ci/pipeline_schedules.rb @@ -90,14 +90,28 @@ module API post ':id/pipeline_schedules' do authorize! :create_pipeline_schedule, user_project - pipeline_schedule = ::Ci::CreatePipelineScheduleService - .new(user_project, current_user, declared_params(include_missing: false)) - .execute + if ::Feature.enabled?(:ci_refactoring_pipeline_schedule_create_service, @project) + response = ::Ci::PipelineSchedules::CreateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute - if pipeline_schedule.persisted? - present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails + pipeline_schedule = response.payload + + if response.success? + present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end else - render_validation_error!(pipeline_schedule) + pipeline_schedule = ::Ci::CreatePipelineScheduleService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if pipeline_schedule.persisted? + present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end end end @@ -121,10 +135,22 @@ module API put ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :update_pipeline_schedule, pipeline_schedule - if pipeline_schedule.update(declared_params(include_missing: false)) - present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails + if ::Feature.enabled?(:ci_refactoring_pipeline_schedule_create_service, @project) + response = ::Ci::PipelineSchedules::UpdateService + .new(pipeline_schedule, current_user, declared_params(include_missing: false)) + .execute + + if response.success? + present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end else - render_validation_error!(pipeline_schedule) + if pipeline_schedule.update(declared_params(include_missing: false)) # rubocop:disable Style/IfInsideElse + present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end end end diff --git a/lib/api/ci/variables.rb b/lib/api/ci/variables.rb index f5331eb75da..49a6ec279fb 100644 --- a/lib/api/ci/variables.rb +++ b/lib/api/ci/variables.rb @@ -63,6 +63,7 @@ module API optional :raw, type: Boolean, desc: 'Whether the variable will be expanded' optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of the variable. Default: env_var' optional :environment_scope, type: String, desc: 'The environment_scope of the variable' + optional :description, type: String, desc: 'The description of the variable' end post ':id/variables' do variable = ::Ci::ChangeVariableService.new( @@ -95,6 +96,7 @@ module API optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' do optional :environment_scope, type: String, desc: 'The environment scope of a variable' end + optional :description, type: String, desc: 'The description of the variable' end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index 25c97932e31..45290cb3e44 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -13,7 +13,6 @@ module API component: ::Packages::Debian::COMPONENT_REGEX, architecture: ::Packages::Debian::ARCHITECTURE_REGEX }.freeze - LIST_PACKAGE = 'list_package' included do feature_category :package_registry @@ -41,8 +40,6 @@ module API package_file = distribution_from!(project).package_files.with_file_name(params[:file_name]).last! - track_debian_package_event 'pull_package' - present_package_file!(package_file) end @@ -73,22 +70,8 @@ module API no_content! # empty component files are not always persisted in DB end - track_debian_package_event LIST_PACKAGE - present_carrierwave_file!(component_file.file) end - - def track_debian_package_event(action) - if project_or_group.is_a?(Project) - project = project_or_group - namespace = project_or_group.namespace - else - project = nil - namespace = project_or_group - end - - track_package_event(action, :debian, project: project, namespace: namespace, user: current_user) - end end rescue_from ArgumentError do |e| @@ -146,7 +129,6 @@ module API get 'Release' do distribution = distribution_from!(project_or_group) - track_debian_package_event LIST_PACKAGE present_carrierwave_file!(distribution.file) end @@ -166,7 +148,6 @@ module API get 'InRelease' do distribution = distribution_from!(project_or_group) - track_debian_package_event LIST_PACKAGE present_carrierwave_file!(distribution.signed_file) end diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index 74ad3bb296f..ec20440f013 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -21,12 +21,18 @@ module API included do helpers ::API::Helpers::Packages::DependencyProxyHelpers + rescue_from ActiveRecord::RecordInvalid do |e| + render_structured_api_error!({ message: e.message, error: e.message }, 400) + end + before do require_packages_enabled! authenticate_non_get! end helpers do + include Gitlab::Utils::StrongMemoize + params :package_name do requires :package_name, type: String, file_path: true, desc: 'Package name', documentation: { example: 'mypackage' } @@ -51,6 +57,12 @@ module API def generate_metadata_service(packages) ::Packages::Npm::GenerateMetadataService.new(params[:package_name], packages) end + + def metadata_cache + ::Packages::Npm::MetadataCache + .find_by_package_name_and_project_id(params[:package_name], project.id) + end + strong_memoize_attr :metadata_cache end params do @@ -80,7 +92,7 @@ module API packages = ::Packages::Npm::PackageFinder.new(package_name, project: project) .execute - not_found! if packages.empty? + not_found!('Package') if packages.empty? track_package_event(:list_tags, :npm, project: project, namespace: project.namespace) @@ -122,6 +134,8 @@ module API track_package_event(:create_tag, :npm, project: project, namespace: project.namespace) + enqueue_sync_metadata_cache_worker(project, package_name) + ::Packages::Npm::CreateTagService.new(package, tag).execute no_content! @@ -156,6 +170,8 @@ module API track_package_event(:delete_tag, :npm, project: project, namespace: project.namespace) + enqueue_sync_metadata_cache_worker(project, package_name) + ::Packages::RemoveTagService.new(package_tag).execute no_content! @@ -202,6 +218,17 @@ module API not_found!('Packages') if packages.empty? + if endpoint_scope == :project && Feature.enabled?(:npm_metadata_cache, project) + if metadata_cache&.file&.exists? + metadata_cache.touch_last_downloaded_at + present_carrierwave_file!(metadata_cache.file) + + break + end + + enqueue_sync_metadata_cache_worker(project, package_name) + end + present ::Packages::Npm::PackagePresenter.new(generate_metadata_service(packages).execute), with: ::API::Entities::NpmPackage end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index e1531847b87..a2b2d781797 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -111,8 +111,6 @@ module API ::Packages::Debian::CreatePackageFileService.new(package: package, current_user: current_user, params: file_params).execute - track_debian_package_event 'push_package' - created! rescue ObjectStorage::RemoteStoreError => e Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id }) diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 3a0eea677b8..8161c2b850f 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -68,6 +68,7 @@ module API desc: 'The status to filter deployments by. One of `created`, `running`, `success`, `failed`, `canceled`, or `blocked`' end + route_setting :authentication, job_token_allowed: true get ':id/deployments' do authorize! :read_deployment, user_project @@ -92,6 +93,7 @@ module API params do requires :deployment_id, type: Integer, desc: 'The ID of the deployment' end + route_setting :authentication, job_token_allowed: true get ':id/deployments/:deployment_id' do authorize! :read_deployment, user_project @@ -129,9 +131,10 @@ module API requires :status, type: String, - desc: 'The status to filter deployments by. One of `running`, `success`, `failed`, or `canceled`', + desc: 'The status of the deployment that is created. One of `running`, `success`, `failed`, or `canceled`', values: %w[running success failed canceled] end + route_setting :authentication, job_token_allowed: true post ':id/deployments' do authorize!(:create_deployment, user_project) authorize!(:create_environment, user_project) @@ -175,6 +178,7 @@ module API desc: 'The new status of the deployment. One of `running`, `success`, `failed`, or `canceled`', values: %w[running success failed canceled] end + route_setting :authentication, job_token_allowed: true put ':id/deployments/:deployment_id' do authorize!(:read_deployment, user_project) @@ -207,6 +211,7 @@ module API params do requires :deployment_id, type: Integer, desc: 'The ID of the deployment' end + route_setting :authentication, job_token_allowed: true delete ':id/deployments/:deployment_id' do deployment = user_project.deployments.find(params[:deployment_id]) @@ -240,7 +245,7 @@ module API use :merge_requests_base_params end - + route_setting :authentication, job_token_allowed: true get ':id/deployments/:deployment_id/merge_requests' do authorize! :read_deployment, user_project diff --git a/lib/api/entities/blob.rb b/lib/api/entities/blob.rb index 12700d99865..b4206679ac9 100644 --- a/lib/api/entities/blob.rb +++ b/lib/api/entities/blob.rb @@ -15,6 +15,13 @@ module API expose :ref expose :startline expose :project_id + expose :group_id, if: ->(object) { object.is_a?(Gitlab::Search::FoundWikiPage) } + + private + + def group_id + object.group&.id + end end end end diff --git a/lib/api/entities/bulk_imports/export_batch_status.rb b/lib/api/entities/bulk_imports/export_batch_status.rb new file mode 100644 index 00000000000..6e4a2fc8f93 --- /dev/null +++ b/lib/api/entities/bulk_imports/export_batch_status.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module API + module Entities + module BulkImports + class ExportBatchStatus < Grape::Entity + expose :status, documentation: { type: 'string', example: 'started', values: %w[started finished failed] } + expose :batch_number, documentation: { type: 'integer', example: 1 } + expose :objects_count, documentation: { type: 'integer', example: 100 } + expose :error, documentation: { type: 'string', example: 'Error message' } + expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + end + end + end +end diff --git a/lib/api/entities/bulk_imports/export_status.rb b/lib/api/entities/bulk_imports/export_status.rb index fee983c6fd8..1e5ee6ec210 100644 --- a/lib/api/entities/bulk_imports/export_status.rb +++ b/lib/api/entities/bulk_imports/export_status.rb @@ -8,6 +8,10 @@ module API expose :status, documentation: { type: 'string', example: 'started', values: %w[started finished failed] } expose :error, documentation: { type: 'string', example: 'Error message' } expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :batched, documentation: { type: 'boolean', example: true } + expose :batches_count, documentation: { type: 'integer', example: 2 } + expose :total_objects_count, documentation: { type: 'integer', example: 100 } + expose :batches, if: ->(export, _options) { export.batched? }, using: ExportBatchStatus end end end diff --git a/lib/api/entities/ci/runner.rb b/lib/api/entities/ci/runner.rb index 9361709b6ed..441e1dc1117 100644 --- a/lib/api/entities/ci/runner.rb +++ b/lib/api/entities/ci/runner.rb @@ -6,6 +6,7 @@ module API class Runner < Grape::Entity expose :id, documentation: { type: 'integer', example: 8 } expose :description, documentation: { type: 'string', example: 'test-1-20150125' } + # TODO: return null in 17.0 and remove in v5 https://gitlab.com/gitlab-org/gitlab/-/issues/415159 expose :ip_address, documentation: { type: 'string', example: '127.0.0.1' } # TODO Remove in v5 in favor of `paused` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/375709 expose :active, documentation: { type: 'boolean', example: true } diff --git a/lib/api/entities/ci/variable.rb b/lib/api/entities/ci/variable.rb index 47597cb77be..4336f82fb5e 100644 --- a/lib/api/entities/ci/variable.rb +++ b/lib/api/entities/ci/variable.rb @@ -14,6 +14,8 @@ module API expose :raw?, as: :raw, if: -> (entity, _) { entity.respond_to?(:raw?) }, documentation: { type: 'boolean' } expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) }, documentation: { type: 'string', example: '*' } + expose :description, if: -> (entity, _) { entity.respond_to?(:description) }, + documentation: { type: 'string', example: 'This variable is being used for ...' } end end end diff --git a/lib/api/entities/dictionary/table.rb b/lib/api/entities/dictionary/table.rb index 8d4e3fb959d..93e82d34b14 100644 --- a/lib/api/entities/dictionary/table.rb +++ b/lib/api/entities/dictionary/table.rb @@ -5,7 +5,7 @@ module API module Dictionary class Table < Grape::Entity expose :table_name, documentation: { type: :string, example: 'users' } - expose :feature_categories, documentation: { type: :array, example: ['database'] } + expose :feature_categories, documentation: { type: :string, is_array: true, example: 'database' } end end end diff --git a/lib/api/entities/group.rb b/lib/api/entities/group.rb index 246fb819890..9296617dac9 100644 --- a/lib/api/entities/group.rb +++ b/lib/api/entities/group.rb @@ -21,6 +21,7 @@ module API expose :full_name, :full_path expose :created_at expose :parent_id + expose :shared_runners_setting expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb index 753c595d65f..27b24a60305 100644 --- a/lib/api/entities/plan_limit.rb +++ b/lib/api/entities/plan_limit.rb @@ -14,6 +14,11 @@ module API expose :enforcement_limit, documentation: { type: 'integer', example: 15000 } expose :generic_packages_max_file_size, documentation: { type: 'integer', example: 5368709120 } expose :helm_max_file_size, documentation: { type: 'integer', example: 5242880 } + expose :limits_history, documentation: { + type: 'object', + example: '{"enforcement_limit"=>[{"timestamp"=>1686909124, "user_id"=>1, "username"=>"x", "value"=>5}], + "notification_limit"=>[{"timestamp"=>1686909124, "user_id"=>2, "username"=>"y", "value"=>7}]}' + } expose :maven_max_file_size, documentation: { type: 'integer', example: 3221225472 } expose :notification_limit, documentation: { type: 'integer', example: 15000 } expose :npm_max_file_size, documentation: { type: 'integer', example: 524288000 } diff --git a/lib/api/entities/project_hook.rb b/lib/api/entities/project_hook.rb index bffb057abed..b85d2747226 100644 --- a/lib/api/entities/project_hook.rb +++ b/lib/api/entities/project_hook.rb @@ -14,6 +14,7 @@ module API expose :job_events, documentation: { type: 'boolean' } expose :releases_events, documentation: { type: 'boolean' } expose :push_events_branch_filter, documentation: { type: 'string', example: 'my-branch-*' } + expose :emoji_events, documentation: { type: 'boolean' } end end end diff --git a/lib/api/entities/protected_ref_access.rb b/lib/api/entities/protected_ref_access.rb index 28e0ef540d5..3cac91ccddc 100644 --- a/lib/api/entities/protected_ref_access.rb +++ b/lib/api/entities/protected_ref_access.rb @@ -5,12 +5,9 @@ module API class ProtectedRefAccess < Grape::Entity expose :id, documentation: { type: 'integer', example: 1 } expose :access_level, documentation: { type: 'integer', example: 40 } - expose :access_level_description, - documentation: { type: 'string', example: 'Maintainers' } do |protected_ref_access| - protected_ref_access.humanize - end + expose :humanize, as: :access_level_description, documentation: { type: 'string', example: 'Maintainers' } expose :deploy_key_id, documentation: { type: 'integer', example: 1 }, - if: ->(access) { access.has_attribute?(:deploy_key_id) && access.deploy_key_id } + if: ->(access) { access.has_attribute?(:deploy_key_id) } end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index bb261079d2a..b94391359ed 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -38,6 +38,7 @@ module API desc: 'List all environments that match a specific state. Accepted values: `available`, `stopping`, or `stopped`. If no state value given, returns all environments' mutually_exclusive :name, :search, message: 'cannot be used together' end + route_setting :authentication, job_token_allowed: true get ':id/environments' do authorize! :read_environment, user_project @@ -66,6 +67,7 @@ module API optional :slug, absence: { message: "is automatically generated and cannot be changed" }, documentation: { hidden: true } optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the new environment. Allowed values are `production`, `staging`, `testing`, `development`, and `other`' end + route_setting :authentication, job_token_allowed: true post ':id/environments' do authorize! :create_environment, user_project @@ -94,6 +96,7 @@ module API optional :slug, absence: { message: "is automatically generated and cannot be changed" }, documentation: { hidden: true } optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the new environment. Allowed values are `production`, `staging`, `testing`, `development`, and `other`' end + route_setting :authentication, job_token_allowed: true put ':id/environments/:environment_id' do authorize! :update_environment, user_project @@ -126,6 +129,7 @@ module API optional :limit, type: Integer, desc: "Maximum number of environments to delete. Defaults to 100", default: 100, values: 1..1000 optional :dry_run, type: Boolean, desc: "Defaults to true for safety reasons. It performs a dry run where no actual deletion will be performed. Set to false to actually delete the environment", default: true end + route_setting :authentication, job_token_allowed: true delete ":id/environments/review_apps" do authorize! :read_environment, user_project @@ -156,6 +160,7 @@ module API params do requires :environment_id, type: Integer, desc: 'The ID of the environment' end + route_setting :authentication, job_token_allowed: true delete ':id/environments/:environment_id' do authorize! :read_environment, user_project @@ -178,6 +183,7 @@ module API requires :environment_id, type: Integer, desc: 'The ID of the environment' optional :force, type: Boolean, default: false, desc: 'Force environment to stop without executing `on_stop` actions' end + route_setting :authentication, job_token_allowed: true post ':id/environments/:environment_id/stop' do authorize! :read_environment, user_project @@ -202,6 +208,7 @@ module API type: DateTime, desc: 'Stop all environments that were last modified or deployed to before this date.' end + route_setting :authentication, job_token_allowed: true post ':id/environments/stop_stale' do authorize! :stop_environment, user_project @@ -229,6 +236,7 @@ module API params do requires :environment_id, type: Integer, desc: 'The ID of the environment' end + route_setting :authentication, job_token_allowed: true get ':id/environments/:environment_id' do authorize! :read_environment, user_project diff --git a/lib/api/files.rb b/lib/api/files.rb index 45e935d7ea2..c140cec658d 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -49,7 +49,14 @@ module API end def content_sha - cache_client.fetch("blob_content_sha256:#{user_project.full_path}:#{@blob.id}") do + cache_client.fetch( + "blob_content_sha256:#{user_project.full_path}:#{@blob.id}", + nil, + { + cache_identifier: 'API::Files#content_sha', + backing_resource: :gitaly + } + ) do @blob.load_all_data! Digest::SHA256.hexdigest(@blob.data) @@ -57,10 +64,8 @@ module API end def cache_client - Gitlab::Cache::Client.build_with_metadata( - cache_identifier: 'API::Files#content_sha', - feature_category: :source_code_management, - backing_resource: :gitaly + @cache_client ||= Gitlab::Cache::Client.new( + Gitlab::Cache::Metrics.new(Gitlab::Cache::Metadata.new(feature_category: :source_code_management)) ) end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 37dfbfdb925..4cac707ff66 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -80,8 +80,13 @@ module API { code: 503, message: 'Service unavailable' } ] end + params do + optional :batched, type: Boolean, desc: 'Whether to export in batches' + end post ':id/export_relations' do - response = ::BulkImports::ExportService.new(portable: user_group, user: current_user).execute + response = ::BulkImports::ExportService + .new(portable: user_group, user: current_user, batched: params[:batched]) + .execute if response.success? accepted! @@ -104,15 +109,32 @@ module API end params do requires :relation, type: String, desc: 'Group relation name' + optional :batched, type: Boolean, desc: 'Whether to download in batches' + optional :batch_number, type: Integer, desc: 'Batch number to download' + + all_or_none_of :batched, :batch_number end get ':id/export_relations/download' do export = user_group.bulk_import_exports.find_by_relation(params[:relation]) - file = export&.upload&.export_file - if file - present_carrierwave_file!(file) + break render_api_error!('Export not found', 404) unless export + + if params[:batched] + batch = export.batches.find_by_batch_number(params[:batch_number]) + batch_file = batch&.upload&.export_file + + break render_api_error!('Export is not batched', 400) unless export.batched? + break render_api_error!('Batch not found', 404) unless batch + break render_api_error!('Batch file not found', 404) unless batch_file + + present_carrierwave_file!(batch_file) else - render_api_error!('404 Not found', 404) + file = export&.upload&.export_file + + break render_api_error!('Export is batched', 400) if export.batched? + break render_api_error!('Export file not found', 404) unless file + + present_carrierwave_file!(file) end end @@ -128,8 +150,19 @@ module API { code: 503, message: 'Service unavailable' } ] end + params do + optional :relation, type: String, desc: 'Group relation name' + end get ':id/export_relations/status' do - present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus + if params[:relation] + export = user_group.bulk_import_exports.find_by_relation(params[:relation]) + + break render_api_error!('Export not found', 404) unless export + + present export, with: Entities::BulkImports::ExportStatus + else + present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus + end end end end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 295bee475c3..f320fa06394 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -59,6 +59,8 @@ module API optional :raw, type: String, desc: 'Whether the variable will be expanded' optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of the variable. Default: env_var' optional :environment_scope, type: String, desc: 'The environment scope of a variable' + optional :description, type: String, desc: 'The description of the variable' + use :optional_group_variable_params_ee end post ':id/variables' do @@ -94,6 +96,7 @@ module API optional :raw, type: String, desc: 'Whether the variable will be expanded' optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of the variable. Default: env_var' optional :environment_scope, type: String, desc: 'The environment scope of a variable' + optional :description, type: String, desc: 'The description of the variable' use :optional_group_variable_params_ee end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index df080c8e666..b616f1b35b3 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -657,6 +657,19 @@ module API Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") end + def track_event(event_name, user_id:, namespace_id: nil, project_id: nil) + return unless user_id.present? + + Gitlab::InternalEvents.track_event( + event_name, + user_id: user_id, + namespace_id: namespace_id, + project_id: project_id + ) + rescue StandardError => error + Gitlab::AppLogger.warn("Internal Event tracking event failed for event: #{event_name}, message: #{error.message}") + end + def order_by_similarity?(allow_unauthorized: true) params[:order_by] == 'similarity' && params[:search].present? && (allow_unauthorized || current_user.present?) end diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb index 88208226c40..fc3f42f0d58 100644 --- a/lib/api/helpers/custom_attributes.rb +++ b/lib/api/helpers/custom_attributes.rb @@ -8,7 +8,7 @@ module API included do helpers do params :with_custom_attributes do - optional :with_custom_attributes, type: Boolean, default: false, desc: 'Include custom attributes in the response' + optional :with_custom_attributes, type: ::Grape::API::Boolean, default: false, desc: 'Include custom attributes in the response' optional :custom_attributes, type: Hash, desc: 'Filter with custom attributes' diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 850cc61af2c..09dd69ef03b 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -41,7 +41,7 @@ module API { required: false, name: :notify_only_broken_pipelines, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Send notifications for broken pipelines' } ].freeze @@ -129,85 +129,85 @@ module API { required: false, name: :commit_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for commit_events' }, { required: false, name: :push_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for push_events' }, { required: false, name: :issues_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for issues_events' }, { required: false, name: :incident_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for incident_events' }, { required: false, name: :alert_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for alert_events' }, { required: false, name: :confidential_issues_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for confidential_issues_events' }, { required: false, name: :merge_requests_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for merge_requests_events' }, { required: false, name: :note_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for note_events' }, { required: false, name: :confidential_note_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for confidential_note_events' }, { required: false, name: :tag_push_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for tag_push_events' }, { required: false, name: :deployment_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for deployment_events' }, { required: false, name: :job_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for job_events' }, { required: false, name: :pipeline_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for pipeline_events' }, { required: false, name: :wiki_page_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable notifications for wiki_page_events' } ].freeze @@ -243,7 +243,7 @@ module API { required: false, name: :app_store_protected_refs, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Only enable for protected refs' } ], @@ -285,7 +285,7 @@ module API { required: false, name: :enable_ssl_verification, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable SSL verification' }, { @@ -343,7 +343,7 @@ module API { required: false, name: :enable_ssl_verification, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'DEPRECATED: This parameter has no effect since SSL verification will always be enabled' } ], @@ -417,7 +417,7 @@ module API { required: false, name: :archive_trace_events, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'When enabled, job logs will be collected by Datadog and shown along pipeline execution traces' }, { @@ -471,7 +471,7 @@ module API { required: false, name: :enable_ssl_verification, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable SSL verification' } ], @@ -485,13 +485,13 @@ module API { required: false, name: :disable_diffs, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Disable code diffs' }, { required: false, name: :send_from_committer_email, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Send from committer' }, { @@ -598,7 +598,7 @@ module API { required: false, name: :colorize_messages, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Colorize messages' } ], @@ -612,7 +612,7 @@ module API { required: false, name: :enable_ssl_verification, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable SSL verification' }, { @@ -668,7 +668,7 @@ module API { required: false, name: :jira_issue_transition_automatic, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable automatic issue transitions' }, { @@ -692,7 +692,7 @@ module API { required: false, name: :comment_on_event_enabled, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable comments inside Jira issues on each GitLab event (commit / merge request)' } ], @@ -750,13 +750,13 @@ module API { required: false, name: :notify_only_broken_pipelines, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Notify only broken pipelines' }, { required: false, name: :notify_only_default_branch, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Send notifications only for the default branch' }, { @@ -946,7 +946,7 @@ module API { required: false, name: :enable_ssl_verification, - type: Boolean, + type: ::Grape::API::Boolean, desc: 'Enable SSL verification' }, { diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index b3c79486465..a82aed507fd 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -29,6 +29,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def retrieve_members(source, params:, deep: false) members = deep ? find_all_members(source) : source_members(source).connected_to_user + members = members.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/417456") members = members.includes(:user) members = members.references(:user).merge(User.search(params[:query], use_minimum_char_limit: false)) if params[:query].present? members = members.where(user_id: params[:user_ids]) if params[:user_ids].present? diff --git a/lib/api/helpers/packages/maven.rb b/lib/api/helpers/packages/maven.rb new file mode 100644 index 00000000000..694a1ec6436 --- /dev/null +++ b/lib/api/helpers/packages/maven.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module API + module Helpers + module Packages + module Maven + extend Grape::API::Helpers + + params :path_and_file_name do + requires :path, + type: String, + desc: 'Package path', + documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } + requires :file_name, + type: String, + desc: 'Package file name', + documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } + end + end + end + end +end diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index be7f57fda0c..a80122c5309 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -6,6 +6,7 @@ module API module Npm include Gitlab::Utils::StrongMemoize include ::API::Helpers::PackagesHelpers + extend ::Gitlab::Utils::Override NPM_ENDPOINT_REQUIREMENTS = { package_name: API::NO_SLASH_URL_PART_REGEX @@ -55,8 +56,7 @@ module API when :group finder = ::Packages::Npm::PackageFinder.new( params[:package_name], - namespace: group, - last_of_each_version: false + namespace: group ) finder.last&.project_id @@ -77,8 +77,7 @@ module API finder = ::Packages::Npm::PackageFinder.new( package_name, - namespace: namespace, - last_of_each_version: false + namespace: namespace ) finder.last&.project_id @@ -86,6 +85,12 @@ module API end strong_memoize_attr :project_id_or_nil + def enqueue_sync_metadata_cache_worker(project, package_name) + return unless Feature.enabled?(:npm_metadata_cache, project) + + ::Packages::Npm::CreateMetadataCacheWorker.perform_async(project.id, package_name) + end + private def top_namespace_from(package_name) @@ -101,6 +106,20 @@ module API group end strong_memoize_attr :group + + override :not_found! + def not_found!(resource = nil) + reason = "#{resource} not found" + message = "404 #{reason}".titleize + render_structured_api_error!({ message: message, error: reason }, 404) + end + + override :bad_request_missing_attribute! + def bad_request_missing_attribute!(attribute) + reason = "\"#{attribute}\" not given" + message = "400 Bad request - #{reason}" + render_structured_api_error!({ message: message, error: reason }, 400) + end end end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index b96e8efba61..642963768f8 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -15,7 +15,6 @@ module API optional :auto_cancel_pending_pipelines, type: String, values: %w(disabled enabled), desc: 'Auto-cancel pending pipelines' optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' optional :service_desk_enabled, type: Boolean, desc: 'Disable or enable the service desk' - optional :keep_latest_artifact, type: Boolean, desc: 'Indicates if the latest artifact should be kept for this project.' # TODO: remove in API v5, replaced by *_access_level optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' @@ -71,7 +70,6 @@ module API optional :squash_commit_template, type: String, desc: 'Template used to create squash commit message' optional :issue_branch_template, type: String, desc: 'Template used to create a branch from an issue' optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" - optional :ci_default_git_depth, type: Integer, desc: 'Default number of revisions for shallow cloning' optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' optional :auto_devops_deploy_strategy, type: String, values: %w(continuous manual timed_incremental), desc: 'Auto Deploy strategy' optional :autoclose_referenced_issues, type: Boolean, desc: 'Flag indication if referenced issues auto-closing is enabled' @@ -101,6 +99,8 @@ module API end params :optional_update_params_ce do + optional :ci_default_git_depth, type: Integer, desc: 'Default number of revisions for shallow cloning' + optional :keep_latest_artifact, type: Boolean, desc: 'Indicates if the latest artifact should be kept for this project.' optional :ci_forward_deployment_enabled, type: Boolean, desc: 'Prevent older deployment jobs that are still pending' optional :ci_allow_fork_pipelines_to_run_in_parent_project, type: Boolean, desc: 'Allow fork merge request pipelines to run in parent project' optional :ci_separated_caches, type: Boolean, desc: 'Enable or disable separated caches based on branch protection.' @@ -205,9 +205,6 @@ module API def filter_attributes_using_license!(attrs) end - def filter_attributes_under_feature_flag!(attrs, project) - end - def validate_git_import_url!(import_url) return if import_url.blank? diff --git a/lib/api/helpers/remote_mirrors_helpers.rb b/lib/api/helpers/remote_mirrors_helpers.rb index efd81a5ac5a..2c00f7cdb14 100644 --- a/lib/api/helpers/remote_mirrors_helpers.rb +++ b/lib/api/helpers/remote_mirrors_helpers.rb @@ -18,7 +18,7 @@ module API use :mirror_branches_setting_ee end - def verify_mirror_branches_setting(attrs, project); end + def verify_mirror_branches_setting(attrs); end end end end diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index 6550808a563..ab7ac6624a8 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -20,7 +20,10 @@ module API end def access_params - { github_access_token: params[:personal_access_token] } + { + github_access_token: params[:personal_access_token], + additional_access_tokens: params[:additional_access_tokens] + } end def client_options @@ -59,6 +62,11 @@ module API requires :target_namespace, type: String, allow_blank: false, desc: 'Namespace or group to import repository into' optional :github_hostname, type: String, desc: 'Custom GitHub enterprise hostname' optional :optional_stages, type: Hash, desc: 'Optional stages of import to be performed' + optional :additional_access_tokens, + type: Array[String], + coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + desc: 'Additional list of personal access tokens', + documentation: { example: 'foo,bar' } end post 'import/github' do result = Import::GithubService.new(client, current_user, params).execute(access_params, provider) diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 5592207c4b5..8783a8dd57c 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -29,21 +29,13 @@ module API end def gitaly_info(project) - shard = repo_type.repository_for(project).shard - { - address: Gitlab::GitalyClient.address(shard), - token: Gitlab::GitalyClient.token(shard), - features: Feature::Gitaly.server_feature_flags - } + gitaly_features = Feature::Gitaly.server_feature_flags + + Gitlab::GitalyClient.connection_data(project.repository_storage).merge(features: gitaly_features) end def gitaly_repository(project) - { - storage_name: project.repository_storage, - relative_path: project.disk_path + '.git', - gl_repository: repo_type.identifier_for_container(project), - gl_project_path: repo_type.repository_for(project).full_path - } + project.repository.gitaly_repository.to_h end def check_feature_enabled @@ -61,7 +53,12 @@ module API end def increment_unique_events - events = params[:unique_counters]&.slice(:agent_users_using_ci_tunnel) + events = params[:unique_counters]&.slice( + :agent_users_using_ci_tunnel, + :k8s_api_proxy_requests_unique_users_via_ci_access, :k8s_api_proxy_requests_unique_agents_via_ci_access, + :k8s_api_proxy_requests_unique_users_via_user_access, :k8s_api_proxy_requests_unique_agents_via_user_access, + :flux_git_push_notified_unique_projects + ) events&.each do |event, entity_ids| increment_unique_values(event, entity_ids) @@ -69,7 +66,10 @@ module API end def increment_count_events - events = params[:counters]&.slice(:gitops_sync, :k8s_api_proxy_request, :flux_git_push_notifications_total) + events = params[:counters]&.slice( + :gitops_sync, :k8s_api_proxy_request, :flux_git_push_notifications_total, + :k8s_api_proxy_requests_via_ci_access, :k8s_api_proxy_requests_via_user_access + ) Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_event_counts(events) end @@ -185,7 +185,7 @@ module API # Load agent agent = ::Clusters::Agent.find(params[:agent_id]) - unauthorized!('Feature disabled for agent') unless ::Gitlab::Kas::UserAccess.enabled_for?(agent) + unauthorized!('Feature disabled for agent') unless ::Gitlab::Kas::UserAccess.enabled? service_response = ::Clusters::Agents::AuthorizeProxyUserService.new(user, agent).execute render_api_error!(service_response[:message], service_response[:reason]) unless service_response.success? @@ -203,10 +203,17 @@ module API optional :gitops_sync, type: Integer, desc: 'The count to increment the gitops_sync metric by' optional :k8s_api_proxy_request, type: Integer, desc: 'The count to increment the k8s_api_proxy_request metric by' optional :flux_git_push_notifications_total, type: Integer, desc: 'The count to increment the flux_git_push_notifications_total metrics by' + optional :k8s_api_proxy_requests_via_ci_access, type: Integer, desc: 'The count to increment the k8s_api_proxy_requests_via_ci_access metric by' + optional :k8s_api_proxy_requests_via_user_access, type: Integer, desc: 'The count to increment the k8s_api_proxy_requests_via_user_access metric by' end optional :unique_counters, type: Hash do optional :agent_users_using_ci_tunnel, type: Array[Integer], desc: 'An array of user ids that have interacted with CI Tunnel' + optional :k8s_api_proxy_requests_unique_users_via_ci_access, type: Array[Integer], desc: 'An array of users that have interacted with the CI tunnel via `ci_access`' + optional :k8s_api_proxy_requests_unique_agents_via_ci_access, type: Array[Integer], desc: 'An array of agents that have interacted with the CI tunnel via `ci_access`' + optional :k8s_api_proxy_requests_unique_users_via_user_access, type: Array[Integer], desc: 'An array of users that have interacted with the CI tunnel via `user_access`' + optional :k8s_api_proxy_requests_unique_agents_via_user_access, type: Array[Integer], desc: 'An array of agents that have interacted with the CI tunnel via `user_access`' + optional :flux_git_push_notified_unique_projects, type: Array[Integer], desc: 'An array of projects that have been notified to reconcile their Flux workloads' end end post '/', feature_category: :deployment_management do diff --git a/lib/api/lint.rb b/lib/api/lint.rb index dee04b6bb00..71965fc05c9 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -4,34 +4,6 @@ module API class Lint < ::API::Base feature_category :pipeline_composition - helpers do - def can_lint_ci? - signup_unrestricted = Gitlab::CurrentSettings.signup_enabled? && !Gitlab::CurrentSettings.signup_limited? - internal_user = current_user.present? && !current_user.external? - is_developer = current_user.present? && current_user.projects.any? { |p| p.member?(current_user, Gitlab::Access::DEVELOPER) } - - signup_unrestricted || internal_user || is_developer - end - end - - namespace :ci do - desc 'REMOVED: Validates the .gitlab-ci.yml content' do - detail 'Checks if CI/CD YAML configuration is valid' - success code: 200, model: Entities::Ci::Lint::Result - tags %w[ci_lint] - end - params do - requires :content, type: String, desc: 'The CI/CD configuration content' - optional :include_merged_yaml, type: Boolean, desc: 'If the expanded CI/CD configuration should be included in the response' - optional :include_jobs, type: Boolean, desc: 'If the list of jobs should be included in the response. This is - false by default' - end - - post '/lint', urgency: :low do - render_api_error!('410 Gone', 410) - end - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Validates a CI YAML configuration with a namespace' do detail 'Checks if a project’s latest (HEAD of the project’s default branch) .gitlab-ci.yml configuration is diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index 5ef60ab0b94..08b172cd608 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -40,10 +40,7 @@ module API context[:skip_project_check] = true end - # Disable comments in markdown for IE browsers because comments in IE - # could allow script execution. - browser = Browser.new(headers['User-Agent']) - context[:allow_comments] = !browser.ie? + context[:allow_comments] = false present({ html: Banzai.render_and_post_process(params[:text], context) }, with: Entities::Markdown) end diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 241cd93f380..eccc55ed158 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -23,14 +23,10 @@ module API helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::DependencyProxyHelpers + helpers ::API::Helpers::Packages::Maven helpers ::API::Helpers::Packages::Maven::BasicAuthHelpers helpers do - params :path_and_file_name do - requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } - requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } - end - def path_exists?(path) return false if path.blank? diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index c29a7eee923..ff9d0e2c371 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -124,6 +124,10 @@ module API merge_requests.each { |mr| mr.check_mergeability(async: true) } end + def batch_process_mergeability_checks(merge_requests) + ::MergeRequests::MergeabilityCheckBatchService.new(merge_requests, current_user).execute + end + params :merge_requests_params do use :merge_requests_base_params use :optional_merge_requests_search_params @@ -177,8 +181,16 @@ module API get ":id/merge_requests", feature_category: :code_review_workflow, urgency: :low do validate_search_rate_limit! if declared_params[:search].present? merge_requests = find_merge_requests(group_id: user_group.id, include_subgroups: true) + options = serializer_options_for(merge_requests).merge(group: user_group) + + if !options[:skip_merge_status_recheck] && ::Feature.enabled?(:batched_api_mergeability_checks, user_group) + batch_process_mergeability_checks(merge_requests) + + # NOTE: skipping individual mergeability checks in the presenter + options[:skip_merge_status_recheck] = true + end - present merge_requests, serializer_options_for(merge_requests).merge(group: user_group) + present merge_requests, options end end diff --git a/lib/api/ml_model_packages.rb b/lib/api/ml_model_packages.rb index fec72b03ffd..35d231d9fe1 100644 --- a/lib/api/ml_model_packages.rb +++ b/lib/api/ml_model_packages.rb @@ -6,7 +6,7 @@ module API include ::API::Helpers::Authentication ML_MODEL_PACKAGES_REQUIREMENTS = { - package_name: API::NO_SLASH_URL_PART_REGEX, + model_name: API::NO_SLASH_URL_PART_REGEX, file_name: API::NO_SLASH_URL_PART_REGEX }.freeze @@ -47,15 +47,15 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/ml_models' do params do - requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.ml_model_name_regex, + requires :model_name, type: String, desc: 'Model name', regexp: Gitlab::Regex.ml_model_name_regex, file_path: true - requires :package_version, type: String, desc: 'Package version', + requires :model_version, type: String, desc: 'Model version', regexp: Gitlab::Regex.ml_model_version_regex requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.ml_model_file_name_regex, file_path: true optional :status, type: String, values: ALLOWED_STATUSES, desc: 'Package status' end - namespace ':package_name/*package_version/:file_name', requirements: ML_MODEL_PACKAGES_REQUIREMENTS do + namespace ':model_name/*model_version/:file_name', requirements: ML_MODEL_PACKAGES_REQUIREMENTS do desc 'Workhorse authorize model package file' do detail 'Introduced in GitLab 16.1' success code: 200 @@ -71,7 +71,7 @@ module API end desc 'Workhorse upload model package file' do - detail 'Introduced in GitLab 16.1' + detail 'Introduced in GitLab 16.2' success code: 201 failure [ { code: 401, message: 'Unauthorized' }, @@ -91,7 +91,12 @@ module API bad_request!('File is too large') if max_file_size_exceeded? - create_package_file_params = declared(params).merge(build: current_authenticated_job) + create_package_file_params = declared(params).merge( + build: current_authenticated_job, + package_name: params[:model_name], + package_version: params[:model_version] + ) + package_file = ::Packages::MlModel::CreatePackageFileService .new(project, current_user, create_package_file_params) .execute @@ -104,6 +109,26 @@ module API forbidden! end + + desc 'Download an ml_model package file' do + detail 'This feature was introduced in GitLab 16.2' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[ml_model_registry] + end + get do + authorize_read_package!(project) + + package = ::Packages::MlModel::PackageFinder.new(project) + .execute!(params[:model_name], params[:model_version]) + package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute! + + present_package_file!(package_file) + end end end end diff --git a/lib/api/npm_instance_packages.rb b/lib/api/npm_instance_packages.rb index 8215296b617..ea92818e76c 100644 --- a/lib/api/npm_instance_packages.rb +++ b/lib/api/npm_instance_packages.rb @@ -6,10 +6,6 @@ module API feature_category :package_registry urgency :low - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) - end - helpers do def endpoint_scope :instance diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 61409909b06..e1d0455b1e2 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -7,7 +7,7 @@ module API urgency :low rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) + render_structured_api_error!({ message: e.message, error: e.message }, 400) end helpers do @@ -78,8 +78,9 @@ module API .new(project, current_user, params.merge(build: current_authenticated_job)).execute if created_package[:status] == :error - render_api_error!(created_package[:message], created_package[:http_status]) + render_structured_api_error!({ message: created_package[:message], error: created_package[:message] }, created_package[:http_status]) else + enqueue_sync_metadata_cache_worker(project, created_package.name) track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, namespace: project.namespace) created_package end diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index 7ff49f326d9..6a02769519f 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -14,6 +14,7 @@ module API urgency :low helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::Npm params do requires :id, types: [String, Integer], desc: 'ID or URL-encoded path of the project' @@ -70,6 +71,8 @@ module API destroy_conditionally!(package_file) do |package_file| package_file.pending_destruction! + + enqueue_sync_metadata_cache_worker(user_project, package.name) if package.npm? end end end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 19e5ed3f9e0..8a72ec051dc 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -127,8 +127,13 @@ module API ] tags ['project_export'] end + params do + optional :batched, type: Boolean, desc: 'Whether to export in batches' + end post ':id/export_relations' do - response = ::BulkImports::ExportService.new(portable: user_project, user: current_user).execute + response = ::BulkImports::ExportService + .new(portable: user_project, user: current_user, batched: params[:batched]) + .execute if response.success? accepted! @@ -152,19 +157,33 @@ module API produces %w[application/octet-stream application/json] end params do - requires :relation, - type: String, - project_portable: true, - desc: 'Project relation name' + requires :relation, type: String, project_portable: true, desc: 'Project relation name' + optional :batched, type: Boolean, desc: 'Whether to download in batches' + optional :batch_number, type: Integer, desc: 'Batch number to download' + + all_or_none_of :batched, :batch_number end get ':id/export_relations/download' do export = user_project.bulk_import_exports.find_by_relation(params[:relation]) - file = export&.upload&.export_file - if file - present_carrierwave_file!(file) + break render_api_error!('Export not found', 404) unless export + + if params[:batched] + batch = export.batches.find_by_batch_number(params[:batch_number]) + batch_file = batch&.upload&.export_file + + break render_api_error!('Export is not batched', 400) unless export.batched? + break render_api_error!('Batch not found', 404) unless batch + break render_api_error!('Batch file not found', 404) unless batch_file + + present_carrierwave_file!(batch_file) else - render_api_error!('404 Not found', 404) + file = export&.upload&.export_file + + break render_api_error!('Export is batched', 400) if export.batched? + break render_api_error!('Export file not found', 404) unless file + + present_carrierwave_file!(file) end end @@ -180,8 +199,19 @@ module API ] tags ['project_export'] end + params do + optional :relation, type: String, desc: 'Project relation name' + end get ':id/export_relations/status' do - present user_project.bulk_import_exports, with: Entities::BulkImports::ExportStatus + if params[:relation] + export = user_project.bulk_import_exports.find_by_relation(params[:relation]) + + break render_api_error!('Export not found', 404) unless export + + present export, with: Entities::BulkImports::ExportStatus + else + present user_project.bulk_import_exports, with: Entities::BulkImports::ExportStatus + end end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index db97f4988e1..c9cba397f5c 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -31,6 +31,7 @@ module API optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :deployment_events, type: Boolean, desc: "Trigger hook on deployment events" optional :releases_events, type: Boolean, desc: "Trigger hook on release events" + optional :emoji_events, type: Boolean, desc: "Trigger hook on emoji events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only" diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 43bd15931ef..2aa6858e41d 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -15,6 +15,7 @@ module API urgency :low helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::Npm helpers do def package strong_memoize(:package) do # rubocop:disable Gitlab/StrongMemoizeAttr @@ -133,6 +134,8 @@ module API destroy_conditionally!(package) do |package| ::Packages::MarkPackageForDestructionService.new(container: package, current_user: current_user).execute + + enqueue_sync_metadata_cache_worker(user_project, package.name) if package.npm? end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7ec9f72e0b2..468f284f136 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -251,6 +251,28 @@ module API present_projects load_projects end + desc 'Get projects that a user has contributed to' do + success code: 200, model: Entities::BasicProjectDetails + failure [{ code: 404, message: '404 User Not Found' }] + tags %w[projects] + is_array true + end + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + use :sort_params + use :pagination + + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + end + get ":user_id/contributed_projects", feature_category: :groups_and_projects, urgency: :low do + user = find_user(params[:user_id]) + not_found!('User') unless user + + contributed_projects = ContributedProjectsFinder.new(user).execute(current_user).joined(user) + present_projects contributed_projects + end + desc 'Get projects starred by a user' do success code: 200, model: Entities::BasicProjectDetails failure [{ code: 404, message: '404 User Not Found' }] @@ -534,7 +556,6 @@ module API attrs = translate_params_for_compatibility(attrs) attrs = add_import_params(attrs) filter_attributes_using_license!(attrs) - filter_attributes_under_feature_flag!(attrs, user_project) verify_update_project_attrs!(user_project, attrs) user_project.remove_avatar! if attrs.key?(:avatar) && attrs[:avatar].nil? diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb index c3c7d9370e0..5346476ed0d 100644 --- a/lib/api/remote_mirrors.rb +++ b/lib/api/remote_mirrors.rb @@ -67,7 +67,7 @@ module API end post ':id/remote_mirrors' do create_params = declared_params(include_missing: false) - verify_mirror_branches_setting(create_params, user_project) + verify_mirror_branches_setting(create_params) new_mirror = user_project.remote_mirrors.create(create_params) if new_mirror.persisted? @@ -99,7 +99,7 @@ module API mirror_params = declared_params(include_missing: false) mirror_params[:id] = mirror_params.delete(:mirror_id) - verify_mirror_branches_setting(mirror_params, user_project) + verify_mirror_branches_setting(mirror_params) update_params = { remote_mirrors_attributes: mirror_params } result = ::Projects::UpdateService diff --git a/lib/api/search.rb b/lib/api/search.rb index 954c3cd9f9e..b14fce13f5e 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -59,6 +59,8 @@ module API end def search(additional_params = {}) + return Kaminari.paginate_array([]) if @project.present? && !project_scope_allowed? + search_service = search_service(additional_params) if search_service.global_search? && !search_service.global_search_enabled_for_scope? forbidden!('Global Search is disabled for this scope') @@ -95,6 +97,10 @@ module API ) end + def project_scope_allowed? + ::Search::Navigation.new(user: current_user, project: @project).tab_enabled_for_project?(params[:scope].to_sym) + end + def snippets? %w(snippet_titles).include?(params[:scope]).to_s end @@ -108,9 +114,7 @@ module API end def verify_search_scope!(resource:) - # In EE we have additional validation requirements for searches. - # Defining this method here as a noop allows us to easily extend it in - # EE, without having to modify this file directly. + # no-op end def search_type(additional_params = {}) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 5d20444cb54..b12ca48829b 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -203,6 +203,13 @@ module API optional :allow_runner_registration_token, type: Boolean, desc: 'Allow registering runners using a registration token' optional :ci_max_includes, type: Integer, desc: 'Maximum number of includes per pipeline' optional :security_policy_global_group_approvers_enabled, type: Boolean, desc: 'Query scan result policy approval groups globally' + optional :slack_app_enabled, type: Grape::API::Boolean, desc: 'Enable the GitLab for Slack app' + given slack_app_enabled: -> (val) { val } do + requires :slack_app_id, type: String, desc: 'The client ID of the GitLab for Slack app' + requires :slack_app_secret, type: String, desc: 'The client secret of the GitLab for Slack app. Used for authenticating OAuth requests from the app' + requires :slack_app_signing_secret, type: String, desc: 'The signing secret of the GitLab for Slack app. Used for authenticating API requests from the app' + requires :slack_app_verification_token, type: String, desc: 'The verification token of the GitLab for Slack app. This method of authentication is deprecated by Slack and used only for authenticating slash commands from the app' + end Gitlab::SSHPublicKey.supported_types.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index 3e2023d769f..0a343093c33 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -53,6 +53,38 @@ module API status :ok end + desc 'Track gitlab internal events' do + detail 'This feature was introduced in GitLab 16.2.' + success code: 200 + failure [ + { code: 403, message: 'Invalid CSRF token is provided' }, + { code: 404, message: 'Not found' } + ] + tags %w[usage_data] + end + params do + requires :event, type: String, desc: 'The event name that should be tracked', + documentation: { example: 'i_quickactions_page' } + optional :namespace_id, type: Integer, desc: 'Namespace ID', + documentation: { example: 1234 } + optional :project_id, type: Integer, desc: 'Project ID', + documentation: { example: 1234 } + end + post 'track_event', urgency: :low do + event_name = params[:event] + namespace_id = params[:namespace_id] + project_id = params[:project_id] + + track_event( + event_name, + user_id: current_user.id, + namespace_id: namespace_id, + project_id: project_id + ) + + status :ok + end + desc 'Get a list of all metric definitions' do detail 'This feature was introduced in GitLab 13.11.' success code: 200 diff --git a/lib/api/user_runners.rb b/lib/api/user_runners.rb new file mode 100644 index 00000000000..edbd0214bb8 --- /dev/null +++ b/lib/api/user_runners.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module API + class UserRunners < ::API::Base + include APIGuard + + resource :user do + before do + authenticate! + end + + allow_access_with_scope :create_runner, if: ->(request) { request.post? } + + desc 'Create a runner owned by currently authenticated user' do + detail 'Create a new runner' + success Entities::Ci::RunnerRegistrationDetails + failure [[400, 'Bad Request'], [403, 'Forbidden']] + tags %w[user runners] + end + params do + requires :runner_type, type: String, values: ::Ci::Runner.runner_types.keys, + desc: %q(Specifies the scope of the runner) + given runner_type: ->(runner_type) { runner_type == 'group_type' } do + requires :group_id, type: Integer, + desc: 'The ID of the group that the runner is created in', + documentation: { example: 1 } + end + given runner_type: ->(runner_type) { runner_type == 'project_type' } do + requires :project_id, type: Integer, + desc: 'The ID of the project that the runner is created in', + documentation: { example: 1 } + end + optional :description, type: String, desc: %q(Description of the runner) + optional :maintenance_note, type: String, + desc: %q(Free-form maintenance notes for the runner (1024 characters)) + optional :paused, type: Boolean, desc: 'Specifies if the runner should ignore new jobs (defaults to false)' + optional :locked, type: Boolean, + desc: 'Specifies if the runner should be locked for the current project (defaults to false)' + optional :access_level, type: String, values: ::Ci::Runner.access_levels.keys, + desc: 'The access level of the runner' + optional :run_untagged, type: Boolean, + desc: 'Specifies if the runner should handle untagged jobs (defaults to true)' + optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + desc: %q(A list of runner tags) + optional :maximum_timeout, type: Integer, + desc: 'Maximum timeout that limits the amount of time (in seconds) that runners can run jobs' + end + post 'runners', urgency: :low, feature_category: :runner_fleet do + attributes = attributes_for_keys( + %i[runner_type group_id project_id description maintenance_note paused locked run_untagged tag_list + access_level maximum_timeout] + ) + + case attributes[:runner_type] + when 'group_type' + attributes[:scope] = ::Group.find_by_id(attributes.delete(:group_id)) + when 'project_type' + attributes[:scope] = ::Project.find_by_id(attributes.delete(:project_id)) + end + + result = ::Ci::Runners::CreateRunnerService.new(user: current_user, params: attributes).execute + if result.error? + message = result.errors.to_sentence + forbidden!(message) if result.reason == :forbidden + bad_request!(message) + end + + present result.payload[:runner], with: Entities::Ci::RunnerRegistrationDetails + end + end + end +end diff --git a/lib/api/users.rb b/lib/api/users.rb index ff36a4cfe95..fff0e9fee06 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -6,7 +6,7 @@ module API include APIGuard include Helpers::CustomAttributes - allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } + allow_access_with_scope :read_user, if: ->(request) { request.get? || request.head? } feature_category :user_profile, %w[ @@ -134,7 +134,7 @@ module API entity = current_user&.can_read_all_resources? ? Entities::UserWithAdmin : Entities::UserBasic if entity == Entities::UserWithAdmin - users = users.preload(:identities, :webauthn_registrations, :namespace, :followers, :followees, :user_preference) + users = users.preload(:identities, :webauthn_registrations, :namespace, :followers, :followees, :user_preference, :user_detail) end users, options = with_custom_attributes(users, { with: entity, current_user: current_user }) @@ -1369,63 +1369,6 @@ module API get 'status', feature_category: :user_profile do present current_user.status || {}, with: Entities::UserStatus end - - desc 'Create a runner owned by currently authenticated user' do - detail 'Create a new runner' - success Entities::Ci::RunnerRegistrationDetails - failure [[400, 'Bad Request'], [403, 'Forbidden']] - tags %w[user runners] - end - params do - requires :runner_type, type: String, values: ::Ci::Runner.runner_types.keys, - desc: %q(Specifies the scope of the runner) - given runner_type: ->(runner_type) { runner_type == 'group_type' } do - requires :group_id, type: Integer, - desc: 'The ID of the group that the runner is created in', - documentation: { example: 1 } - end - given runner_type: ->(runner_type) { runner_type == 'project_type' } do - requires :project_id, type: Integer, - desc: 'The ID of the project that the runner is created in', - documentation: { example: 1 } - end - optional :description, type: String, desc: %q(Description of the runner) - optional :maintenance_note, type: String, - desc: %q(Free-form maintenance notes for the runner (1024 characters)) - optional :paused, type: Boolean, desc: 'Specifies if the runner should ignore new jobs (defaults to false)' - optional :locked, type: Boolean, - desc: 'Specifies if the runner should be locked for the current project (defaults to false)' - optional :access_level, type: String, values: ::Ci::Runner.access_levels.keys, - desc: 'The access level of the runner' - optional :run_untagged, type: Boolean, - desc: 'Specifies if the runner should handle untagged jobs (defaults to true)' - optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, - desc: %q(A list of runner tags) - optional :maximum_timeout, type: Integer, - desc: 'Maximum timeout that limits the amount of time (in seconds) that runners can run jobs' - end - post 'runners', urgency: :low, feature_category: :runner_fleet do - attributes = attributes_for_keys( - %i[runner_type group_id project_id description maintenance_note paused locked run_untagged tag_list - access_level maximum_timeout] - ) - - case attributes[:runner_type] - when 'group_type' - attributes[:scope] = ::Group.find_by_id(attributes.delete(:group_id)) - when 'project_type' - attributes[:scope] = ::Project.find_by_id(attributes.delete(:project_id)) - end - - result = ::Ci::Runners::CreateRunnerService.new(user: current_user, params: attributes).execute - if result.error? - message = result.errors.to_sentence - forbidden!(message) if result.reason == :forbidden - bad_request!(message) - end - - present result.payload[:runner], with: Entities::Ci::RunnerRegistrationDetails - end end end end diff --git a/lib/atlassian/jira_connect/serializers/build_entity.rb b/lib/atlassian/jira_connect/serializers/build_entity.rb index b595d0c2a92..31ab4ece8fe 100644 --- a/lib/atlassian/jira_connect/serializers/build_entity.rb +++ b/lib/atlassian/jira_connect/serializers/build_entity.rb @@ -22,13 +22,7 @@ module Atlassian expose :references def issue_keys - commit_message_issue_keys = JiraIssueKeyExtractor.new(pipeline.git_commit_message).issue_keys - - # extract Jira issue keys from either the source branch/ref or the merge request title. - @issue_keys ||= commit_message_issue_keys + pipeline.all_merge_requests.flat_map do |mr| - src = "#{mr.source_branch} #{mr.title} #{mr.description}" - JiraIssueKeyExtractor.new(src).issue_keys - end.uniq + @issue_keys ||= (pipeline_commit_issue_keys + pipeline_mrs_issue_keys).uniq end private @@ -89,6 +83,18 @@ module Atlassian def update_sequence_id options[:update_sequence_id] || Client.generate_update_sequence_id end + + def pipeline_commit_issue_keys + JiraIssueKeyExtractor.new(pipeline.git_commit_message).issue_keys + end + + # Extract Jira issue keys from either the source branch/ref, merge request title or merge request description. + def pipeline_mrs_issue_keys + pipeline.all_merge_requests.flat_map do |mr| + src = "#{mr.source_branch} #{mr.title} #{mr.description}" + JiraIssueKeyExtractor.new(src).issue_keys + end + end end end end diff --git a/lib/atlassian/jira_connect/serializers/deployment_entity.rb b/lib/atlassian/jira_connect/serializers/deployment_entity.rb index 9ef1666b61c..96e7b1726cb 100644 --- a/lib/atlassian/jira_connect/serializers/deployment_entity.rb +++ b/lib/atlassian/jira_connect/serializers/deployment_entity.rb @@ -6,6 +6,8 @@ module Atlassian class DeploymentEntity < Grape::Entity include Gitlab::Routing + COMMITS_LIMIT = 5_000 + format_with(:iso8601, &:iso8601) expose :schema_version, as: :schemaVersion @@ -22,9 +24,7 @@ module Atlassian expose :environment_entity, as: :environment def issue_keys - return [] unless build&.pipeline.present? - - @issue_keys ||= BuildEntity.new(build.pipeline).issue_keys + @issue_keys ||= (issue_keys_from_pipeline + issue_keys_from_commits_since_last_deploy).uniq end private @@ -74,7 +74,7 @@ module Atlassian end def pipeline_entity - PipelineEntity.new(build.pipeline) if build&.pipeline.present? + PipelineEntity.new(build.pipeline) if pipeline? end def environment_entity @@ -84,6 +84,44 @@ module Atlassian def update_sequence_id options[:update_sequence_id] || Client.generate_update_sequence_id end + + def pipeline? + build&.pipeline.present? + end + + def issue_keys_from_pipeline + return [] unless pipeline? + + BuildEntity.new(build.pipeline).issue_keys + end + + # Extract Jira issue keys from commits made to the deployment's branch or tag + # since the last successful deployment was made to the environment. + def issue_keys_from_commits_since_last_deploy + return [] if Feature.disabled?(:jira_deployment_issue_keys, project) + + last_deployed_commit = environment + .successful_deployments + .id_not_in(deployment.id) + .ordered + .find_by_ref(deployment.ref) + &.commit + + commits = project.repository.commits( + deployment.ref, + before: deployment.commit.created_at, + after: last_deployed_commit&.created_at, + skip_merges: true, + limit: COMMITS_LIMIT + ) + + # Include this deploy's commit, as the `before:` param in `Repository#list_commits_by` excluded it. + commits << deployment.commit + + commits.flat_map do |commit| + JiraIssueKeyExtractor.new(commit.message).issue_keys + end.compact + end end end end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 28bc78a3932..12656cb3702 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -63,7 +63,7 @@ module Backup progress.flush end ensure - ::Gitlab::Database::EachDatabase.each_database_connection( + ::Gitlab::Database::EachDatabase.each_connection( only: base_models_for_backup.keys, include_shared: false ) do |connection, _| Gitlab::Database::TransactionTimeoutSettings.new(connection).restore_timeouts @@ -259,7 +259,7 @@ module Backup @database_to_snapshot_id = {} if @database_to_snapshot_id.empty? - ::Gitlab::Database::EachDatabase.each_database_connection( + ::Gitlab::Database::EachDatabase.each_connection( only: base_models_for_backup.keys, include_shared: false ) do |connection, database_name| @database_to_snapshot_id[database_name] = nil diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 56726665d14..199da8821d9 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -58,8 +58,11 @@ module Backup end def enqueue_consecutive_projects - project_relation.find_each(batch_size: 1000) do |project| - enqueue_project(project) + cross_join_issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/417467" + ::Gitlab::Database.allow_cross_joins_across_databases(url: cross_join_issue) do + project_relation.find_each(batch_size: 1000) do |project| + enqueue_project(project) + end end end diff --git a/lib/banzai/filter/references/external_issue_reference_filter.rb b/lib/banzai/filter/references/external_issue_reference_filter.rb index 1061a9917dd..ead816abab3 100644 --- a/lib/banzai/filter/references/external_issue_reference_filter.rb +++ b/lib/banzai/filter/references/external_issue_reference_filter.rb @@ -24,8 +24,15 @@ module Banzai # # Returns a String replaced with the return of the block. def references_in(text, pattern = object_reference_pattern) - text.gsub(pattern) do |match| - yield match, $~[:issue] + case pattern + when Regexp + text.gsub(pattern) do |match| + yield match, $~[:issue] + end + when Gitlab::UntrustedRegexp + pattern.replace_gsub(text) do |match| + yield match, match[:issue] + end end end diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index a687ae2882e..5353d3f4e49 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -206,7 +206,8 @@ module Banzai end def replace_text_when_pattern_matches(node, index, pattern) - return unless node.text =~ pattern + return if pattern.is_a?(Gitlab::UntrustedRegexp) && !pattern.match?(node.text) + return if pattern.is_a?(Regexp) && !(pattern =~ node.text) content = node.to_html html = yield content diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb index d6b6fdb7149..a3784004087 100644 --- a/lib/banzai/filter/references/user_reference_filter.rb +++ b/lib/banzai/filter/references/user_reference_filter.rb @@ -65,10 +65,13 @@ module Banzai # The keys of this Hash are the namespace paths, the values the # corresponding Namespace objects. def namespaces - @namespaces ||= Namespace.eager_load(:owner, :route) - .where_full_path_in(usernames) - .index_by(&:full_path) - .transform_keys(&:downcase) + cross_join_issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/417466" + Gitlab::Database.allow_cross_joins_across_databases(url: cross_join_issue) do + @namespaces ||= Namespace.eager_load(:owner, :route) + .where_full_path_in(usernames) + .index_by(&:full_path) + .transform_keys(&:downcase) + end end # Returns all usernames referenced in the current document. diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 0031ccc7011..f1dada9176f 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -9,6 +9,7 @@ module Banzai Filter::SanitizationFilter, Filter::AssetProxyFilter, Filter::EmojiFilter, + Filter::CustomEmojiFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter, *reference_filters diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 48e2bcc9a11..ec96181e7f1 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -97,9 +97,12 @@ module Banzai def find_users_for_groups(ids) return [] if ids.empty? - User.joins(:group_members).where(members: { - source_id: Namespace.where(id: ids).where('mentions_disabled IS NOT TRUE').select(:id) - }).to_a + cross_join_issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/417466" + ::Gitlab::Database.allow_cross_joins_across_databases(url: cross_join_issue) do + User.joins(:group_members).where(members: { + source_id: Namespace.where(id: ids).where('mentions_disabled IS NOT TRUE').select(:id) + }).to_a + end end def find_users_for_projects(ids) diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb index 04febebff8e..e1a0e5cf2fb 100644 --- a/lib/bulk_imports/common/extractors/ndjson_extractor.rb +++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb @@ -33,7 +33,7 @@ module BulkImports def download_service(context) @download_service ||= BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(relation), + relative_url: context.entity.relation_download_url_path(relation, context.extra[:batch_number]), tmpdir: tmpdir, filename: filename ) diff --git a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb index 68bd64dc2ff..0bf4d341aad 100644 --- a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb @@ -6,6 +6,8 @@ module BulkImports class LfsObjectsPipeline include Pipeline + file_extraction_pipeline! + def extract(_context) download_service.execute decompression_service.execute @@ -48,7 +50,7 @@ module BulkImports def download_service BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(relation), + relative_url: context.entity.relation_download_url_path(relation, context.extra[:batch_number]), tmpdir: tmpdir, filename: targz_filename ) diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb index 06132791ea6..81ce20db9ab 100644 --- a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -10,6 +10,8 @@ module BulkImports AvatarLoadingError = Class.new(StandardError) + file_extraction_pipeline! + def extract(_context) download_service.execute decompression_service.execute @@ -46,7 +48,7 @@ module BulkImports def download_service BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(relation), + relative_url: context.entity.relation_download_url_path(relation, context.extra[:batch_number]), tmpdir: tmpdir, filename: targz_filename ) diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index c879ec41d86..b83d67c359d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -88,7 +88,7 @@ module ContainerRegistry def supports_tag_delete? strong_memoize(:supports_tag_delete) do registry_features = Gitlab::CurrentSettings.container_registry_features || [] - next true if ::Gitlab.com? && registry_features.include?(REGISTRY_TAG_DELETE_FEATURE) + next true if ::Gitlab.com_except_jh? && registry_features.include?(REGISTRY_TAG_DELETE_FEATURE) response = faraday.run_request(:options, '/v2/name/tags/reference/tag', '', {}) response.success? && response.headers['allow']&.include?('DELETE') diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index 00877bb5a48..f5982e96622 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -85,7 +85,7 @@ module ContainerRegistry def supports_gitlab_api? strong_memoize(:supports_gitlab_api) do registry_features = Gitlab::CurrentSettings.container_registry_features || [] - next true if ::Gitlab.com? && registry_features.include?(REGISTRY_GITLAB_V1_API_FEATURE) + next true if ::Gitlab.com_except_jh? && registry_features.include?(REGISTRY_GITLAB_V1_API_FEATURE) with_token_faraday do |faraday_client| response = faraday_client.get('/gitlab/v1/') diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index 06160b55f5c..51a66958ba0 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -4,15 +4,15 @@ module ExpandVariables VARIABLES_REGEXP = /\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/.freeze class << self - def expand(value, variables) - replace_with(value, variables) do |vars_hash, last_match| - match_or_blank_value(vars_hash, last_match) + def expand(value, variables, expand_file_refs: true) + replace_with(value, variables) do |collection, last_match| + match_or_blank_value(collection, last_match, expand_file_refs: expand_file_refs) end end - def expand_existing(value, variables) - replace_with(value, variables) do |vars_hash, last_match| - match_or_original_value(vars_hash, last_match) + def expand_existing(value, variables, expand_file_refs: true) + replace_with(value, variables) do |collection, last_match| + match_or_original_value(collection, last_match, expand_file_refs: expand_file_refs) end end @@ -25,37 +25,32 @@ module ExpandVariables private def replace_with(value, variables) - variables_hash = nil + # We lazily fabricate the variables collection in case there is no variable in the value string. + # `collection` needs to be initialized to nil here + # so that it is memoized in the closure block for `gsub`. + collection = nil value.gsub(VARIABLES_REGEXP) do - variables_hash ||= transform_variables(variables) - yield(variables_hash, Regexp.last_match) + collection ||= Gitlab::Ci::Variables::Collection.fabricate(variables) + yield(collection, Regexp.last_match) end end - def match_or_blank_value(variables, last_match) - variables[last_match[1] || last_match[2]] - end - - def match_or_original_value(variables, last_match) - match_or_blank_value(variables, last_match) || last_match[0] - end + def match_or_blank_value(collection, last_match, expand_file_refs:) + match = last_match[1] || last_match[2] + replacement = collection[match] - def transform_variables(variables) - # Lazily initialise variables - variables = variables.call if variables.is_a?(Proc) - - # Convert Collection to variables - variables = variables.to_hash if variables.is_a?(Gitlab::Ci::Variables::Collection) - - # Convert hash array to variables - if variables.is_a?(Array) - variables = variables.each_with_object({}) do |variable, hash| - hash[variable[:key]] = variable[:value] - end + if replacement.nil? + nil + elsif replacement.file? + expand_file_refs ? replacement.value : last_match + else + replacement.value end + end - variables + def match_or_original_value(collection, last_match, expand_file_refs:) + match_or_blank_value(collection, last_match, expand_file_refs: expand_file_refs) || last_match[0] end end end diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb index 49ec564eb8d..2a48b66bb5c 100644 --- a/lib/extracts_ref.rb +++ b/lib/extracts_ref.rb @@ -100,7 +100,7 @@ module ExtractsRef # rubocop:enable Gitlab/ModuleWithInstanceVariables def tree - @tree ||= @repo.tree(@commit.id, @path) # rubocop:disable Gitlab/ModuleWithInstanceVariables + @tree ||= @repo.tree(@commit.id, @path, ref_type: ref_type) # rubocop:disable Gitlab/ModuleWithInstanceVariables end def extract_ref_path diff --git a/lib/extracts_ref/requested_ref.rb b/lib/extracts_ref/requested_ref.rb new file mode 100644 index 00000000000..f20018b5ef4 --- /dev/null +++ b/lib/extracts_ref/requested_ref.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module ExtractsRef + class RequestedRef + include Gitlab::Utils::StrongMemoize + + SYMBOLIC_REF_PREFIX = %r{((refs/)?(heads|tags)/)+} + def initialize(repository, ref_type:, ref:) + @ref_type = ref_type + @ref = ref + @repository = repository + end + + attr_reader :repository, :ref_type, :ref + + def find + case ref_type + when 'tags' + { ref_type: ref_type, commit: tag } + when 'heads' + { ref_type: ref_type, commit: branch } + else + commit_without_ref_type + end + end + + private + + def commit_without_ref_type + if commit.nil? + { ref_type: nil, commit: nil } + elsif commit.id == ref + # ref is probably complete 40 character sha + { ref_type: nil, commit: commit } + elsif tag.present? + { ref_type: 'tags', commit: tag, ambiguous: branch.present? } + elsif branch.present? + { ref_type: 'heads', commit: branch } + else + { ref_type: nil, commit: commit, ambiguous: ref.match?(SYMBOLIC_REF_PREFIX) } + end + end + + def commit + repository.commit(ref) + end + strong_memoize_attr :commit + + def tag + raw_commit = repository.find_tag(ref)&.dereferenced_target + ::Commit.new(raw_commit, repository.container) if raw_commit + end + strong_memoize_attr :tag + + def branch + raw_commit = repository.find_branch(ref)&.dereferenced_target + ::Commit.new(raw_commit, repository.container) if raw_commit + end + strong_memoize_attr :branch + end +end diff --git a/lib/generators/batched_background_migration/batched_background_migration_generator.rb b/lib/generators/batched_background_migration/batched_background_migration_generator.rb index c68ed52c1a0..44aff6fe17a 100644 --- a/lib/generators/batched_background_migration/batched_background_migration_generator.rb +++ b/lib/generators/batched_background_migration/batched_background_migration_generator.rb @@ -6,9 +6,12 @@ module BatchedBackgroundMigration class BatchedBackgroundMigrationGenerator < ActiveRecord::Generators::Base source_root File.expand_path('templates', __dir__) - class_option :table_name - class_option :column_name - class_option :feature_category + class_option :table_name, type: :string, required: true, desc: "Table from which records we will be batching" + class_option :column_name, type: :string, required: true, desc: "Column to use for batching", default: :id + class_option :feature_category, type: :string, required: true, + desc: "Feature category to which this batched background migration belongs to" + class_option :ee_only, type: :boolean, desc: "Generate files for EE-only batched background migration", + default: false def validate! raise ArgumentError, "table_name is required" unless table_name.present? @@ -29,15 +32,32 @@ module BatchedBackgroundMigration end def create_batched_background_migration_class_and_specs - template( - "batched_background_migration_job.template", - File.join("lib/gitlab/background_migration/#{file_name}.rb") - ) + if ee_only? + template( + "ee_batched_background_migration_job.template", + File.join("ee/lib/ee/gitlab/background_migration/#{file_name}.rb") + ) - template( - "batched_background_migration_job_spec.template", - File.join("spec/lib/gitlab/background_migration/#{file_name}_spec.rb") - ) + template( + "foss_batched_background_migration_job.template", + File.join("lib/gitlab/background_migration/#{file_name}.rb") + ) + + template( + "batched_background_migration_job_spec.template", + File.join("ee/spec/lib/ee/gitlab/background_migration/#{file_name}_spec.rb") + ) + else + template( + "batched_background_migration_job.template", + File.join("lib/gitlab/background_migration/#{file_name}.rb") + ) + + template( + "batched_background_migration_job_spec.template", + File.join("spec/lib/gitlab/background_migration/#{file_name}_spec.rb") + ) + end end def create_dictionary_file @@ -65,6 +85,10 @@ module BatchedBackgroundMigration options[:feature_category] end + def ee_only? + options[:ee_only] + end + def current_milestone version = Gem::Version.new(File.read('VERSION')) version.release.segments.first(2).join('.') diff --git a/lib/generators/batched_background_migration/templates/ee_batched_background_migration_job.template b/lib/generators/batched_background_migration/templates/ee_batched_background_migration_job.template new file mode 100644 index 00000000000..b36fc216acd --- /dev/null +++ b/lib/generators/batched_background_migration/templates/ee_batched_background_migration_job.template @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/database/batched_background_migrations.html +# for more information on how to use batched background migrations + +# Update below commented lines with appropriate values. + +module EE + module Gitlab + module BackgroundMigration + module <%= class_name %> + extend ActiveSupport::Concern + extend ::Gitlab::Utils::Override + + prepended do + # operation_name :my_operation + # scope_to ->(relation) { relation.where(column: "value") } + end + + override :perform + def perform + each_sub_batch do |sub_batch| + # Your action on each sub_batch + end + end + end + end + end +end diff --git a/lib/generators/batched_background_migration/templates/foss_batched_background_migration_job.template b/lib/generators/batched_background_migration/templates/foss_batched_background_migration_job.template new file mode 100644 index 00000000000..6a39c32955d --- /dev/null +++ b/lib/generators/batched_background_migration/templates/foss_batched_background_migration_job.template @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # TODO Add a top-level documentation comment for the class + class <%= class_name %> < BatchedMigrationJob + feature_category :<%= feature_category %> + + def perform; end + end + end +end + +Gitlab::BackgroundMigration::<%= class_name %>.prepend_mod diff --git a/lib/generators/gitlab/analytics/internal_events_generator.rb b/lib/generators/gitlab/analytics/internal_events_generator.rb index a85cdd352d5..d4c3a10c00e 100644 --- a/lib/generators/gitlab/analytics/internal_events_generator.rb +++ b/lib/generators/gitlab/analytics/internal_events_generator.rb @@ -26,8 +26,8 @@ module Gitlab end end.freeze - NEGATIVE_ANSWERS = %w[no n].freeze - POSITIVE_ANSWERS = %w[yes y].freeze + NEGATIVE_ANSWERS = %w[no n No NO N].freeze + POSITIVE_ANSWERS = %w[yes y Yes YES Y].freeze TOP_LEVEL_DIR = 'config' TOP_LEVEL_DIR_EE = 'ee' DESCRIPTION_MIN_LENGTH = 50 @@ -81,7 +81,7 @@ module Gitlab type: :string, optional: false, desc: 'Name of the event that this metric counts' - class_option :unique_on, + class_option :unique, type: :string, optional: false, desc: 'Name of the event property that this metric counts' @@ -185,7 +185,7 @@ module Gitlab end def key_path(time_frame) - "count_distinct_#{options[:unique_on]}_from_#{event}_#{time_frame}" + "count_distinct_#{options[:unique].sub('.', '_')}_from_#{event}_#{time_frame}" end def metric_file_path(time_frame) @@ -204,7 +204,7 @@ module Gitlab validate_tiers! - %i[unique_on event mr section stage group].each do |option| + %i[unique event mr section stage group].each do |option| raise "The option: --#{option} is missing" unless options.key? option end diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb index 6ac8de407b0..81f02c004af 100644 --- a/lib/gitlab/access/branch_protection.rb +++ b/lib/gitlab/access/branch_protection.rb @@ -45,6 +45,63 @@ module Gitlab def fully_protected? level == PROTECTION_FULL end + + def to_hash + # translate the original integer values into a json payload + # that matches the protected branches API: + # https://docs.gitlab.com/ee/api/protected_branches.html#update-a-protected-branch + case level + when PROTECTION_NONE + self.class.protection_none + when PROTECTION_DEV_CAN_PUSH + self.class.protection_partial + when PROTECTION_FULL + self.class.protected_fully + when PROTECTION_DEV_CAN_MERGE + self.class.protected_against_developer_pushes + when PROTECTION_DEV_CAN_INITIAL_PUSH + self.class.protected_after_initial_push + end + end + + class << self + def protection_none + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: true + } + end + + def protection_partial + protection_none.merge(allow_force_push: false) + end + + def protected_fully + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false + } + end + + def protected_against_developer_pushes + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: true + } + end + + def protected_after_initial_push + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: true, + developer_can_initial_push: true + } + end + end end end end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 5b136431ce7..3840a560c57 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -33,7 +33,6 @@ module Gitlab :has_required_attributes?, :hosts, :metric_id, - :metrics_dashboard_url, :monitoring_tool, :resolved?, :runbook, diff --git a/lib/gitlab/alert_management/payload/managed_prometheus.rb b/lib/gitlab/alert_management/payload/managed_prometheus.rb index 2236e60a0c6..4ed21108d3e 100644 --- a/lib/gitlab/alert_management/payload/managed_prometheus.rb +++ b/lib/gitlab/alert_management/payload/managed_prometheus.rb @@ -35,18 +35,6 @@ module Gitlab gitlab_alert&.environment || super end - def metrics_dashboard_url - return unless gitlab_alert - - metrics_dashboard_project_prometheus_alert_url( - project, - gitlab_alert.prometheus_metric_id, - environment_id: environment.id, - embedded: true, - **alert_embed_window_params - ) - end - private def plain_gitlab_fingerprint diff --git a/lib/gitlab/alert_management/payload/prometheus.rb b/lib/gitlab/alert_management/payload/prometheus.rb index 76f3da8366b..15fa91646c8 100644 --- a/lib/gitlab/alert_management/payload/prometheus.rb +++ b/lib/gitlab/alert_management/payload/prometheus.rb @@ -78,18 +78,6 @@ module Gitlab rescue URI::InvalidURIError, KeyError end - def metrics_dashboard_url - return unless environment && full_query && title - - metrics_dashboard_project_environment_url( - project, - environment, - embed_json: dashboard_json, - embedded: true, - **alert_embed_window_params - ) - end - def has_required_attributes? project && title && starts_at_raw end @@ -108,29 +96,6 @@ module Gitlab def plain_gitlab_fingerprint [starts_at_raw, title, full_query].join('/') end - - # Formatted for parsing by JS - def alert_embed_window_params - { - start: (starts_at - METRIC_TIME_WINDOW).utc.strftime('%FT%TZ'), - end: (starts_at + METRIC_TIME_WINDOW).utc.strftime('%FT%TZ') - } - end - - def dashboard_json - { - panel_groups: [{ - panels: [{ - type: 'area-chart', - title: title, - y_label: gitlab_y_label, - metrics: [{ - query_range: full_query - }] - }] - }] - }.to_json - end end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 0ea52b7b7c8..67fc2ae2fcc 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -47,11 +47,16 @@ module Gitlab Attribute.new(:root_caller_id, String), Attribute.new(:merge_action_status, String) ].freeze + private_constant :APPLICATION_ATTRIBUTES def self.known_keys KNOWN_KEYS end + def self.application_attributes + APPLICATION_ATTRIBUTES + end + def self.with_context(args, &block) application_context = new(**args) application_context.use(&block) @@ -79,12 +84,13 @@ module Gitlab end def initialize(**args) - unknown_attributes = args.keys - APPLICATION_ATTRIBUTES.map(&:name) + unknown_attributes = args.keys - self.class.application_attributes.map(&:name) raise ArgumentError, "#{unknown_attributes} are not known keys" if unknown_attributes.any? @set_values = args.keys assign_attributes(args) + set_attr_readers end # rubocop: disable Metrics/CyclomaticComplexity @@ -122,12 +128,14 @@ module Gitlab attr_reader :set_values - APPLICATION_ATTRIBUTES.each do |attr| - lazy_attr_reader attr.name, type: attr.type + def set_attr_readers + self.class.application_attributes.each do |attr| + self.class.lazy_attr_reader attr.name, type: attr.type + end end def assign_hash_if_value(hash, attribute_name) - unless KNOWN_KEYS.include?(attribute_name) + unless self.class.known_keys.include?(attribute_name) raise ArgumentError, "unknown attribute `#{attribute_name}`" end @@ -137,7 +145,7 @@ module Gitlab end def assign_attributes(values) - values.slice(*APPLICATION_ATTRIBUTES.map(&:name)).each do |name, value| + values.slice(*self.class.application_attributes.map(&:name)).each do |name, value| instance_variable_set("@#{name}", value) end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 83d94d168a0..1bb92b7fa62 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -9,7 +9,8 @@ module Gitlab API_SCOPE = :api READ_API_SCOPE = :read_api READ_USER_SCOPE = :read_user - API_SCOPES = [API_SCOPE, READ_API_SCOPE, READ_USER_SCOPE].freeze + CREATE_RUNNER_SCOPE = :create_runner + API_SCOPES = [API_SCOPE, READ_API_SCOPE, READ_USER_SCOPE, CREATE_RUNNER_SCOPE].freeze PROFILE_SCOPE = :profile EMAIL_SCOPE = :email @@ -236,6 +237,10 @@ module Gitlab user.can?(:read_project, project) end + def bot_user_can_read_project?(user, project) + (user.project_bot? || user.security_policy_bot?) && can_read_project?(user, project) + end + def valid_oauth_token?(token) token && token.accessible? && valid_scoped_token?(token, Doorkeeper.configuration.scopes) end @@ -251,7 +256,8 @@ module Gitlab read_registry: [:read_container_image], write_registry: [:create_container_image], read_repository: [:download_code], - write_repository: [:download_code, :push_code] + write_repository: [:download_code, :push_code], + create_runner: [:create_instance_runner, :create_runner] } scopes.flat_map do |scope| @@ -316,7 +322,7 @@ module Gitlab return unless build.project.builds_enabled? if build.user - return unless build.user.can_log_in_with_non_expired_password? || (build.user.project_bot? && can_read_project?(build.user, build.project)) + return unless build.user.can_log_in_with_non_expired_password? || bot_user_can_read_project?(build.user, build.project) # If user is assigned to build, use restricted credentials of user Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 4a610b26290..966520655a5 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -30,6 +30,7 @@ module Gitlab DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN' RUNNER_TOKEN_PARAM = :token RUNNER_JOB_TOKEN_PARAM = :token + PATH_DEPENDENT_FEED_TOKEN_REGEX = /\A#{User::FEED_TOKEN_PREFIX}(\h{64})-(\d+)\z/ # Check the Rails session for valid authentication details def find_user_from_warden @@ -54,7 +55,7 @@ module Gitlab token = current_request.params[:feed_token].presence || current_request.params[:rss_token].presence return unless token - User.find_by_feed_token(token) || raise(UnauthorizedError) + find_feed_token_user(token) || raise(UnauthorizedError) end def find_user_from_bearer_token @@ -195,6 +196,8 @@ module Gitlab when AccessTokenValidationService::EXPIRED raise ExpiredError when AccessTokenValidationService::REVOKED + revoke_token_family(access_token) + raise RevokedError when AccessTokenValidationService::IMPERSONATION_DISABLED raise ImpersonationDisabled @@ -277,6 +280,30 @@ module Gitlab PersonalAccessToken.find_by_token(password) end + def find_feed_token_user(token) + find_user_from_path_feed_token(token) || User.find_by_feed_token(token) + end + + def find_user_from_path_feed_token(token) + glft = token.match(PATH_DEPENDENT_FEED_TOKEN_REGEX) + + return unless glft + + # make sure that user id uses decimal notation + user_id = glft[2].to_i(10) + digest = glft[1] + + user = User.find_by_id(user_id) + return unless user + + feed_token = user.feed_token + our_digest = OpenSSL::HMAC.hexdigest("SHA256", feed_token, current_request.path) + + return unless ActiveSupport::SecurityUtils.secure_compare(digest, our_digest) + + user + end + def parsed_oauth_token Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) end @@ -374,6 +401,12 @@ module Gitlab raise UnauthorizedError unless job end end + + def revoke_token_family(token) + return unless Feature.enabled?(:pat_reuse_detection) + + PersonalAccessTokens::RevokeTokenFamilyService.new(token).execute + end end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 30896637eff..13ca4f01154 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -264,7 +264,7 @@ module Gitlab return {} unless options['tls_options'] # Dup so we don't overwrite the original value - custom_options = options['tls_options'].dup.delete_if { |_, value| value.nil? || value.blank? } + custom_options = options['tls_options'].to_hash.delete_if { |_, value| value.nil? || value.blank? } custom_options.symbolize_keys! if custom_options[:cert] diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml index 116c84c3759..9424686340f 100644 --- a/lib/gitlab/background_migration/.rubocop.yml +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -59,3 +59,8 @@ Migration/BackgroundMigrationBaseClass: - 'base_job.rb' - 'batched_migration_job.rb' - 'logger.rb' + +BackgroundMigration/AvoidSilentRescueExceptions: + Enabled: true + Description: >- + Rescuing errors in batched background migration jobs can lead to undesired results diff --git a/lib/gitlab/background_migration/backfill_missing_ci_cd_settings.rb b/lib/gitlab/background_migration/backfill_missing_ci_cd_settings.rb new file mode 100644 index 00000000000..e3ad63aac2e --- /dev/null +++ b/lib/gitlab/background_migration/backfill_missing_ci_cd_settings.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # backfills project_ci_cd_settings + class BackfillMissingCiCdSettings < BatchedMigrationJob + # migrations only version of `project_ci_cd_settings` table + class ProjectCiCdSetting < ::ApplicationRecord + self.table_name = 'project_ci_cd_settings' + end + + operation_name :backfill_missing_ci_cd_settings + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch = sub_batch.where(%{ + NOT EXISTS ( + SELECT 1 + FROM project_ci_cd_settings + WHERE project_ci_cd_settings.project_id = projects.id + ) + }) + next unless sub_batch.present? + + ci_cd_attributes = sub_batch.map do |project| + { + project_id: project.id, + default_git_depth: 20, + forward_deployment_enabled: true + } + end + + ProjectCiCdSetting.insert_all(ci_cd_attributes) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_uuid_conversion_column_in_vulnerability_occurrences.rb b/lib/gitlab/background_migration/backfill_uuid_conversion_column_in_vulnerability_occurrences.rb new file mode 100644 index 00000000000..4dccd3fd852 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_uuid_conversion_column_in_vulnerability_occurrences.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This batched background migration will backfill values for `uuid_convert_string_to_uuid` column in + # vulnerability_occurrences table to allow us to migrate the column type from `varchar(36)` to `uuid` + class BackfillUuidConversionColumnInVulnerabilityOccurrences < BatchedMigrationJob + operation_name :backfill_uuid_conversion_column_in_vulnerability_occurrences + scope_to ->(relation) do + relation.where("uuid_convert_string_to_uuid = '00000000-0000-0000-0000-000000000000'::uuid") + end + feature_category :vulnerability_management + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all("uuid_convert_string_to_uuid = uuid::uuid") + end + end + end + end +end diff --git a/lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl.rb b/lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl.rb new file mode 100644 index 00000000000..2672498b627 --- /dev/null +++ b/lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Redis + # BackfillProjectPipelineStatusTtl cleans up keys written by + # Gitlab::Cache::Ci::ProjectPipelineStatus by adding a minimum 8-hour ttl + # to all keys. This either sets or extends the ttl of matching keys. + # + class BackfillProjectPipelineStatusTtl # rubocop:disable Migration/BackgroundMigrationBaseClass + def perform(keys) + # spread out deletes over a 4 hour period starting in 8 hours time + ttl_duration = 10.hours.to_i + ttl_jitter = 2.hours.to_i + + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + keys.each { |key| pipeline.expire(key, ttl_duration + rand(-ttl_jitter..ttl_jitter)) } + end + end + end + + def scan_match_pattern + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:*:pipeline_status" + end + + def redis + @redis ||= ::Redis.new(Gitlab::Redis::Cache.params) + end + end + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index b2630a7ad7a..4beb8f54abf 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -9,6 +9,8 @@ module Gitlab class ProjectPipelineStatus include Gitlab::Utils::StrongMemoize + STATUS_KEY_TTL = 8.hours + attr_accessor :sha, :status, :ref, :project, :loaded def self.load_for_project(project) @@ -89,12 +91,17 @@ module Gitlab self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) self.status = nil if self.status.empty? + + redis.expire(cache_key, STATUS_KEY_TTL) end end def store_in_cache with_redis do |redis| - redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + redis.pipelined do |p| + p.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + p.expire(cache_key, STATUS_KEY_TTL) + end end end diff --git a/lib/gitlab/cache/client.rb b/lib/gitlab/cache/client.rb index 37d6cac8d43..1e2962a5151 100644 --- a/lib/gitlab/cache/client.rb +++ b/lib/gitlab/cache/client.rb @@ -5,61 +5,40 @@ module Gitlab # It replaces Rails.cache with metrics support class Client DEFAULT_BACKING_RESOURCE = :unknown + DEFAULT_FEATURE_CATEGORY = :not_owned - # Build Cache client with the metadata support - # - # @param cache_identifier [String] defines the location of the cache definition - # Example: "ProtectedBranches::CacheService#fetch" - # @param feature_category [Symbol] name of the feature category (from config/feature_categories.yml) - # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES) - # @return [Gitlab::Cache::Client] - def self.build_with_metadata( - cache_identifier:, - feature_category:, - backing_resource: DEFAULT_BACKING_RESOURCE - ) - new(Metadata.new( - cache_identifier: cache_identifier, - feature_category: feature_category, - backing_resource: backing_resource - )) - end - - def initialize(metadata, backend: Rails.cache) - @metadata = metadata - @metrics = Metrics.new(metadata) + def initialize(metrics, backend: Rails.cache) + @metrics = metrics @backend = backend end - def read(name) - read_result = backend.read(name) + def read(name, options = nil, labels = {}) + read_result = backend.read(name, options) if read_result.nil? - metrics.increment_cache_miss + metrics.increment_cache_miss(labels) else - metrics.increment_cache_hit + metrics.increment_cache_hit(labels) end read_result end - def fetch(name, options = nil, &block) - read_result = read(name) + def fetch(name, options = nil, labels = {}, &block) + read_result = read(name, options, labels) return read_result unless block || read_result backend.fetch(name, options) do - metrics.observe_cache_generation(&block) + metrics.observe_cache_generation(labels, &block) end end delegate :write, :exist?, :delete, to: :backend - attr_reader :metadata, :metrics - private - attr_reader :backend + attr_reader :metrics, :backend end end end diff --git a/lib/gitlab/cache/metadata.rb b/lib/gitlab/cache/metadata.rb index de35b332300..03ee48399d9 100644 --- a/lib/gitlab/cache/metadata.rb +++ b/lib/gitlab/cache/metadata.rb @@ -12,12 +12,12 @@ module Gitlab # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES) # @return [Gitlab::Cache::Metadata] def initialize( - cache_identifier:, - feature_category:, + cache_identifier: nil, + feature_category: Client::DEFAULT_FEATURE_CATEGORY, backing_resource: Client::DEFAULT_BACKING_RESOURCE ) @cache_identifier = cache_identifier - @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) + @feature_category = fetch_feature_category!(feature_category) @backing_resource = fetch_backing_resource!(backing_resource) end @@ -25,6 +25,12 @@ module Gitlab private + def fetch_feature_category!(feature_category) + return feature_category if feature_category == Client::DEFAULT_FEATURE_CATEGORY + + Gitlab::FeatureCategories.default.get!(feature_category) + end + def fetch_backing_resource!(resource) return resource if VALID_BACKING_RESOURCES.include?(resource) diff --git a/lib/gitlab/cache/metrics.rb b/lib/gitlab/cache/metrics.rb index d9c80f076b9..26a1f346f13 100644 --- a/lib/gitlab/cache/metrics.rb +++ b/lib/gitlab/cache/metrics.rb @@ -12,14 +12,14 @@ module Gitlab # Increase cache hit counter # - def increment_cache_hit - counter.increment(labels.merge(cache_hit: true)) + def increment_cache_hit(labels = {}) + counter.increment(base_labels.merge(labels, cache_hit: true)) end # Increase cache miss counter # - def increment_cache_miss - counter.increment(labels.merge(cache_hit: false)) + def increment_cache_miss(labels = {}) + counter.increment(base_labels.merge(labels, cache_hit: false)) end # Measure the duration of cacheable action @@ -29,12 +29,12 @@ module Gitlab # cacheable_action # end # - def observe_cache_generation(&block) + def observe_cache_generation(labels = {}, &block) real_start = Gitlab::Metrics::System.monotonic_time value = yield - histogram.observe({}, Gitlab::Metrics::System.monotonic_time - real_start) + histogram.observe(base_labels.merge(labels), Gitlab::Metrics::System.monotonic_time - real_start) value end @@ -44,20 +44,24 @@ module Gitlab attr_reader :cache_metadata def counter - @counter ||= Gitlab::Metrics.counter(:redis_hit_miss_operations_total, "Hit/miss Redis cache counter") + @counter ||= Gitlab::Metrics.counter( + :redis_hit_miss_operations_total, + "Hit/miss Redis cache counter", + base_labels + ) end def histogram @histogram ||= Gitlab::Metrics.histogram( :redis_cache_generation_duration_seconds, 'Duration of Redis cache generation', - labels, + base_labels, DEFAULT_BUCKETS ) end - def labels - @labels ||= { + def base_labels + @base_labels ||= { cache_identifier: cache_metadata.cache_identifier, feature_category: cache_metadata.feature_category, backing_resource: cache_metadata.backing_resource diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 194e3f6e938..3fd7e44985e 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -117,6 +117,7 @@ module Gitlab def bulk_access_checks! Gitlab::Checks::LfsCheck.new(self).validate! + Gitlab::Checks::GlobalFileSizeCheck.new(self).validate! end def blank_rev?(rev) diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 1186b532baf..bce4f969284 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -74,7 +74,7 @@ module Gitlab lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user_access.user.id).take if lfs_lock - return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.username}" end end end diff --git a/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb new file mode 100644 index 00000000000..78f1716274e --- /dev/null +++ b/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + module FileSizeCheck + class AllowExistingOversizedBlobs + def initialize(project:, changes:, file_size_limit_megabytes:) + @project = project + @changes = changes + @oldrevs = changes.pluck(:oldrev).compact # rubocop:disable CodeReuse/ActiveRecord just plucking from an array + @file_size_limit_megabytes = file_size_limit_megabytes + end + + def find(timeout: nil) + oversize_blobs = any_oversize_blobs.find(timeout: timeout) + + return oversize_blobs unless oldrevs.present? + + revs_paths = oldrevs.product(oversize_blobs.map(&:path)) + existing_blobs = project.repository.blobs_at(revs_paths, blob_size_limit: 1) + map_existing_path_to_size = existing_blobs.group_by(&:path).transform_values { |blobs| blobs.map(&:size).max } + + # return blobs that are going to be over the limit that were previously within the limit + oversize_blobs.select { |blob| map_existing_path_to_size.fetch(blob.path, 0) <= file_size_limit_megabytes } + end + + private + + attr_reader :project, :changes, :newrevs, :oldrevs, :file_size_limit_megabytes + + def any_oversize_blobs + AnyOversizedBlobs.new(project: project, changes: changes, + file_size_limit_megabytes: file_size_limit_megabytes) + end + end + end + end +end diff --git a/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb new file mode 100644 index 00000000000..35f969dbb46 --- /dev/null +++ b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + module FileSizeCheck + class AnyOversizedBlobs + def initialize(project:, changes:, file_size_limit_megabytes:) + @project = project + @newrevs = changes.pluck(:newrev).compact # rubocop:disable CodeReuse/ActiveRecord just plucking from an array + @file_size_limit_megabytes = file_size_limit_megabytes + end + + def find(timeout: nil) + blobs = project.repository.new_blobs(newrevs, dynamic_timeout: timeout) + + blobs.select do |blob| + ::Gitlab::Utils.bytes_to_megabytes(blob.size) > file_size_limit_megabytes + end + end + + private + + attr_reader :project, :newrevs, :file_size_limit_megabytes + end + end + end +end diff --git a/lib/gitlab/checks/global_file_size_check.rb b/lib/gitlab/checks/global_file_size_check.rb new file mode 100644 index 00000000000..418d2d32b57 --- /dev/null +++ b/lib/gitlab/checks/global_file_size_check.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class GlobalFileSizeCheck < BaseBulkChecker + MAX_FILE_SIZE_MB = 100 + LOG_MESSAGE = 'Checking for blobs over the file size limit' + + def validate! + return unless Feature.enabled?(:global_file_size_check, project) + + Gitlab::AppJsonLogger.info(LOG_MESSAGE) + logger.log_timed(LOG_MESSAGE) do + Gitlab::Checks::FileSizeCheck::AllowExistingOversizedBlobs.new( + project: project, + changes: changes, + file_size_limit_megabytes: MAX_FILE_SIZE_MB + ).find + + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/393535 + # - set limit per plan tier + # - raise an error if large blobs are found + end + + true + end + end + end +end diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index 2eb8df01d58..0d8f6f3ea40 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -14,7 +14,8 @@ module Gitlab def initialize(job) @job = job - raise ArgumentError, 'Job does not have artifacts' unless @job.artifacts? + raise Error, 'Job doesnt exist' unless @job + raise Error, 'Job does not have artifacts' unless @job.artifacts? validate! end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index 17b9f30db33..8b503290e6e 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -32,18 +32,13 @@ module Gitlab if @rule_list.nil? Result.new(when: @default_when) elsif matched_rule = match_rule(pipeline, context) - result = Result.new( + Result.new( when: matched_rule.attributes[:when] || @default_when, start_in: matched_rule.attributes[:start_in], allow_failure: matched_rule.attributes[:allow_failure], - variables: matched_rule.attributes[:variables] + variables: matched_rule.attributes[:variables], + needs: matched_rule.attributes[:needs] ) - - if Feature.enabled?(:introduce_rules_with_needs, pipeline.project) - result.needs = matched_rule.attributes[:needs] - end - - result else Result.new(when: 'never') end diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index 27a7611ffdd..e0ef598da1b 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -73,9 +73,7 @@ module Gitlab end def latest_version_sha - return unless catalog_resource = project&.catalog_resource - - catalog_resource.latest_version&.sha + project.releases.latest&.sha end end end diff --git a/lib/gitlab/ci/config/README.md b/lib/gitlab/ci/config/README.md new file mode 100644 index 00000000000..e850afc5253 --- /dev/null +++ b/lib/gitlab/ci/config/README.md @@ -0,0 +1,178 @@ +# `::Gitlab::Ci::Config` module overview + +`::Gitlab::Ci::Config` is a concrete implementation of abstract +`::Gitlab::Config` module. It's being used to build, traverse and translate +hierarchical, user-provided, CI configuration, usually provided in +`.gitlab-ci.yml` and included files. + +## High-level Overview + +`::Gitlab::Ci::Config` is an indirection layer between user-provided data and +GitLab itself. + +1. A user provides YAML configuration in `.gitlab-ci.yml` and all included files. +1. `::Gitlab::Ci::Config` loads the provided YAML using Ruby standard `Psych` library. +1. The resulting Hash is then passed to the module to build an Abstract Syntax Tree. +1. The module validates, transforms, translates and augments the data to build + a stable representation of user-provided configuration. + +This additional layer helps us to validate the user-provided configuration and +surface any errors to a user if it is not valid. In case of a valid +configuration, it makes it possible to build a stable representation of +config that we can depend on. + +For example, both following configurations using the +[environment](https://docs.gitlab.com/ee/ci/yaml/#environment) +keyword are correct: + +```yaml +# First way to define an environment: + +deploy: + environment: production + script: cap deploy + +# Second way to define an environment: + +deploy: + environment: + name: production + url: https://prod.example.com + kubernetes: + namespace: production +``` + +This demonstrates the concept of hidden / expanding complexity: if users need +more flexibility, they can opt-in into using a much more elaborate syntax to +configure their environments. **We use this technique to make it possible for +simplicity to coexist with flexibility without additional complexity**. + +`::Gitlab::Ci::Config` allows us to achieve this, because it is an indirection +layer, that translates user-provided configuration into a known and expected +format when users can achieve the same thing in `.gitlab-ci.yml` in a few +different ways. + +## Hierarchical configuration + +`.gitlab-ci.yml` configuration is hierarchical but same keywords can often be +used on different levels in the hierarchy. `::Gitlab::Ci::Config` module makes +it easier to manage the complexity that stems from having same keyword +available in [many different places](https://docs.gitlab.com/ee/ci/yaml/#default): + +```yaml +default: + image: ruby:3.0 + +rspec: + script: bundle exec rspec + +rspec 2.7: + image: ruby:2.7 + script: bundle exec rspec +``` + +We can achieve that, because in `::Gitlab::Ci::Config` most of the keywords are +implemented within separate Ruby classes, that then can be reused: + +```ruby +# Simplified version of an entry class that describes a Docker image. +# +class Gitlab::Ci::Config::Entry + class Image < ::Gitlab::Config::Entry::Node + + validates :config, allowed_keys: ALLOWED_IMAGE_CONFIG_KEYS + + def value + if string? + { name: @config } + elsif hash? + { + name: @config[:name], + entrypoint: @config[:entrypoint], + ports: (ports_value if ports_defined?), + pull_policy: pull_policy_value + } + else + {} + end + end + end +end +``` + +The config above is a simple demonstration of the translation layer, into a +stable configuration, depending on what simplification strategy has been used +by a user. There more complex examples, though: + +```ruby +module Gitlab::Ci::Config::Entry + class Need < ::Gitlab::Config::Entry::Simplifiable + strategy :JobString, if: -> (config) { config.is_a?(String) } + + strategy :JobHash, + if: -> (config) { config.is_a?(Hash) && same_pipeline_need?(config) } + + strategy :CrossPipelineDependency, + if: -> (config) { config.is_a?(Hash) && cross_pipeline_need?(config) } + + # [ ... ] + end +end +``` + +Every time we load config, an Abstract Syntax Tree is being built, because +nodes / entries know what the child nodes can be: + +```ruby +# Simplified root entry code +# +module Gitlab::Ci::Config::Entry + class Root < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + entry :default, Entry::Default, + description: 'Default configuration for all jobs.' + + entry :include, Entry::Includes, + description: 'List of external YAML files to include.' + + entry :before_script, Entry::Commands, + description: 'Script that will be executed before each job.' + + entry :image, Entry::Image, + description: 'Docker image that will be used to execute jobs.' + + entry :services, Entry::Services, + description: 'Docker images that will be linked to the container.' + + entry :after_script, Entry::Commands, + description: 'Script that will be executed after each job.' + + entry :variables, Entry::Variables, + description: 'Environment variables that will be used.' + + # [ ... ] + end +end +``` + +Loading the configuration script mentioned at the beginning of this pargraph +will result in build a following AST: + +``` +Entry::Root +`- + |- Entry::Default + | `- Entry::Image('ruby:3.0') + | + |- Entry::Job('rspec') + | `- Entry::Script('bundle exec rspec') + | + |- Entry::Job('rspec 2.7') + | |- Entry::Image('ruby:2.7) + | `- Entry::Script('bundle exec rspec') +``` + +The AST will be validated, and eventually will generate a stable representation +of configuration that we can use to persist pipelines / stages / jobs in the +database, and start pipeline processing. diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 273d78bd583..f23fa2e6401 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -19,12 +19,15 @@ module Gitlab end def content - strong_memoize(:content) do - Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) - rescue Gitlab::Ci::ArtifactFileReader::Error => error - errors.push(error.message) # TODO this memoizes the error message as a content! - end + return unless context.parent_pipeline.present? + + Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) + rescue Gitlab::Ci::ArtifactFileReader::Error => error + errors.push(error.message) + + nil end + strong_memoize_attr :content def metadata super.merge( diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 61d95c8d4e6..8bcb2a389d2 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -90,15 +90,7 @@ module Gitlab end def load_and_validate_expanded_hash! - context.logger.instrument(:config_file_fetch_content_hash) do - content_result # calling the method loads YAML then memoizes the content result - end - - context.logger.instrument(:config_file_interpolate_result) do - interpolator.interpolate! - end - - return validate_interpolation! unless interpolator.valid? + return errors.push("`#{masked_location}`: #{content_result.error}") unless content_result.valid? context.logger.instrument(:config_file_expand_content_includes) do expanded_content_hash # calling the method expands then memoizes the result @@ -109,36 +101,24 @@ module Gitlab protected - def content_result - ::Gitlab::Ci::Config::Yaml - .load_result!(content, project: context.project) - end - strong_memoize_attr :content_result - def content_inputs # TODO: remove support for `with` syntax in 16.1, see https://gitlab.com/gitlab-org/gitlab/-/issues/408369 # In the interim prefer `inputs` over `with` while allow either syntax. params.to_h.slice(:inputs, :with).each_value.first end - strong_memoize_attr :content_inputs - - def content_hash - interpolator.interpolate! - interpolator.to_hash - end - strong_memoize_attr :content_hash - - def interpolator - Yaml::Interpolator.new(content_result, content_inputs, context) + def content_result + context.logger.instrument(:config_file_fetch_content_hash) do + ::Gitlab::Ci::Config::Yaml::Loader.new(content, inputs: content_inputs, current_user: context.user).load + end end - strong_memoize_attr :interpolator + strong_memoize_attr :content_result def expanded_content_hash - return if content_hash.blank? + return if content_result.content.blank? strong_memoize(:expanded_content_hash) do - expand_includes(content_hash) + expand_includes(content_result.content) end end @@ -148,12 +128,6 @@ module Gitlab end end - def validate_interpolation! - return if interpolator.valid? - - errors.push("`#{masked_location}`: #{interpolator.error_message}") - end - def expand_includes(hash) External::Processor.new(hash, context.mutate(expand_context_attrs)).perform end diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 134306332e6..59e666b8bb5 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -17,16 +17,12 @@ module Gitlab end def evaluate(context) - if Feature.enabled?(:ci_support_include_rules_when_never, context.project) - if @rule_list.nil? - Result.new('always') - elsif matched_rule = match_rule(context) - Result.new(matched_rule.attributes[:when]) - else - Result.new('never') - end + if @rule_list.nil? + Result.new('always') + elsif matched_rule = match_rule(context) + Result.new(matched_rule.attributes[:when]) else - LegacyResult.new(@rule_list.nil? || match_rule(context)) + Result.new('never') end end @@ -55,12 +51,6 @@ module Gitlab self.when != 'never' end end - - LegacyResult = Struct.new(:result) do - def pass? - !!result - end - end end end end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index f74ef95a832..e3010ac3fdb 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -4,21 +4,17 @@ module Gitlab module Ci class Config module Yaml + LoadError = Class.new(StandardError) + class << self - def load!(content, project: nil) - Loader.new(content, project: project).to_result.then do |result| - ## - # raise an error for backwards compatibility - # - raise result.error unless result.valid? + def load!(content, current_user: nil) + Loader.new(content, current_user: current_user).load.then do |result| + raise result.error_class, result.error if !result.valid? && result.error_class.present? + raise LoadError, result.error unless result.valid? result.content end end - - def load_result!(content, project: nil) - Loader.new(content, project: project).to_result - end end end end diff --git a/lib/gitlab/ci/config/yaml/interpolator.rb b/lib/gitlab/ci/config/yaml/interpolator.rb index 4ae191dfedf..2909c2ac798 100644 --- a/lib/gitlab/ci/config/yaml/interpolator.rb +++ b/lib/gitlab/ci/config/yaml/interpolator.rb @@ -5,42 +5,23 @@ module Gitlab class Config module Yaml ## - # Config::Yaml::Interpolation performs includable file interpolation, and surfaces all possible interpolation + # Config::Yaml::Interpolator performs CI config file interpolation, and surfaces all possible interpolation # errors. It is designed to provide an external file's validation context too. # class Interpolator - include ::Gitlab::Utils::StrongMemoize + attr_reader :config, :args, :current_user, :errors - attr_reader :config, :args, :ctx, :errors - - def initialize(config, args, ctx = nil) + def initialize(config, args, current_user: nil) @config = config @args = args.to_h - @ctx = ctx + @current_user = current_user @errors = [] - - validate! end def valid? @errors.none? end - def ready? - ## - # Interpolation is ready when it has been either interrupted by an error or finished with a result. - # - @result || @errors.any? - end - - def interpolate? - enabled? && has_header? && valid? - end - - def has_header? - config.has_header? && config.header.present? - end - def to_hash @result.to_h end @@ -55,43 +36,25 @@ module Gitlab @errors.first(3).join(', ') end - ## - # TODO Add `instrument.logger` instrumentation blocks: - # https://gitlab.com/gitlab-org/gitlab/-/issues/396722 - # def interpolate! - return {} unless valid? - return @result ||= content.to_h unless interpolate? + return @errors.push(config.error) unless config.valid? + return @result ||= config.content unless config.has_header? return @errors.concat(header.errors) unless header.valid? return @errors.concat(inputs.errors) unless inputs.valid? return @errors.concat(context.errors) unless context.valid? return @errors.concat(template.errors) unless template.valid? - if ctx&.user - ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('ci_interpolation_users', values: ctx.user.id) + if current_user.present? + ::Gitlab::UsageDataCounters::HLLRedisCounter + .track_event('ci_interpolation_users', values: current_user.id) end @result ||= template.interpolated.to_h.deep_symbolize_keys end - strong_memoize_attr :interpolate! private - def validate! - return errors.push('content does not have a valid YAML syntax') unless config.valid? - - return unless has_header? && !enabled? - - errors.push('can not evaluate included file because interpolation is disabled') - end - - def enabled? - return false if ctx.nil? - - ::Feature.enabled?(:ci_includable_files_interpolation, ctx.project) - end - def header @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header| header.key = 'header' diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb index 924a1f2e46b..fb24a2874e4 100644 --- a/lib/gitlab/ci/config/yaml/loader.rb +++ b/lib/gitlab/ci/config/yaml/loader.rb @@ -5,33 +5,45 @@ module Gitlab class Config module Yaml class Loader + include Gitlab::Utils::StrongMemoize + AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 - def initialize(content, project: nil) + def initialize(content, inputs: {}, current_user: nil) @content = content - @project = project + @current_user = current_user + @inputs = inputs end - def to_result - Yaml::Result.new(config: load!, error: nil) - rescue ::Gitlab::Config::Loader::FormatError => e - Yaml::Result.new(error: e) - end + def load + yaml_result = load_uninterpolated_yaml - private + return yaml_result unless yaml_result.valid? - attr_reader :content, :project + interpolator = Yaml::Interpolator.new(yaml_result, inputs, current_user: current_user) - def ensure_custom_tags - @ensure_custom_tags ||= begin - AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + interpolator.interpolate! - true + if interpolator.valid? + # This Result contains only the interpolated config and does not have a header + Yaml::Result.new(config: interpolator.to_hash, error: nil) + else + Yaml::Result.new(error: interpolator.error_message) end end - def load! + private + + attr_reader :content, :current_user, :inputs + + def load_uninterpolated_yaml + Yaml::Result.new(config: load_yaml!, error: nil) + rescue ::Gitlab::Config::Loader::FormatError => e + Yaml::Result.new(error: e.message, error_class: e) + end + + def load_yaml! ensure_custom_tags ::Gitlab::Config::Loader::MultiDocYaml.new( @@ -41,6 +53,14 @@ module Gitlab reject_empty: true ).load! end + + def ensure_custom_tags + @ensure_custom_tags ||= begin + AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + + true + end + end end end end diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb index 6b53adc3a57..6b20eeae203 100644 --- a/lib/gitlab/ci/config/yaml/result.rb +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -5,11 +5,12 @@ module Gitlab class Config module Yaml class Result - attr_reader :error + attr_reader :error, :error_class - def initialize(config: nil, error: nil) + def initialize(config: nil, error: nil, error_class: nil) @config = Array.wrap(config) @error = error + @error_class = error_class end def valid? diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 9e71a9e8e91..6ce662bdead 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -3,6 +3,8 @@ module Gitlab module Ci class JwtV2 < Jwt + include Gitlab::Utils::StrongMemoize + DEFAULT_AUD = Settings.gitlab.base_url GITLAB_HOSTED_RUNNER = 'gitlab-hosted' SELF_HOSTED_RUNNER = 'self-hosted' @@ -48,31 +50,35 @@ module Gitlab sha: pipeline.sha } - if Feature.enabled?(:ci_jwt_v2_ref_uri_claim, pipeline.project) + if project_config&.source == :repository_source additional_claims[:ci_config_ref_uri] = ci_config_ref_uri + additional_claims[:ci_config_sha] = pipeline.sha end super.merge(additional_claims) end def ci_config_ref_uri - project_config = Gitlab::Ci::ProjectConfig.new( + "#{project_config&.url}@#{pipeline.source_ref_path}" + rescue StandardError => e + # We don't want endpoints relying on this code to fail if there's an error here. + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, pipeline_id: pipeline.id) + nil + end + + def project_config + Gitlab::Ci::ProjectConfig.new( project: project, sha: pipeline.sha, pipeline_source: pipeline.source&.to_sym, pipeline_source_bridge: pipeline.source_bridge ) - - return unless project_config&.source == :repository_source - - "#{project_config.url}@#{pipeline.source_ref_path}" - - # Errors are rescued to mitigate risk. This can be removed if no errors are observed. - # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117923#note_1387660746 for context. rescue StandardError => e + # We don't want endpoints relying on this code to fail if there's an error here. Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, pipeline_id: pipeline.id) nil end + strong_memoize_attr(:project_config) def runner_environment return unless runner diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 21408beb8cb..ee1da82f285 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -126,8 +126,8 @@ module Gitlab compare_key: data['cve'] || '', location: location, evidence: evidence, - severity: parse_severity_level(data['severity']), - confidence: parse_confidence_level(data['confidence']), + severity: ::Enums::Vulnerability.parse_severity_level(data['severity']), + confidence: ::Enums::Vulnerability.parse_confidence_level(data['confidence']), scanner: create_scanner(top_level_scanner_data || data['scanner']), scan: report&.scan, identifiers: identifiers, @@ -260,14 +260,6 @@ module Gitlab ::Gitlab::Ci::Reports::Security::Link.new(name: link['name'], url: link['url']) end - def parse_severity_level(input) - input&.downcase.then { |value| ::Enums::Vulnerability.severity_levels.key?(value) ? value : 'unknown' } - end - - def parse_confidence_level(input) - input&.downcase.then { |value| ::Enums::Vulnerability.confidence_levels.key?(value) ? value : 'unknown' } - end - def create_location(location_data) raise NotImplementedError end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 92d9d170575..e39482481c7 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6] + cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6] }.freeze VERSIONS_TO_REMOVE_IN_17_0 = %w[].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..5563acbe232 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/cluster-image-scanning-report-format.json @@ -0,0 +1,1035 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/container-scanning-report-format.json new file mode 100644 index 00000000000..820811100ea --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/container-scanning-report-format.json @@ -0,0 +1,967 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..63e39395772 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/coverage-fuzzing-report-format.json @@ -0,0 +1,925 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dast-report-format.json new file mode 100644 index 00000000000..86e62558a39 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dast-report-format.json @@ -0,0 +1,1330 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dependency-scanning-report-format.json new file mode 100644 index 00000000000..c08cbcffc5b --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dependency-scanning-report-format.json @@ -0,0 +1,1033 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/sast-report-format.json new file mode 100644 index 00000000000..f1869950d20 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/sast-report-format.json @@ -0,0 +1,920 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/secret-detection-report-format.json new file mode 100644 index 00000000000..e9bfd6186a3 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/secret-detection-report-format.json @@ -0,0 +1,944 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 035167f1a74..b8b70a6b6b6 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -22,7 +22,7 @@ module Gitlab return error('Insufficient permissions to create a new pipeline') end - unless allowed_to_write_ref? + unless allowed_to_run_pipeline? error("You do not have sufficient permission to run a pipeline on '#{command.ref}'. Please select a different branch or contact your administrator for assistance.") end end @@ -37,6 +37,10 @@ module Gitlab can?(current_user, :create_pipeline, project) end + def allowed_to_run_pipeline? + allowed_to_write_ref? + end + def allowed_to_write_ref? access = Gitlab::UserAccess.new(current_user, container: project) diff --git a/lib/gitlab/ci/project_config/bridge.rb b/lib/gitlab/ci/project_config/bridge.rb index c342ab2c215..45aa330508f 100644 --- a/lib/gitlab/ci/project_config/bridge.rb +++ b/lib/gitlab/ci/project_config/bridge.rb @@ -10,6 +10,11 @@ module Gitlab pipeline_source_bridge.yaml_for_downstream end + # Bridge.yaml_for_downstream injects an `include` + def internal_include_prepended? + true + end + def source :bridge_source end diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb index 7dfd528fd6f..a08cf27b74c 100644 --- a/lib/gitlab/ci/project_config/repository.rb +++ b/lib/gitlab/ci/project_config/repository.rb @@ -4,6 +4,8 @@ module Gitlab module Ci class ProjectConfig class Repository < Source + extend ::Gitlab::Utils::Override + def content strong_memoize(:content) do next unless file_in_repository? diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb index 68853ca8296..5f37c3bad7b 100644 --- a/lib/gitlab/ci/project_config/source.rb +++ b/lib/gitlab/ci/project_config/source.rb @@ -5,7 +5,6 @@ module Gitlab class ProjectConfig class Source include Gitlab::Utils::StrongMemoize - extend ::Gitlab::Utils::Override def initialize(project, sha, custom_content, pipeline_source, pipeline_source_bridge) @project = project diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb index fbb8644c1b0..b7af6ea17c3 100644 --- a/lib/gitlab/ci/reports/sbom/source.rb +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -11,6 +11,22 @@ module Gitlab @source_type = type @data = data end + + def source_file_path + data.dig('source_file', 'path') + end + + def input_file_path + data.dig('input_file', 'path') + end + + def packager + data.dig('package_manager', 'name') + end + + def language + data.dig('language', 'name') + end end end end diff --git a/lib/gitlab/ci/reports/security/link.rb b/lib/gitlab/ci/reports/security/link.rb index 1c4c05cd9ac..6804d2b2a29 100644 --- a/lib/gitlab/ci/reports/security/link.rb +++ b/lib/gitlab/ci/reports/security/link.rb @@ -18,6 +18,10 @@ module Gitlab url: url }.compact end + + def ==(other) + name == other.name && url == other.url + end end end end diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 49d3c270bac..46d0b92b243 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.37.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 49d3c270bac..46d0b92b243 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.37.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index f4a13d61ba2..b1e498a9d09 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index c1a3daa7f5b..5a7e69b62d9 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index a3c7c6baf02..dac559db8d5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index f16c28e7b60..87d0894b67a 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -20,18 +20,18 @@ cache: paths: - ${TF_ROOT}/.terraform/ -.terraform:fmt: &terraform_fmt +.terraform:fmt: stage: validate script: - gitlab-terraform fmt allow_failure: true -.terraform:validate: &terraform_validate +.terraform:validate: stage: validate script: - gitlab-terraform validate -.terraform:build: &terraform_build +.terraform:build: stage: build script: - gitlab-terraform plan @@ -46,7 +46,7 @@ cache: reports: terraform: ${TF_ROOT}/plan.json -.terraform:deploy: &terraform_deploy +.terraform:deploy: stage: deploy script: - gitlab-terraform apply @@ -56,7 +56,7 @@ cache: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: manual -.terraform:destroy: &terraform_destroy +.terraform:destroy: stage: cleanup script: - gitlab-terraform destroy diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 793030d302a..d2b929cf995 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -22,7 +22,7 @@ variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project TF_STATE_NAME: default # The name of the state file used by the GitLab Managed Terraform state backend -.terraform:fmt: &terraform_fmt +.terraform:fmt: stage: validate script: - gitlab-terraform fmt @@ -33,7 +33,7 @@ variables: when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. -.terraform:validate: &terraform_validate +.terraform:validate: stage: validate script: - gitlab-terraform validate @@ -43,7 +43,7 @@ variables: when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. -.terraform:build: &terraform_build +.terraform:build: stage: build script: - gitlab-terraform plan @@ -63,7 +63,7 @@ variables: when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. -.terraform:deploy: &terraform_deploy +.terraform:deploy: stage: deploy script: - gitlab-terraform apply @@ -73,7 +73,7 @@ variables: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: manual -.terraform:destroy: &terraform_destroy +.terraform:destroy: stage: cleanup script: - gitlab-terraform destroy diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index fb0d300338b..ae2edd6f3fa 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -37,9 +37,15 @@ publish: # Compare the version in package.json to all published versions. # If the package.json version has not yet been published, run `npm publish`. + # If $SIGSTORE_ID_TOKEN is set this template will generate a provenance + # document. For more information refer to the documentation: https://docs.gitlab.com/ee/ci/yaml/signing_examples/ - | if [[ "$(npm view ${NPM_PACKAGE_NAME} versions)" != *"'${NPM_PACKAGE_VERSION}'"* ]]; then - npm publish + if [[ -n "${SIGSTORE_ID_TOKEN}" ]]; then + npm publish --provenance + else + npm publish + fi echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" else echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 9960a6fbdf5..f77ef262236 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -8,6 +8,21 @@ module Gitlab attr_reader :errors + def self.fabricate(input) + case input + when Array + new(input) + when Hash + new(input.map { |key, value| { key: key, value: value } }) + when Proc + fabricate(input.call) + when self + input + else + raise ArgumentError, "Unknown `#{input.class}` variable collection!" + end + end + def initialize(variables = [], errors = nil) @variables = [] @variables_by_key = Hash.new { |h, k| h[k] = [] } diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 0fcf11121fa..73452d83bce 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -17,6 +17,10 @@ module Gitlab @variable = { key: key, value: value, public: public, file: file, masked: masked, raw: raw } end + def key + @variable.fetch(:key) + end + def value @variable.fetch(:value) end diff --git a/lib/gitlab/ci/variables/downstream/base.rb b/lib/gitlab/ci/variables/downstream/base.rb new file mode 100644 index 00000000000..6845ed4cc1b --- /dev/null +++ b/lib/gitlab/ci/variables/downstream/base.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Downstream + class Base + def initialize(context) + @context = context + end + + private + + attr_reader :context + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb b/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb new file mode 100644 index 00000000000..6690e9f1c1f --- /dev/null +++ b/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Downstream + class ExpandableVariableGenerator < Base + def for(item) + expanded_value = ::ExpandVariables.expand(item.value, context.all_bridge_variables) + + [{ key: item.key, value: expanded_value }] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/downstream/generator.rb b/lib/gitlab/ci/variables/downstream/generator.rb new file mode 100644 index 00000000000..93c995cc918 --- /dev/null +++ b/lib/gitlab/ci/variables/downstream/generator.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Downstream + class Generator + include Gitlab::Utils::StrongMemoize + + Context = Struct.new(:all_bridge_variables, keyword_init: true) + + def initialize(bridge) + @bridge = bridge + + context = Context.new(all_bridge_variables: bridge.variables) + + @raw_variable_generator = RawVariableGenerator.new(context) + @expandable_variable_generator = ExpandableVariableGenerator.new(context) + end + + def calculate + calculate_downstream_variables + .reverse # variables priority + .uniq { |var| var[:key] } # only one variable key to pass + .reverse + end + + private + + attr_reader :bridge, :all_bridge_variables + + def calculate_downstream_variables + # The order of this list refers to the priority of the variables + # The variables added later takes priority. + downstream_yaml_variables + + downstream_pipeline_variables + + downstream_pipeline_schedule_variables + end + + def downstream_yaml_variables + return [] unless bridge.forward_yaml_variables? + + build_downstream_variables_from(bridge.yaml_variables) + end + + def downstream_pipeline_variables + return [] unless bridge.forward_pipeline_variables? + + pipeline_variables = bridge.pipeline_variables.to_a + build_downstream_variables_from(pipeline_variables) + end + + def downstream_pipeline_schedule_variables + return [] unless bridge.forward_pipeline_variables? + + pipeline_schedule_variables = bridge.pipeline_schedule_variables.to_a + build_downstream_variables_from(pipeline_schedule_variables) + end + + def build_downstream_variables_from(variables) + Gitlab::Ci::Variables::Collection.fabricate(variables).flat_map do |item| + if item.raw? + @raw_variable_generator.for(item) + else + @expandable_variable_generator.for(item) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/downstream/raw_variable_generator.rb b/lib/gitlab/ci/variables/downstream/raw_variable_generator.rb new file mode 100644 index 00000000000..42c795b4398 --- /dev/null +++ b/lib/gitlab/ci/variables/downstream/raw_variable_generator.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Downstream + class RawVariableGenerator < Base + def for(item) + [{ key: item.key, value: item.value, raw: true }] + end + end + end + end + end +end diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index 6cadb9424f7..5811b6223a3 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -17,6 +17,13 @@ module Gitlab return end + if bucket_prefix.present? + error_message = "Uploads are configured with a bucket prefix '#{bucket_prefix}'.\n" + error_message += "Unfortunately, prefixes are not supported for this Rake task.\n" + # At the moment, Fog does not provide a cloud-agnostic way of iterating through a bucket with a prefix. + raise error_message + end + logger.info "Looking for orphaned remote uploads to remove#{'. Dry run' if dry_run}..." each_orphan_file do |file| @@ -77,6 +84,10 @@ module Gitlab def configuration Gitlab.config.uploads.object_store end + + def bucket_prefix + configuration.bucket_prefix + end end end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index b39d2a02f02..c6ce0aa6160 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative '../utils' # Gitlab::Utils +require 'gitlab/utils/all' # Gitlab::Utils module Gitlab module Cluster diff --git a/lib/gitlab/config/README.md b/lib/gitlab/config/README.md new file mode 100644 index 00000000000..355dbdc8cfe --- /dev/null +++ b/lib/gitlab/config/README.md @@ -0,0 +1,29 @@ +# `::Gitlab::Config` module overview + +`::Gitlab::Config` is an abstract module used to build, traverse and translate +any kind of hierarchical, user-provided configuration. + +The most complex and widely used implementation is `::Gitlab::Ci::Config` +facade class. Please see `lib/gitlab/ci/config/README.md` for more information +around how it works. + +## High-level Overview + +The main motivation behind how `::Gitlab::Config` and `::Gitlab::Ci::Config` +work is to build an indirection layer between complex user-provided +configuration and GitLab itself. This helps us to extend configuration keywords +in a backwards-compatible way, and make sure that validation and transformation +rules are encapsulated within domain classes, what significantly helps to +reduce cognitive load on Engineers working on that part of the codebase. + +`Gitlab::Config` is a tool to work with hierarchical configuration: + +1. First we parse YAML with Ruby standard library `Psych`. +1. The resulting hash is being used to initialize a concrete implementation of `Gitlab::Config`. +1. In `::Gitlab::Ci::Config` abstract classes from `::Gitlab::Config` have their implementations. +1. Each domain class represents one or a group of hierarchical YAML entries, like `job:artifacts`. +1. Each entry knows what subentires are supported and how to validate them. +1. Upon loading a configuration we build an abstract syntax tree, and validate configuration. +1. If there are errors, the module can surface them to a user. +1. In case of config being valid, the config gets translated and augmented. +1. The result is a consistent representation that we can depend on in other parts of the codebase. diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 8fec5cf3303..e1e9e4720bb 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -12,13 +12,14 @@ module Gitlab author_url = build_author_url(build.commit, commit) - data = { + { object_kind: 'build', ref: build.ref, tag: build.tag, before_sha: build.before_sha, sha: build.sha, + retries_count: build.retries_count, # TODO: should this be not prefixed with build_? # Leaving this way to have backward compatibility @@ -69,10 +70,6 @@ module Gitlab environment: build_environment(build) } - - data[:retries_count] = build.retries_count if Feature.enabled?(:job_webhook_retries_count, project) - - data end private diff --git a/lib/gitlab/data_builder/emoji.rb b/lib/gitlab/data_builder/emoji.rb new file mode 100644 index 00000000000..63562eca155 --- /dev/null +++ b/lib/gitlab/data_builder/emoji.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module Emoji + extend self + + def build(award_emoji, user, action) + project = award_emoji.awardable.project + data = build_base_data(project, user, award_emoji, action) + + if award_emoji.awardable.is_a?(::Note) + note = award_emoji.awardable + data[:note] = note.hook_attrs + noteable = note.noteable + else + noteable = award_emoji.awardable + end + + if noteable.respond_to?(:hook_attrs) + data[noteable.class.underscore.to_sym] = noteable.hook_attrs + else + Gitlab::AppLogger.error( + "Error building payload data for emoji webhook. #{noteable.class} does not respond to hook_attrs.") + end + + data + end + + def build_base_data(project, user, award_emoji, action) + base_data = { + object_kind: 'emoji', + event_type: action, + user: user.hook_attrs, + project_id: project.id, + project: project.hook_attrs, + object_attributes: award_emoji.hook_attrs + } + + base_data[:object_attributes][:awarded_on_url] = Gitlab::UrlBuilder.build(award_emoji.awardable) + base_data + end + end + end +end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index f941c57a6dd..46110937132 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -12,6 +12,7 @@ module Gitlab before: "95790bf891e76fee5e1747ab589903a6a1f80f22", after: "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", ref: "refs/heads/master", + ref_protected: true, checkout_sha: "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", message: "Hello World", user_id: 4, @@ -55,6 +56,7 @@ module Gitlab # before: String, # after: String, # ref: String, + # ref_protected: Boolean, # user_id: String, # user_name: String, # user_username: String, @@ -116,6 +118,7 @@ module Gitlab before: oldrev, after: newrev, ref: ref, + ref_protected: project.protected_for?(ref), checkout_sha: checkout_sha(project.repository, newrev, ref), message: message, user_id: user.id, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index da9ebf4ab0f..fd83f27ef31 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -145,7 +145,7 @@ module Gitlab # Database configured. Returns true even if the database is shared def self.has_config?(database_name) ActiveRecord::Base.configurations - .configs_for(env_name: Rails.env, name: database_name.to_s, include_replicas: true) + .configs_for(env_name: Rails.env, name: database_name.to_s, include_hidden: true) .present? end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index d7128a20a0b..db05635c73d 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -95,7 +95,7 @@ module Gitlab async_index = Gitlab::Database::AsyncIndexes::PostgresAsyncIndex.find_or_create_by!(name: index_name) do |rec| rec.table_name = table_name - rec.definition = definition + rec.definition = definition.to_s.strip end Gitlab::AppLogger.info( diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb index a3c600a4519..98eb282e43f 100644 --- a/lib/gitlab/database/async_indexes/postgres_async_index.rb +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -13,6 +13,8 @@ module Gitlab MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH MAX_DEFINITION_LENGTH = 2048 + before_validation :remove_whitespaces + validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } validates :table_name, presence: true, length: { maximum: MAX_TABLE_NAME_LENGTH } validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH } @@ -29,6 +31,10 @@ module Gitlab private + def remove_whitespaces + definition.strip! if definition.present? + end + def ensure_correct_schema_and_table_name return unless table_name diff --git a/lib/gitlab/database/ci_builds_partitioning.rb b/lib/gitlab/database/ci_builds_partitioning.rb new file mode 100644 index 00000000000..9f8b19f2d23 --- /dev/null +++ b/lib/gitlab/database/ci_builds_partitioning.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class CiBuildsPartitioning + include AsyncDdlExclusiveLeaseGuard + + ATTEMPTS = 5 + LOCK_TIMEOUT = 10.seconds + LEASE_TIMEOUT = 30.minutes + + FK_NAME = :fk_e20479742e_p + TEMP_FK_NAME = :temp_fk_e20479742e_p + NEXT_PARTITION_ID = 101 + BUILDS_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_builds_101' + ANNOTATION_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_job_annotations_101' + RUNNER_MACHINE_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_runner_machine_builds_101' + + def initialize(logger: Gitlab::AppLogger) + @connection = ::Ci::ApplicationRecord.connection + @timing_configuration = Array.new(ATTEMPTS) { [LOCK_TIMEOUT, 3.minutes] } + @logger = logger + end + + def execute + return unless can_execute? + + try_obtain_lease do + swap_foreign_keys + create_new_ci_builds_partition + create_new_job_annotations_partition + create_new_runner_machine_partition + end + + rescue StandardError => e + log_info("Failed to execute: #{e.message}") + end + + private + + attr_reader :connection, :timing_configuration, :logger + + delegate :quote_table_name, :quote_column_name, to: :connection + + def swap_foreign_keys + if new_foreign_key_exists? + log_info('Foreign key already renamed, nothing to do') + + return + end + + with_lock_retries do + connection.execute drop_old_foreign_key_sql + + rename_constraint :p_ci_builds_metadata, TEMP_FK_NAME, FK_NAME + + each_partition do |partition| + rename_constraint partition.identifier, TEMP_FK_NAME, FK_NAME + end + end + + log_info('Foreign key successfully renamed') + end + + def create_new_ci_builds_partition + if connection.table_exists?(BUILDS_PARTITION_NAME) + log_info('p_ci_builds partition exists, nothing to do') + return + end + + with_lock_retries do + connection.execute new_ci_builds_partition_sql + end + + log_info('Partition for p_ci_builds successfully created') + end + + def create_new_job_annotations_partition + if connection.table_exists?(ANNOTATION_PARTITION_NAME) + log_info('p_ci_job_annotations partition exists, nothing to do') + return + end + + with_lock_retries do + connection.execute new_job_annotations_partition_sql + end + + log_info('Partition for p_ci_job_annotations successfully created') + end + + def create_new_runner_machine_partition + if connection.table_exists?(RUNNER_MACHINE_PARTITION_NAME) + log_info('p_ci_runner_machine_builds partition exists, nothing to do') + return + end + + with_lock_retries do + connection.execute new_runner_machine_partition_sql + end + + log_info('Partition for p_ci_runner_machine_builds successfully created') + end + + def can_execute? + return false if process_disabled? + return false unless Gitlab.com? + + if vacuum_running? + log_info('Autovacuum detected') + + return false + end + + true + end + + def process_disabled? + ::Feature.disabled?(:complete_p_ci_builds_partitioning) + end + + def new_foreign_key_exists? + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresForeignKey + .by_constrained_table_name_or_identifier(:p_ci_builds_metadata) + .by_referenced_table_name(:p_ci_builds) + .by_name(FK_NAME) + .exists? + end + end + + def vacuum_running? + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresAutovacuumActivity + .wraparound_prevention + .for_tables(%i[ci_builds ci_builds_metadata]) + .any? + end + end + + def drop_old_foreign_key_sql + <<~SQL.squish + SET LOCAL statement_timeout TO '11s'; + + LOCK TABLE ci_builds, p_ci_builds_metadata IN ACCESS EXCLUSIVE MODE; + + ALTER TABLE p_ci_builds_metadata DROP CONSTRAINT #{FK_NAME}; + SQL + end + + def rename_constraint(table_name, old_name, new_name) + connection.execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} + SQL + end + + def new_ci_builds_partition_sql + <<~SQL + SET LOCAL statement_timeout TO '11s'; + + LOCK ci_pipelines, ci_stages IN SHARE ROW EXCLUSIVE MODE; + LOCK TABLE ONLY p_ci_builds IN ACCESS EXCLUSIVE MODE; + + CREATE TABLE IF NOT EXISTS #{BUILDS_PARTITION_NAME} + PARTITION OF p_ci_builds + FOR VALUES IN (#{NEXT_PARTITION_ID}); + SQL + end + + def new_job_annotations_partition_sql + <<~SQL + SET LOCAL statement_timeout TO '11s'; + + LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE; + LOCK TABLE ONLY p_ci_job_annotations IN ACCESS EXCLUSIVE MODE; + + CREATE TABLE IF NOT EXISTS #{ANNOTATION_PARTITION_NAME} + PARTITION OF p_ci_job_annotations + FOR VALUES IN (#{NEXT_PARTITION_ID}); + SQL + end + + def new_runner_machine_partition_sql + <<~SQL + SET LOCAL statement_timeout TO '11s'; + + LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE; + LOCK TABLE ONLY p_ci_runner_machine_builds IN ACCESS EXCLUSIVE MODE; + + CREATE TABLE IF NOT EXISTS #{RUNNER_MACHINE_PARTITION_NAME} + PARTITION OF p_ci_runner_machine_builds + FOR VALUES IN (#{NEXT_PARTITION_ID}); + SQL + end + + def with_lock_retries(&block) + Gitlab::Database::WithLockRetries.new( + timing_configuration: timing_configuration, + connection: connection, + logger: logger, + klass: self.class + ).run(raise_on_exhaustion: true, &block) + end + + def each_partition(&block) + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresPartitionedTable.each_partition(:p_ci_builds_metadata, &block) + end + end + + def log_info(message) + logger.info(message: message, class: self.class.to_s) + end + + def connection_db_config + ::Ci::ApplicationRecord.connection_db_config + end + + def lease_timeout + LEASE_TIMEOUT + end + end + end +end diff --git a/lib/gitlab/database/database_connection_info.rb b/lib/gitlab/database/database_connection_info.rb index 57ecbcd64ae..f0cafcf041b 100644 --- a/lib/gitlab/database/database_connection_info.rb +++ b/lib/gitlab/database/database_connection_info.rb @@ -6,6 +6,7 @@ module Gitlab :name, :description, :gitlab_schemas, + :lock_gitlab_schemas, :klass, :fallback_database, :db_dir, @@ -20,6 +21,7 @@ module Gitlab self.name = name.to_sym self.gitlab_schemas = gitlab_schemas.map(&:to_sym) self.klass = klass.constantize + self.lock_gitlab_schemas = (lock_gitlab_schemas || []).map(&:to_sym) self.fallback_database = fallback_database&.to_sym self.db_dir = Rails.root.join(db_dir || 'db') end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index b1af62e4875..e1974aac371 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -4,7 +4,7 @@ module Gitlab module Database module EachDatabase class << self - def each_database_connection(only: nil, include_shared: true) + def each_connection(only: nil, include_shared: true) selected_names = Array.wrap(only) base_models = select_base_models(selected_names) @@ -18,7 +18,6 @@ module Gitlab end end end - alias_method :each_db_connection, :each_database_connection def each_model_connection(models, only_on: nil, &blk) selected_databases = Array.wrap(only_on).map(&:to_sym) diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 9b58284b389..0bd357b7730 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -23,6 +23,21 @@ module Gitlab tables.map { |table| table_schema!(table) }.to_set end + # Mainly used for test tables + # It maps table names prefixes to gitlab_schemas. + # The order of keys matter. Prefixes that contain other prefixes should come first. + IMPLICIT_GITLAB_SCHEMAS = { + '_test_gitlab_main_clusterwide_' => :gitlab_main_clusterwide, + '_test_gitlab_main_cell_' => :gitlab_main_cell, + '_test_gitlab_main_' => :gitlab_main, + '_test_gitlab_ci_' => :gitlab_ci, + '_test_gitlab_embedding_' => :gitlab_embedding, + '_test_gitlab_geo_' => :gitlab_geo, + '_test_gitlab_pm_' => :gitlab_pm, + '_test_' => :gitlab_shared, + 'pg_' => :gitlab_internal + }.freeze + # rubocop:disable Metrics/CyclomaticComplexity def self.table_schema(name) schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.` @@ -54,19 +69,11 @@ module Gitlab # All tables from `information_schema.` are marked as `internal` return :gitlab_internal if schema_name == 'information_schema' - return :gitlab_main if table_name.start_with?('_test_gitlab_main_') - - return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_') - - return :gitlab_embedding if table_name.start_with?('_test_gitlab_embedding_') - - return :gitlab_geo if table_name.start_with?('_test_gitlab_geo_') - - # All tables that start with `_test_` without a following schema are shared and ignored - return :gitlab_shared if table_name.start_with?('_test_') + IMPLICIT_GITLAB_SCHEMAS.each do |prefix, gitlab_schema| + return gitlab_schema if table_name.start_with?(prefix) + end - # All `pg_` tables are marked as `internal` - return :gitlab_internal if table_name.start_with?('pg_') + nil end # rubocop:enable Metrics/CyclomaticComplexity diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 02f14e020c1..2c480eb2cdc 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -39,7 +39,7 @@ module Gitlab @load_balancer = load_balancer end - def select_all(arel, name = nil, binds = [], preparable: nil) + def select_all(arel, name = nil, binds = [], preparable: nil, async: false) if arel.respond_to?(:locked) && arel.locked # SELECT ... FOR UPDATE queries should be sent to the primary. current_session.write! diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 23476e1f5e9..f6144b7b772 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -12,6 +12,8 @@ module Gitlab REPLICA_SUFFIX = '_replica' + attr_accessor :service_discovery + attr_reader :host_list, :configuration # configuration - An instance of `LoadBalancing::Configuration` that @@ -45,6 +47,8 @@ module Gitlab # If no secondaries were available this method will use the primary # instead. def read(&block) + service_discovery&.log_refresh_thread_interruption + conflict_retried = 0 while host @@ -103,6 +107,8 @@ module Gitlab # Yields a connection that can be used for both reads and writes. def read_write + service_discovery&.log_refresh_thread_interruption + connection = nil transaction_open = nil @@ -285,7 +291,7 @@ module Gitlab def pool ActiveRecord::Base.connection_handler.retrieve_connection_pool( @configuration.connection_specification_name, - role: ActiveRecord::Base.writing_role, + role: ActiveRecord.writing_role, shard: ActiveRecord::Base.default_shard ) || raise(::ActiveRecord::ConnectionNotEstablished) end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 57a588db8a8..a0b0ad19f73 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -15,12 +15,14 @@ module Gitlab class ServiceDiscovery EmptyDnsResponse = Class.new(StandardError) + attr_accessor :refresh_thread, :refresh_thread_last_run, :refresh_thread_interruption_logged + attr_reader :interval, :record, :record_type, :disconnect_timeout, :load_balancer MAX_SLEEP_ADJUSTMENT = 10 - MAX_DISCOVERY_RETRIES = 3 + DISCOVERY_THREAD_REFRESH_DELTA = 5 RETRY_DELAY_RANGE = (0.1..0.2).freeze @@ -74,8 +76,10 @@ module Gitlab # rubocop:enable Metrics/ParameterLists def start - Thread.new do + self.refresh_thread = Thread.new do loop do + self.refresh_thread_last_run = Time.current + next_sleep_duration = perform_service_discovery # We slightly randomize the sleep() interval. This should reduce @@ -103,15 +107,6 @@ module Gitlab # Slightly randomize the retry delay so that, in the case of a total # dns outage, all starting services do not pressure the dns server at the same time. sleep(rand(RETRY_DELAY_RANGE)) - rescue Exception => error # rubocop:disable Lint/RescueException - # All exceptions are logged to find any pattern and solve https://gitlab.com/gitlab-org/gitlab/-/issues/364370 - # This will be removed in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120173 - Gitlab::Database::LoadBalancing::Logger.error( - event: :service_discovery_unexpected_exception, - message: "Service discovery encountered an uncaught error: #{error.message}" - ) - - raise end interval @@ -214,6 +209,20 @@ module Gitlab ) end + def log_refresh_thread_interruption + return if refresh_thread_last_run.blank? || refresh_thread_interruption_logged || + (refresh_thread_last_run + DISCOVERY_THREAD_REFRESH_DELTA.minutes).future? + + Gitlab::Database::LoadBalancing::Logger.error( + event: :service_discovery_refresh_thread_interrupt, + refresh_thread_last_run: refresh_thread_last_run, + thread_status: refresh_thread&.status&.to_s, + thread_backtrace: refresh_thread&.backtrace&.join('\n') + ) + + self.refresh_thread_interruption_logged = true + end + private def record_type_for(type) diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index eceea1d8d9c..2e65e1c8e56 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -55,6 +55,8 @@ module Gitlab sv = ServiceDiscovery.new(load_balancer, **configuration.service_discovery) + load_balancer.service_discovery = sv + sv.perform_service_discovery sv.start if @start_service_discovery diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 291f483e6e4..256c524e989 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -11,6 +11,7 @@ module Gitlab include Migrations::ConstraintsHelpers include Migrations::ExtensionHelpers include Migrations::SidekiqHelpers + include Migrations::RedisHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb index 55c4fd6a7af..fe456fab505 100644 --- a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb +++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb @@ -11,7 +11,9 @@ module Gitlab end def exec_migration(connection, direction) - return super if %w[main ci].exclude?(Gitlab::Database.db_config_name(connection)) + db_config_name = Gitlab::Database.db_config_name(connection) + db_info = Gitlab::Database.all_database_connections.fetch(db_config_name) + return super if db_info.lock_gitlab_schemas.empty? return super if automatic_lock_on_writes_disabled? # This compares the tables only on the `public` schema. Partitions are not affected @@ -20,7 +22,7 @@ module Gitlab new_tables = connection.tables - tables new_tables.each do |table_name| - lock_writes_on_table(connection, table_name) if should_lock_writes_on_table?(table_name) + lock_writes_on_table(connection, table_name) if should_lock_writes_on_table?(db_info, table_name) end end @@ -39,16 +41,17 @@ module Gitlab end end - def should_lock_writes_on_table?(table_name) - # currently gitlab_schema represents only present existing tables, this is workaround for deleted tables - # that should be skipped as they will be removed in a future migration. + def should_lock_writes_on_table?(db_info, table_name) + # We skip locking writes on tables that are scheduled for deletion in a future migration return false if Gitlab::Database::GitlabSchema.deleted_tables_to_schema[table_name] table_schema = Gitlab::Database::GitlabSchema.table_schema!(table_name.to_s) - return false unless %i[gitlab_main gitlab_ci].include?(table_schema) - - Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) + # This takes into consideration which database mode is used. + # In single-db and single-db-ci-connection the main database includes gitlab_ci tables, + # so we don't lock them there. + Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) && + db_info.lock_gitlab_schemas.include?(table_schema) end # with_retries creates new a transaction. So we set it to false if the connection is diff --git a/lib/gitlab/database/migrations/redis_helpers.rb b/lib/gitlab/database/migrations/redis_helpers.rb new file mode 100644 index 00000000000..41a2841da7c --- /dev/null +++ b/lib/gitlab/database/migrations/redis_helpers.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module RedisHelpers + SCAN_START_CURSOR = '0' + + # Check if the migration exists before enqueueing the worker + def queue_redis_migration_job(job_name) + RedisMigrationWorker.fetch_migrator!(job_name) + RedisMigrationWorker.perform_async(job_name, SCAN_START_CURSOR) + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index ed55081c9ab..dc9ea304aac 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -32,7 +32,7 @@ module Gitlab result_dir = background_migrations_dir(for_database, legacy_mode) # Only one loop iteration since we pass `only:` here - Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + Gitlab::Database::EachDatabase.each_connection(only: for_database) do |connection| from_id = batched_migrations_last_id(for_database).read runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner @@ -68,7 +68,7 @@ module Gitlab runner = nil base_dir = background_migrations_dir(for_database, false) - Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + Gitlab::Database::EachDatabase.each_connection(only: for_database) do |connection| runner = Gitlab::Database::Migrations::BatchedMigrationLastId .new(connection, base_dir) end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index af853c933ba..c5e0b361df5 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -57,6 +57,20 @@ module Gitlab job_arguments: migration.job_arguments ) + # If no rows match, the next_bounds are nil. + # This will only happen if there are zero rows to match from the current sampling point to the end + # of the table + # Simulate the approach in the actual background migration worker by not sampling a batch + # from this range. + # (The actual worker would finish the migration, but we may find batches that can be sampled elsewhere + # in the table) + if next_bounds.nil? + # If the migration has no work to do across the entire table, sampling can get stuck + # in a loop if we don't mark the attempted batches as completed + completed_batches << (batch_start..(batch_start + migration.batch_size)) + next + end + batch_min, batch_max = next_bounds job = migration.create_batched_job!(batch_min, batch_max) @@ -65,7 +79,7 @@ module Gitlab job end - end + end.reject(&:nil?) # Remove skipped batches from the lazy list of batches to test job_class_name = migration.job_class_name diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 9895a68ec8d..48f58920d52 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -40,7 +40,7 @@ module Gitlab next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel) model_connection_name = model.connection_db_config.name - Gitlab::Database::EachDatabase.each_db_connection(include_shared: false) do |connection, connection_name| + Gitlab::Database::EachDatabase.each_connection(include_shared: false) do |connection, connection_name| if connection_name != model_connection_name PartitionManager.new(model, connection: connection).sync_partitions end @@ -64,7 +64,7 @@ module Gitlab Gitlab::AppLogger.info(message: 'Dropping detached postgres partitions') - Gitlab::Database::EachDatabase.each_database_connection do + Gitlab::Database::EachDatabase.each_connection do DetachedPartitionDropper.new.perform end diff --git a/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb b/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb deleted file mode 100644 index 88affaa9757..00000000000 --- a/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -# This patch will be included in the next Rails release: https://github.com/rails/rails/pull/42368 -raise 'This patch can be removed' if Rails::VERSION::MAJOR > 6 - -# rubocop:disable Gitlab/ModuleWithInstanceVariables -module Gitlab - module Database - module PostgresqlAdapter - module EmptyQueryPing - # ActiveRecord uses `SELECT 1` to check if the connection is alive - # We patch this here to use an empty query instead, which is a bit faster - def active? - @lock.synchronize do - @connection.query ';' - end - true - rescue PG::Error - false - end - end - end - end -end -# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 739e573b6c4..9c860ebc6aa 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -20,7 +20,7 @@ module Gitlab end def self.invoke(database = nil) - Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| + Gitlab::Database::EachDatabase.each_connection do |connection, connection_name| next if database && database.to_s != connection_name.to_s Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) @@ -59,6 +59,7 @@ module Gitlab # most bloated indexes for reindexing. def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION) IndexSelection.new(candidate_indexes).take(maximum_records).each do |index| + Gitlab::Database::CiBuildsPartitioning.new.execute Coordinator.new(index).perform end end diff --git a/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb deleted file mode 100644 index 32d638380ea..00000000000 --- a/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Adapters - class ColumnDatabaseAdapter - def initialize(query_result) - @query_result = query_result - end - - def name - @name ||= query_result['column_name'] - end - - def table_name - query_result['table_name'] - end - - def data_type - query_result['data_type'] - end - - def default - return unless query_result['column_default'] - - return if name == 'id' || query_result['column_default'].include?('nextval') - - "DEFAULT #{query_result['column_default']}" - end - - def nullable - 'NOT NULL' if query_result['not_null'] - end - - def partition_key? - query_result['partition_key'] - end - - private - - attr_reader :query_result - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb deleted file mode 100644 index 20814b098c1..00000000000 --- a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb +++ /dev/null @@ -1,139 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Adapters - UndefinedPGType = Class.new(StandardError) - - class ColumnStructureSqlAdapter - NOT_NULL_CONSTR = :CONSTR_NOTNULL - DEFAULT_CONSTR = :CONSTR_DEFAULT - - MAPPINGS = { - 't' => 'true', - 'f' => 'false' - }.freeze - - attr_reader :table_name - - def initialize(table_name, pg_query_stmt, partitioning_stmt) - @table_name = table_name - @pg_query_stmt = pg_query_stmt - @partitioning_stmt = partitioning_stmt - end - - def name - @name ||= pg_query_stmt.colname - end - - def data_type - type(pg_query_stmt.type_name) - end - - def default - return if name == 'id' - - value = parse_node(constraints.find { |node| node.constraint.contype == DEFAULT_CONSTR }) - - return unless value - - "DEFAULT #{value}" - end - - def nullable - 'NOT NULL' if constraints.any? { |node| node.constraint.contype == NOT_NULL_CONSTR } - end - - def partition_key? - partition_keys.include?(name) - end - - private - - attr_reader :pg_query_stmt, :partitioning_stmt - - def constraints - @constraints ||= pg_query_stmt.constraints - end - - # Returns the node type - # - # pg_type:: type alias, used internally by postgres, +int4+, +int8+, +bool+, +varchar+ - # type:: type name, like +integer+, +bigint+, +boolean+, +character varying+. - # array_ext:: adds the +[]+ extension for array types. - # precision_ext:: adds the precision, if have any, like +(255)+, +(6)+. - # - # @info +timestamp+ and +timestamptz+ have a particular case when precision is defined. - # In this case, the order of the statement needs to be re-arranged from - # timestamp without time zone(6) to timestamp(6) without a time zone. - def type(node) - pg_type = parse_node(node.names.last) - type = PgTypes::TYPES.fetch(pg_type).dup - array_ext = '[]' if node.array_bounds.any? - precision_ext = "(#{node.typmods.map { |typmod| parse_node(typmod) }.join(',')})" if node.typmods.any? - - if %w[timestamp timestamptz].include?(pg_type) - type.gsub!('timestamp', ['timestamp', precision_ext].compact.join('')) - precision_ext = nil - end - - [type, precision_ext, array_ext].compact.join('') - rescue KeyError => exception - raise UndefinedPGType, exception.message - end - - # Parses PGQuery nodes recursively - # - # :constraint:: nodes that groups column default info - # :partition_elem:: node that store partition key info - # :func_cal:: nodes that stores functions, like +now()+ - # :a_const:: nodes that stores constant values, like +t+, +f+, +0.0.0.0+, +255+, +1.0+ - # :type_cast:: nodes that stores casting values, like +'name'::text+, +'0.0.0.0'::inet+ - # else:: extract node values in the last iteration of the recursion, like +int4+, +1.0+, +now+, +255+ - # - # @note boolean types types are mapped from +t+, +f+ to +true+, +false+ - def parse_node(node) - return unless node - - case node.node - when :constraint - parse_node(node.constraint.raw_expr) - when :partition_elem - node.partition_elem.name - when :func_call - "#{parse_node(node.func_call.funcname.first)}()" - when :a_const - parse_a_const(node.a_const) - when :type_cast - value = parse_node(node.type_cast.arg) - type = type(node.type_cast.type_name) - separator = MAPPINGS.key?(value) ? '' : "::#{type}" - - [MAPPINGS.fetch(value, "'#{value}'"), separator].compact.join('') - else - get_value_from_key(node, key: node.node) - end - end - - def parse_a_const(a_const) - return unless a_const - - type = a_const.val - get_value_from_key(a_const, key: type) - end - - def get_value_from_key(node, key:) - node.to_h[key].values.last - end - - def partition_keys - return [] unless partitioning_stmt - - @partition_keys ||= partitioning_stmt.part_params.map { |key_stmt| parse_node(key_stmt) } - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb deleted file mode 100644 index 3b45f5c77ca..00000000000 --- a/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Adapters - class ForeignKeyDatabaseAdapter - def initialize(query_result) - @query_result = query_result - end - - def name - "#{query_result['schema']}.#{query_result['foreign_key_name']}" - end - - def table_name - query_result['table_name'] - end - - def statement - query_result['foreign_key_definition'] - end - - private - - attr_reader :query_result - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb deleted file mode 100644 index e4c1e1adab3..00000000000 --- a/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Adapters - class ForeignKeyStructureSqlAdapter - STATEMENT_REGEX = /\bREFERENCES\s\K\S+\K\s\(/ - EXTRACT_REGEX = /\bFOREIGN KEY.*/ - - def initialize(parsed_stmt) - @parsed_stmt = parsed_stmt - end - - def name - "#{schema_name}.#{foreign_key_name}" - end - - def table_name - parsed_stmt.relation.relname - end - - # PgQuery parses FK statements with an extra space in the referenced table column. - # This extra space needs to be removed. - # - # @example REFERENCES ci_pipelines (id) => REFERENCES ci_pipelines(id) - def statement - deparse_stmt[EXTRACT_REGEX].gsub(STATEMENT_REGEX, '(') - end - - private - - attr_reader :parsed_stmt - - def schema_name - parsed_stmt.relation.schemaname - end - - def foreign_key_name - parsed_stmt.cmds.first.alter_table_cmd.def.constraint.conname - end - - def deparse_stmt - PgQuery.deparse_stmt(parsed_stmt) - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb deleted file mode 100644 index 858bf618f44..00000000000 --- a/lib/gitlab/database/schema_validation/database.rb +++ /dev/null @@ -1,166 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Database - STATIC_PARTITIONS_SCHEMA = 'gitlab_partitions_static' - - def initialize(connection) - @connection = connection - end - - def fetch_index_by_name(index_name) - index_map[index_name] - end - - def fetch_trigger_by_name(trigger_name) - trigger_map[trigger_name] - end - - def fetch_foreign_key_by_name(foreign_key_name) - foreign_key_map[foreign_key_name] - end - - def fetch_table_by_name(table_name) - table_map[table_name] - end - - def index_exists?(index_name) - index_map[index_name].present? - end - - def trigger_exists?(trigger_name) - trigger_map[trigger_name].present? - end - - def foreign_key_exists?(foreign_key_name) - fetch_foreign_key_by_name(foreign_key_name).present? - end - - def table_exists?(table_name) - fetch_table_by_name(table_name).present? - end - - def indexes - index_map.values - end - - def triggers - trigger_map.values - end - - def foreign_keys - foreign_key_map.values - end - - def tables - table_map.values - end - - private - - attr_reader :connection - - def schemas - @schemas ||= [STATIC_PARTITIONS_SCHEMA, connection.current_schema] - end - - def index_map - @index_map ||= - fetch_indexes.transform_values! do |index_stmt| - SchemaObjects::Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) - end - end - - def trigger_map - @trigger_map ||= - fetch_triggers.transform_values! do |trigger_stmt| - SchemaObjects::Trigger.new(PgQuery.parse(trigger_stmt).tree.stmts.first.stmt.create_trig_stmt) - end - end - - def foreign_key_map - @foreign_key_map ||= fetch_fks.each_with_object({}) do |stmt, result| - adapter = Adapters::ForeignKeyDatabaseAdapter.new(stmt) - - result[adapter.name] = SchemaObjects::ForeignKey.new(adapter) - end - end - - def table_map - @table_map ||= fetch_tables.transform_values! do |stmt| - columns = stmt.map { |column| SchemaObjects::Column.new(Adapters::ColumnDatabaseAdapter.new(column)) } - - SchemaObjects::Table.new(stmt.first['table_name'], columns) - end - end - - def fetch_indexes - sql = <<~SQL - SELECT indexname, indexdef - FROM pg_indexes - WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ($1, $2); - SQL - - connection.select_rows(sql, nil, schemas).to_h - end - - def fetch_triggers - sql = <<~SQL - SELECT triggers.tgname, pg_get_triggerdef(triggers.oid) - FROM pg_catalog.pg_trigger triggers - INNER JOIN pg_catalog.pg_class rel ON triggers.tgrelid = rel.oid - INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace - WHERE triggers.tgisinternal IS FALSE - AND nsp.nspname IN ($1, $2) - SQL - - connection.select_rows(sql, nil, schemas).to_h - end - - def fetch_tables - sql = <<~SQL - SELECT - table_information.relname AS table_name, - col_information.attname AS column_name, - col_information.attnotnull AS not_null, - col_information.attnum = ANY(pg_partitioned_table.partattrs) as partition_key, - format_type(col_information.atttypid, col_information.atttypmod) AS data_type, - pg_get_expr(col_default_information.adbin, col_default_information.adrelid) AS column_default - FROM pg_attribute AS col_information - JOIN pg_class AS table_information ON col_information.attrelid = table_information.oid - JOIN pg_namespace AS schema_information ON table_information.relnamespace = schema_information.oid - LEFT JOIN pg_partitioned_table ON pg_partitioned_table.partrelid = table_information.oid - LEFT JOIN pg_attrdef AS col_default_information ON col_information.attrelid = col_default_information.adrelid - AND col_information.attnum = col_default_information.adnum - WHERE NOT col_information.attisdropped - AND col_information.attnum > 0 - AND table_information.relkind IN ('r', 'p') - AND schema_information.nspname IN ($1, $2) - SQL - - connection.exec_query(sql, nil, schemas).group_by { |row| row['table_name'] } - end - - def fetch_fks - sql = <<~SQL - SELECT - pg_namespace.nspname::text AS schema, - pg_class.relname::text AS table_name, - pg_constraint.conname AS foreign_key_name, - pg_get_constraintdef(pg_constraint.oid) AS foreign_key_definition - FROM pg_constraint - INNER JOIN pg_class ON pg_constraint.conrelid = pg_class.oid - INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid - WHERE contype = 'f' - AND pg_namespace.nspname = $1 - AND pg_constraint.conparentid = 0 - SQL - - connection.exec_query(sql, nil, [connection.current_schema]) - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/inconsistency.rb b/lib/gitlab/database/schema_validation/inconsistency.rb deleted file mode 100644 index 766f48ef339..00000000000 --- a/lib/gitlab/database/schema_validation/inconsistency.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Inconsistency - def initialize(validator_class, structure_sql_object, database_object) - @validator_class = validator_class - @structure_sql_object = structure_sql_object - @database_object = database_object - end - - def error_message - format(validator_class::ERROR_MESSAGE, object_name) - end - - def type - validator_class.name.demodulize.underscore - end - - def object_type - structure_sql_object&.class&.name&.demodulize || database_object&.class&.name&.demodulize - end - - def table_name - structure_sql_object&.table_name || database_object&.table_name - end - - def object_name - structure_sql_object&.name || database_object&.name - end - - def diff - Diffy::Diff.new(structure_sql_statement, database_statement) - end - - def inspect - <<~MSG - #{'-' * 54} - #{error_message} - Diff: - #{diff.to_s(:color)} - #{'-' * 54} - MSG - end - - def structure_sql_statement - return unless structure_sql_object - - "#{structure_sql_object.statement}\n" - end - - def database_statement - return unless database_object - - "#{database_object.statement}\n" - end - - private - - attr_reader :validator_class, :structure_sql_object, :database_object - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/pg_types.rb b/lib/gitlab/database/schema_validation/pg_types.rb deleted file mode 100644 index 0a1999d056e..00000000000 --- a/lib/gitlab/database/schema_validation/pg_types.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class PgTypes - TYPES = { - 'bool' => 'boolean', - 'bytea' => 'bytea', - 'char' => '"char"', - 'int8' => 'bigint', - 'int2' => 'smallint', - 'int4' => 'integer', - 'regproc' => 'regproc', - 'text' => 'text', - 'oid' => 'oid', - 'tid' => 'tid', - 'xid' => 'xid', - 'cid' => 'cid', - 'json' => 'json', - 'xml' => 'xml', - 'pg_node_tree' => 'pg_node_tree', - 'pg_ndistinct' => 'pg_ndistinct', - 'pg_dependencies' => 'pg_dependencies', - 'pg_mcv_list' => 'pg_mcv_list', - 'xid8' => 'xid8', - 'path' => 'path', - 'polygon' => 'polygon', - 'float4' => 'real', - 'float8' => 'double precision', - 'circle' => 'circle', - 'money' => 'money', - 'macaddr' => 'macaddr', - 'inet' => 'inet', - 'cidr' => 'cidr', - 'macaddr8' => 'macaddr8', - 'aclitem' => 'aclitem', - 'bpchar' => 'character', - 'varchar' => 'character varying', - 'date' => 'date', - 'time' => 'time without time zone', - 'timestamp' => 'timestamp without time zone', - 'timestamptz' => 'timestamp with time zone', - 'interval' => 'interval', - 'timetz' => 'time with time zone', - 'bit' => 'bit', - 'varbit' => 'bit varying', - 'numeric' => 'numeric', - 'refcursor' => 'refcursor', - 'regprocedure' => 'regprocedure', - 'regoper' => 'regoper', - 'regoperator' => 'regoperator', - 'regclass' => 'regclass', - 'regcollation' => 'regcollation', - 'regtype' => 'regtype', - 'regrole' => 'regrole', - 'regnamespace' => 'regnamespace', - 'uuid' => 'uuid', - 'pg_lsn' => 'pg_lsn', - 'tsvector' => 'tsvector', - 'gtsvector' => 'gtsvector', - 'tsquery' => 'tsquery', - 'regconfig' => 'regconfig', - 'regdictionary' => 'regdictionary', - 'jsonb' => 'jsonb', - 'jsonpath' => 'jsonpath', - 'txid_snapshot' => 'txid_snapshot', - 'pg_snapshot' => 'pg_snapshot' - }.freeze - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/runner.rb b/lib/gitlab/database/schema_validation/runner.rb deleted file mode 100644 index 7a02c8a16d6..00000000000 --- a/lib/gitlab/database/schema_validation/runner.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Runner - def initialize(structure_sql, database, validators: Validators::BaseValidator.all_validators) - @structure_sql = structure_sql - @database = database - @validators = validators - end - - def execute - validators.flat_map { |c| c.new(structure_sql, database).execute } - end - - private - - attr_reader :structure_sql, :database, :validators - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/base.rb b/lib/gitlab/database/schema_validation/schema_objects/base.rb deleted file mode 100644 index 43d30dc54ae..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/base.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Base - def initialize(parsed_stmt) - @parsed_stmt = parsed_stmt - end - - def name - raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" - end - - def table_name - parsed_stmt.relation.relname - end - - def statement - @statement ||= PgQuery.deparse_stmt(parsed_stmt) - end - - private - - attr_reader :parsed_stmt - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/column.rb b/lib/gitlab/database/schema_validation/schema_objects/column.rb deleted file mode 100644 index bd219300a13..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/column.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Column - def initialize(adapter) - @adapter = adapter - end - - attr_reader :adapter - - delegate :name, :table_name, :partition_key?, to: :adapter - - def statement - [name, adapter.data_type, adapter.default, adapter.nullable].compact.join(' ') - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb b/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb deleted file mode 100644 index b616b1a72b7..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class ForeignKey - def initialize(adapter) - @adapter = adapter - end - - # Foreign key name should include the schema, as the same name could be used across different schemas - # - # @example public.foreign_key_name - def name - @name ||= adapter.name - end - - def table_name - @table_name ||= adapter.table_name - end - - def statement - @statement ||= adapter.statement - end - - private - - attr_reader :adapter - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/index.rb b/lib/gitlab/database/schema_validation/schema_objects/index.rb deleted file mode 100644 index 28d61b18266..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/index.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Index < Base - def name - parsed_stmt.idxname - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/table.rb b/lib/gitlab/database/schema_validation/schema_objects/table.rb deleted file mode 100644 index de2e675d20e..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/table.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Table - def initialize(name, columns) - @name = name - @columns = columns - end - - attr_reader :name, :columns - - def table_name - name - end - - def statement - format('CREATE TABLE %s (%s)', name, columns_statement) - end - - def fetch_column_by_name(column_name) - columns.find { |column| column.name == column_name } - end - - def column_exists?(column_name) - fetch_column_by_name(column_name).present? - end - - private - - def columns_statement - columns.reject(&:partition_key?).map(&:statement).join(', ') - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/trigger.rb b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb deleted file mode 100644 index 508e6b27ed3..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/trigger.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Trigger < Base - def name - parsed_stmt.trigname - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb deleted file mode 100644 index 4d6fa17f0fc..00000000000 --- a/lib/gitlab/database/schema_validation/structure_sql.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class StructureSql - DEFAULT_SCHEMA = 'public' - - def initialize(structure_file_path, schema_name = DEFAULT_SCHEMA) - @structure_file_path = structure_file_path - @schema_name = schema_name - end - - def index_exists?(index_name) - indexes.find { |index| index.name == index_name }.present? - end - - def trigger_exists?(trigger_name) - triggers.find { |trigger| trigger.name == trigger_name }.present? - end - - def foreign_key_exists?(foreign_key_name) - foreign_keys.find { |fk| fk.name == foreign_key_name }.present? - end - - def fetch_table_by_name(table_name) - tables.find { |table| table.name == table_name } - end - - def table_exists?(table_name) - fetch_table_by_name(table_name).present? - end - - def indexes - @indexes ||= map_with_default_schema(index_statements, SchemaObjects::Index) - end - - def triggers - @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger) - end - - def foreign_keys - @foreign_keys ||= foreign_key_statements.map do |stmt| - stmt.relation.schemaname = schema_name if stmt.relation.schemaname == '' - - SchemaObjects::ForeignKey.new(Adapters::ForeignKeyStructureSqlAdapter.new(stmt)) - end - end - - def tables - @tables ||= table_statements.map do |stmt| - table_name = stmt.relation.relname - partition_stmt = stmt.partspec - - columns = stmt.table_elts.select { |n| n.node == :column_def }.map do |column| - adapter = Adapters::ColumnStructureSqlAdapter.new(table_name, column.column_def, partition_stmt) - SchemaObjects::Column.new(adapter) - end - - SchemaObjects::Table.new(table_name, columns) - end - end - - private - - attr_reader :structure_file_path, :schema_name - - def index_statements - statements.filter_map { |s| s.stmt.index_stmt } - end - - def trigger_statements - statements.filter_map { |s| s.stmt.create_trig_stmt } - end - - def table_statements - statements.filter_map { |s| s.stmt.create_stmt } - end - - def foreign_key_statements - constraint_statements(:CONSTR_FOREIGN) - end - - # Filter constraint statement nodes - # - # @param constraint_type [Symbol] node type. One of CONSTR_PRIMARY, CONSTR_CHECK, CONSTR_EXCLUSION, - # CONSTR_UNIQUE or CONSTR_FOREIGN. - def constraint_statements(constraint_type) - alter_table_statements(:AT_AddConstraint).filter do |stmt| - stmt.cmds.first.alter_table_cmd.def.constraint.contype == constraint_type - end - end - - # Filter alter table statement nodes - # - # @param subtype [Symbol] node subtype +AT_AttachPartition+, +AT_ColumnDefault+ or +AT_AddConstraint+ - def alter_table_statements(subtype) - statements.filter_map do |statement| - node = statement.stmt.alter_table_stmt - - next unless node - - node if node.cmds.first.alter_table_cmd.subtype == subtype - end - end - - def statements - @statements ||= parsed_structure_file.tree.stmts - end - - def parsed_structure_file - PgQuery.parse(File.read(structure_file_path)) - end - - def map_with_default_schema(statements, validation_class) - statements.map do |statement| - statement.relation.schemaname = schema_name if statement.relation.schemaname == '' - - validation_class.new(statement) - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/track_inconsistency.rb b/lib/gitlab/database/schema_validation/track_inconsistency.rb deleted file mode 100644 index 6e167653d32..00000000000 --- a/lib/gitlab/database/schema_validation/track_inconsistency.rb +++ /dev/null @@ -1,138 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class TrackInconsistency - COLUMN_TEXT_LIMIT = 6144 - - def initialize(inconsistency, project, user) - @inconsistency = inconsistency - @project = project - @user = user - end - - def execute - return unless Gitlab.com? - return refresh_issue if inconsistency_record.present? - - result = ::Issues::CreateService.new( - container: project, - current_user: user, - params: params, - perform_spam_check: false).execute - - track_inconsistency(result[:issue]) if result.success? - end - - private - - attr_reader :inconsistency, :project, :user - - def track_inconsistency(issue) - schema_inconsistency_model.create!( - issue: issue, - object_name: inconsistency.object_name, - table_name: inconsistency.table_name, - valitador_name: inconsistency.type, - diff: inconsistency_diff - ) - end - - def params - { - title: issue_title, - description: description, - issue_type: 'issue', - labels: default_labels + group_labels - } - end - - def issue_title - "New schema inconsistency: #{inconsistency.object_name}" - end - - def description - <<~MSG - We have detected a new schema inconsistency. - - **Table name:** #{inconsistency.table_name}\ - **Object name:** #{inconsistency.object_name}\ - **Validator name:** #{inconsistency.type}\ - **Object type:** #{inconsistency.object_type}\ - **Error message:** #{inconsistency.error_message} - - - **Structure.sql statement:** - - ```sql - #{inconsistency.structure_sql_statement} - ``` - - **Database statement:** - - ```sql - #{inconsistency.database_statement} - ``` - - **Diff:** - - ```diff - #{inconsistency.diff} - - ``` - - - For more information, please contact the database team. - MSG - end - - def group_labels - dictionary = YAML.safe_load(File.read(table_file_path)) - - dictionary['feature_categories'].to_a.filter_map do |feature_category| - Gitlab::Database::ConvertFeatureCategoryToGroupLabel.new(feature_category).execute - end - rescue Errno::ENOENT - [] - end - - def default_labels - %w[database database-inconsistency-report type::maintenance severity::4] - end - - def table_file_path - Rails.root.join(Gitlab::Database::GitlabSchema.dictionary_paths.first, "#{inconsistency.table_name}.yml") - end - - def schema_inconsistency_model - Gitlab::Database::SchemaValidation::SchemaInconsistency - end - - def refresh_issue - return if inconsistency_diff == inconsistency_record.diff # Nothing to refresh - - note = ::Notes::CreateService.new( - inconsistency_record.issue.project, - user, - { noteable_type: 'Issue', noteable: inconsistency_record.issue, note: description } - ).execute - - inconsistency_record.update!(diff: inconsistency_diff) if note.persisted? - end - - def inconsistency_diff - @inconsistency_diff ||= inconsistency.diff.to_s.first(COLUMN_TEXT_LIMIT) - end - - def inconsistency_record - @inconsistency_record ||= schema_inconsistency_model.with_open_issues.find_by( - object_name: inconsistency.object_name, - table_name: inconsistency.table_name, - valitador_name: inconsistency.type - ) - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/base_validator.rb b/lib/gitlab/database/schema_validation/validators/base_validator.rb deleted file mode 100644 index ee322e50a2c..00000000000 --- a/lib/gitlab/database/schema_validation/validators/base_validator.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class BaseValidator - ERROR_MESSAGE = 'A schema inconsistency has been found' - - def initialize(structure_sql, database) - @structure_sql = structure_sql - @database = database - end - - def self.all_validators - [ - ExtraTables, - ExtraTableColumns, - ExtraIndexes, - ExtraTriggers, - ExtraForeignKeys, - MissingTables, - MissingTableColumns, - MissingIndexes, - MissingTriggers, - MissingForeignKeys, - DifferentDefinitionTables, - DifferentDefinitionIndexes, - DifferentDefinitionTriggers, - DifferentDefinitionForeignKeys - ] - end - - def execute - raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" - end - - private - - attr_reader :structure_sql, :database - - def build_inconsistency(validator_class, structure_sql_object, database_object) - Inconsistency.new(validator_class, structure_sql_object, database_object) - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb deleted file mode 100644 index 8969fa76cd8..00000000000 --- a/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class DifferentDefinitionForeignKeys < BaseValidator - ERROR_MESSAGE = "The %s foreign key has a different statement between structure.sql and database" - - def execute - structure_sql.foreign_keys.filter_map do |structure_sql_fk| - database_fk = database.fetch_foreign_key_by_name(structure_sql_fk.name) - - next if database_fk.nil? - next if database_fk.statement == structure_sql_fk.statement - - build_inconsistency(self.class, structure_sql_fk, database_fk) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb b/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb deleted file mode 100644 index ba12b3cdc61..00000000000 --- a/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class DifferentDefinitionIndexes < BaseValidator - ERROR_MESSAGE = "The %s index has a different statement between structure.sql and database" - - def execute - structure_sql.indexes.filter_map do |structure_sql_index| - database_index = database.fetch_index_by_name(structure_sql_index.name) - - next if database_index.nil? - next if database_index.statement == structure_sql_index.statement - - build_inconsistency(self.class, structure_sql_index, database_index) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb b/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb deleted file mode 100644 index 9fbddbd3fcd..00000000000 --- a/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class DifferentDefinitionTables < BaseValidator - ERROR_MESSAGE = "The table %s has a different column statement between structure.sql and database" - - def execute - structure_sql.tables.filter_map do |structure_sql_table| - table_name = structure_sql_table.name - database_table = database.fetch_table_by_name(table_name) - - next unless database_table - - db_diffs, structure_diffs = column_diffs(database_table, structure_sql_table.columns) - - if db_diffs.any? - build_inconsistency(self.class, - SchemaObjects::Table.new(table_name, db_diffs), - SchemaObjects::Table.new(table_name, structure_diffs)) - end - end - end - - private - - def column_diffs(db_table, columns) - db_diffs = [] - structure_diffs = [] - - columns.each do |column| - db_column = db_table.fetch_column_by_name(column.name) - - next unless db_column - - next if db_column.statement == column.statement - - db_diffs << db_column - structure_diffs << column - end - - [db_diffs, structure_diffs] - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb deleted file mode 100644 index 79ffe9a6a98..00000000000 --- a/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class DifferentDefinitionTriggers < BaseValidator - ERROR_MESSAGE = "The %s trigger has a different statement between structure.sql and database" - - def execute - structure_sql.triggers.filter_map do |structure_sql_trigger| - database_trigger = database.fetch_trigger_by_name(structure_sql_trigger.name) - - next if database_trigger.nil? - next if database_trigger.statement == structure_sql_trigger.statement - - build_inconsistency(self.class, structure_sql_trigger, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb deleted file mode 100644 index 887e86c7bfd..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraForeignKeys < BaseValidator - ERROR_MESSAGE = "The foreign key %s is present in the database, but not in the structure.sql file" - - def execute - database.foreign_keys.filter_map do |database_fk| - next if structure_sql.foreign_key_exists?(database_fk.name) - - build_inconsistency(self.class, nil, database_fk) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_indexes.rb b/lib/gitlab/database/schema_validation/validators/extra_indexes.rb deleted file mode 100644 index c8d3749894b..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_indexes.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraIndexes < BaseValidator - ERROR_MESSAGE = "The index %s is present in the database, but not in the structure.sql file" - - def execute - database.indexes.filter_map do |database_index| - next if structure_sql.index_exists?(database_index.name) - - build_inconsistency(self.class, nil, database_index) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb b/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb deleted file mode 100644 index 823b01cf808..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraTableColumns < BaseValidator - ERROR_MESSAGE = "The table %s has columns present in the database, but not in the structure.sql file" - - def execute - database.tables.filter_map do |database_table| - table_name = database_table.name - structure_sql_table = structure_sql.fetch_table_by_name(table_name) - - next unless structure_sql_table - - inconsistencies = database_table.columns.filter_map do |database_table_column| - next if structure_sql_table.column_exists?(database_table_column.name) - - database_table_column - end - - if inconsistencies.any? - build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies)) - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_tables.rb b/lib/gitlab/database/schema_validation/validators/extra_tables.rb deleted file mode 100644 index 99e98eb8f67..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_tables.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraTables < BaseValidator - ERROR_MESSAGE = "The table %s is present in the database, but not in the structure.sql file" - - def execute - database.tables.filter_map do |database_table| - next if structure_sql.table_exists?(database_table.name) - - build_inconsistency(self.class, nil, database_table) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_triggers.rb b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb deleted file mode 100644 index 37dcbc53e2e..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_triggers.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraTriggers < BaseValidator - ERROR_MESSAGE = "The trigger %s is present in the database, but not in the structure.sql file" - - def execute - database.triggers.filter_map do |database_trigger| - next if structure_sql.trigger_exists?(database_trigger.name) - - build_inconsistency(self.class, nil, database_trigger) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb deleted file mode 100644 index b20f8474426..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingForeignKeys < BaseValidator - ERROR_MESSAGE = "The foreign key %s is missing from the database" - - def execute - structure_sql.foreign_keys.filter_map do |structure_sql_fk| - next if database.foreign_key_exists?(structure_sql_fk.name) - - build_inconsistency(self.class, structure_sql_fk, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_indexes.rb b/lib/gitlab/database/schema_validation/validators/missing_indexes.rb deleted file mode 100644 index 7f81aaccf0f..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_indexes.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingIndexes < BaseValidator - ERROR_MESSAGE = "The index %s is missing from the database" - - def execute - structure_sql.indexes.filter_map do |structure_sql_index| - next if database.index_exists?(structure_sql_index.name) - - build_inconsistency(self.class, structure_sql_index, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb b/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb deleted file mode 100644 index b49d53823ee..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingTableColumns < BaseValidator - ERROR_MESSAGE = "The table %s has columns missing from the database" - - def execute - structure_sql.tables.filter_map do |structure_sql_table| - table_name = structure_sql_table.name - database_table = database.fetch_table_by_name(table_name) - - next unless database_table - - inconsistencies = structure_sql_table.columns.filter_map do |structure_table_column| - next if database_table.column_exists?(structure_table_column.name) - - structure_table_column - end - - if inconsistencies.any? - build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies)) - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_tables.rb b/lib/gitlab/database/schema_validation/validators/missing_tables.rb deleted file mode 100644 index f1c9383487d..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_tables.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingTables < BaseValidator - ERROR_MESSAGE = "The table %s is missing from the database" - - def execute - structure_sql.tables.filter_map do |structure_sql_table| - next if database.table_exists?(structure_sql_table.name) - - build_inconsistency(self.class, structure_sql_table, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_triggers.rb b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb deleted file mode 100644 index 36236463bbf..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_triggers.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingTriggers < BaseValidator - ERROR_MESSAGE = "The trigger %s is missing from the database" - - def execute - structure_sql.triggers.filter_map do |structure_sql_trigger| - next if database.trigger_exists?(structure_sql_trigger.name) - - build_inconsistency(self.class, structure_sql_trigger, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index 02e0da022f9..aa880b709fe 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -13,7 +13,7 @@ module Gitlab end def unlock_writes - Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| + Gitlab::Database::EachDatabase.each_connection do |connection, database_name| tables_to_lock(connection) do |table_name, schema_name| # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE @@ -28,7 +28,7 @@ module Gitlab # It locks the tables on the database where they don't belong. Also it unlocks the tables # on the database where they belong def lock_writes - Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name| + Gitlab::Database::EachDatabase.each_connection(include_shared: false) do |connection, database_name| schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) tables_to_lock(connection) do |table_name, schema_name| diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 18ff7c28e17..31b214a4af9 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -42,8 +42,10 @@ module Gitlab with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do if Gitlab::Redis::ClusterUtil.cluster?(redis) - Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| - keys.each { |key| pipeline.get(key) } + redis.with_readonly_pipeline do + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + keys.each { |key| pipeline.get(key) } + end end else redis.mget(keys) diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 215ba77db13..5d0e6ea61e1 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -32,6 +32,9 @@ module Gitlab def execute raise ProjectNotFound if project.nil? + # Verification emails should never create issues + return if handled_custom_email_address_verification? + create_issue_or_note if from_address @@ -70,6 +73,27 @@ module Gitlab attr_reader :project_id, :project_path, :service_desk_key + def contains_custom_email_address_verification_subaddress? + return false unless Feature.enabled?(:service_desk_custom_email, project) + + # Verification email only has one recipient + mail.to.first.include?(ServiceDeskSetting::CUSTOM_EMAIL_VERIFICATION_SUBADDRESS) + end + + def handled_custom_email_address_verification? + return false unless contains_custom_email_address_verification_subaddress? + + ::ServiceDesk::CustomEmailVerifications::UpdateService.new( + project: project, + current_user: nil, + params: { + mail: mail + } + ).execute + + true + end + def project_from_key return unless match = service_desk_key.match(PROJECT_KEY_PATTERN) diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 51d250ea98c..ee11105537b 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -8,7 +8,7 @@ module Gitlab class Receiver include Gitlab::Utils::StrongMemoize - RECEIVED_HEADER_REGEX = /for\s+\<(.+)\>/.freeze + RECEIVED_HEADER_REGEX = /for\s+\<([^<]+)\>/.freeze # Errors that are purely from users and not anything we can control USER_ERRORS = [ diff --git a/lib/gitlab/error_tracking/error_repository.rb b/lib/gitlab/error_tracking/error_repository.rb index 3871305c9c5..79557838abf 100644 --- a/lib/gitlab/error_tracking/error_repository.rb +++ b/lib/gitlab/error_tracking/error_repository.rb @@ -15,12 +15,7 @@ module Gitlab # # @return [self] def self.build(project) - strategy = - if Feature.enabled?(:gitlab_error_tracking, project) - OpenApiStrategy.new(project) - else - ActiveRecordStrategy.new(project) - end + strategy = OpenApiStrategy.new(project) new(strategy) end diff --git a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb deleted file mode 100644 index 01e7fbda384..00000000000 --- a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ErrorTracking - class ErrorRepository - class ActiveRecordStrategy - def initialize(project) - @project = project - end - - def report_error( - name:, description:, actor:, platform:, - environment:, level:, occurred_at:, payload: - ) - error = project_errors.report_error( - name: name, # Example: ActionView::MissingTemplate - description: description, # Example: Missing template posts/show in... - actor: actor, # Example: PostsController#show - platform: platform, # Example: ruby - timestamp: occurred_at - ) - - # The payload field contains all the data on error including stacktrace in jsonb. - # Together with occurred_at these are 2 main attributes that we need to save here. - error.events.create!( - environment: environment, - description: description, - level: level, - occurred_at: occurred_at, - payload: payload - ) - rescue ActiveRecord::ActiveRecordError => e - handle_exceptions(e) - end - - def find_error(id) - project_error(id).to_sentry_detailed_error - rescue ActiveRecord::ActiveRecordError => e - handle_exceptions(e) - end - - def list_errors(filters:, query:, sort:, limit:, cursor:) - errors = project_errors - errors = filter_by_status(errors, filters[:status]) - errors = sort(errors, sort) - errors = errors.keyset_paginate(cursor: cursor, per_page: limit) - # query is not supported - - pagination = ErrorRepository::Pagination.new(errors.cursor_for_next_page, errors.cursor_for_previous_page) - - [errors.map(&:to_sentry_error), pagination] - end - - def last_event_for(id) - project_error(id).last_event&.to_sentry_error_event - rescue ActiveRecord::ActiveRecordError => e - handle_exceptions(e) - end - - def update_error(id, **attributes) - project_error(id).update(attributes) - end - - def dsn_url(public_key) - gitlab = Settings.gitlab - - custom_port = Settings.gitlab_on_standard_port? ? nil : ":#{gitlab.port}" - - base_url = [ - gitlab.protocol, - "://", - public_key, - '@', - gitlab.host, - custom_port, - gitlab.relative_url_root - ].join('') - - "#{base_url}/api/v4/error_tracking/collector/#{project.id}" - end - - private - - attr_reader :project - - def project_errors - ::ErrorTracking::Error.where(project: project) # rubocop:disable CodeReuse/ActiveRecord - end - - def project_error(id) - project_errors.find(id) - end - - def filter_by_status(errors, status) - return errors unless ::ErrorTracking::Error.statuses.key?(status) - - errors.for_status(status) - end - - def sort(errors, sort) - return errors.order_id_desc unless sort - - errors.sort_by_attribute(sort) - end - - def handle_exceptions(exception) - case exception - when ActiveRecord::RecordInvalid - raise RecordInvalidError, exception.message - else - raise DatabaseError, exception.message - end - end - end - end - end -end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index ab0df39e512..c141398bee0 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -6,7 +6,8 @@ module Gitlab module GrpcErrorProcessor extend Gitlab::ErrorTracking::Processor::Concerns::ProcessesExceptions - DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)') + # Braces added by gRPC Ruby code: https://github.com/grpc/grpc/blob/0e38b075ffff72ab2ad5326e3f60ba6dcc234f46/src/ruby/lib/grpc/errors.rb#L46 + DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:\{(.*)\}') class << self def call(event) diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index 52ad67d6f8b..a32f837eee9 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -21,6 +21,10 @@ module Gitlab payload['exception.cause_class'] = exception.cause.class.name end + if gitaly_metadata = find_gitaly_metadata(exception) + payload['exception.gitaly'] = gitaly_metadata.to_s + end + if sql = find_sql(exception) payload['exception.sql'] = sql end @@ -35,6 +39,16 @@ module Gitlab end end + def find_gitaly_metadata(exception) + if exception.is_a?(::Gitlab::Git::BaseError) + exception.metadata + elsif exception.is_a?(::GRPC::BadStatus) + exception.metadata[::Gitlab::Git::BaseError::METADATA_KEY] + elsif exception.cause.present? + find_gitaly_metadata(exception.cause) + end + end + private def normalize_query(sql) diff --git a/lib/gitlab/git/base_error.rb b/lib/gitlab/git/base_error.rb index 0b0fdef54cc..330e947844c 100644 --- a/lib/gitlab/git/base_error.rb +++ b/lib/gitlab/git/base_error.rb @@ -4,6 +4,7 @@ require 'grpc' module Gitlab module Git class BaseError < StandardError + METADATA_KEY = :gitaly_error_metadata DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m.freeze GRPC_CODES = { '0' => 'ok', @@ -25,12 +26,15 @@ module Gitlab '16' => 'unauthenticated' }.freeze - attr_reader :status, :code, :service + attr_reader :status, :code, :service, :metadata def initialize(msg = nil) super && return if msg.nil? - set_grpc_error_code(msg) if msg.is_a?(::GRPC::BadStatus) + if msg.is_a?(::GRPC::BadStatus) + set_grpc_error_code(msg) + set_grpc_error_metadata(msg) + end super(build_raw_message(msg)) end @@ -46,6 +50,10 @@ module Gitlab @code = GRPC_CODES[@status.to_s] @service = 'git' end + + def set_grpc_error_metadata(grpc_error) + @metadata = grpc_error.metadata.fetch(METADATA_KEY, {}).clone + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 11eb0a584ab..c0601c7795c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -320,7 +320,7 @@ module Gitlab end def first_ref_by_oid(repo) - ref = repo.refs_by_oid(oid: id, limit: 1)&.first + ref = repo.refs_by_oid(oid: id, limit: 1).first return unless ref diff --git a/lib/gitlab/git/finders/refs_finder.rb b/lib/gitlab/git/finders/refs_finder.rb new file mode 100644 index 00000000000..a0117bc0fa9 --- /dev/null +++ b/lib/gitlab/git/finders/refs_finder.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Finders + class RefsFinder + attr_reader :repository, :search, :ref_type + + UnknownRefTypeError = Class.new(StandardError) + + def initialize(repository, search:, ref_type:) + @repository = repository + @search = search + @ref_type = ref_type + end + + def execute + pattern = [prefix, search, "*"].compact.join + + repository.list_refs( + [pattern] + ) + end + + private + + def prefix + case ref_type + when :branches + Gitlab::Git::BRANCH_REF_PREFIX + when :tags + Gitlab::Git::TAG_REF_PREFIX + else + raise UnknownRefTypeError, "ref_type must be one of [:branches, :tags]" + end + end + end + end + end +end diff --git a/lib/gitlab/git/hook_env.rb b/lib/gitlab/git/hook_env.rb index f93ab19fc65..2524d4c4cfb 100644 --- a/lib/gitlab/git/hook_env.rb +++ b/lib/gitlab/git/hook_env.rb @@ -14,7 +14,7 @@ module Gitlab # # This class is thread-safe via RequestStore. class HookEnv - WHITELISTED_VARIABLES = %w[ + ALLOWLISTED_VARIABLES = %w[ GIT_OBJECT_DIRECTORY_RELATIVE GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze @@ -25,7 +25,7 @@ module Gitlab raise "missing gl_repository" if gl_repository.blank? Gitlab::SafeRequestStore[:gitlab_git_env] ||= {} - Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = whitelist_git_env(env) + Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = allowlist_git_env(env) end def self.all(gl_repository) @@ -46,8 +46,8 @@ module Gitlab env end - def self.whitelist_git_env(env) - env.select { |key, _| WHITELISTED_VARIABLES.include?(key.to_s) }.with_indifferent_access + def self.allowlist_git_env(env) + env.select { |key, _| ALLOWLISTED_VARIABLES.include?(key.to_s) }.with_indifferent_access end end end diff --git a/lib/gitlab/git/keep_around.rb b/lib/gitlab/git/keep_around.rb index 38f0e47c4c7..5835a7001af 100644 --- a/lib/gitlab/git/keep_around.rb +++ b/lib/gitlab/git/keep_around.rb @@ -19,6 +19,8 @@ module Gitlab end def execute(shas) + return if disabled? + shas.uniq.each do |sha| next unless sha.present? && commit_by(oid: sha) @@ -32,6 +34,8 @@ module Gitlab end def kept_around?(sha) + return true if disabled? + ref_exists?(keep_around_ref_name(sha)) end @@ -40,6 +44,11 @@ module Gitlab private + def disabled? + Feature.enabled?(:disable_keep_around_refs, @repository, type: :ops) || + (@repository.project && Feature.enabled?(:disable_keep_around_refs, @repository.project, type: :ops)) + end + def keep_around_ref_name(sha) "refs/#{::Repository::REF_KEEP_AROUND}/#{sha}" end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ed45d3eb030..71be986882c 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -337,9 +337,15 @@ module Gitlab # Return repo size in megabytes def size - size = gitaly_repository_client.repository_size + if Feature.enabled?(:use_repository_info_for_repository_size) + bytes = gitaly_repository_client.repository_info.size - (size.to_f / 1024).round(2) + (bytes.to_f / 1024 / 1024).round(2) + else + kilobytes = gitaly_repository_client.repository_size + + (kilobytes.to_f / 1024).round(2) + end end # Return git object directory size in bytes @@ -401,11 +407,12 @@ module Gitlab newrevs = newrevs.uniq.sort - @new_blobs ||= Hash.new do |h, revs| - h[revs] = blobs(['--not', '--all', '--not'] + newrevs, with_paths: true, dynamic_timeout: dynamic_timeout) - end - - @new_blobs[newrevs] + @new_blobs ||= {} + @new_blobs[newrevs] ||= blobs( + ['--not', '--all', '--not'] + newrevs, + with_paths: true, + dynamic_timeout: dynamic_timeout + ).to_a end # List blobs reachable via a set of revisions. Supports the @@ -554,10 +561,10 @@ module Gitlab # Limit of 0 means there is no limit. def refs_by_oid(oid:, limit: 0, ref_patterns: nil) wrapped_gitaly_errors do - gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit, ref_patterns: ref_patterns) + gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit, ref_patterns: ref_patterns) || [] end rescue CommandError, TypeError, NoRepository - nil + [] end # Returns url for submodule diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index df3d8165ef2..140dc791135 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -26,6 +26,11 @@ module Gitlab repository.gitaly_commit_client.tree_entries( repository, sha, path, recursive, skip_flat_paths, pagination_params) end + + # Incorrect revision or path could lead to index error. + # We silently handle such errors by returning an empty set of entries and cursor. + rescue Gitlab::Git::Index::IndexError + [[], nil] end private diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb index 3fe3702cfe1..37d3921d6d5 100644 --- a/lib/gitlab/gitaly_client/call.rb +++ b/lib/gitlab/gitaly_client/call.rb @@ -32,6 +32,8 @@ module Gitlab end rescue StandardError => err store_timings + set_gitaly_error_metadata(err) if err.is_a?(::GRPC::BadStatus) + raise err end @@ -44,6 +46,9 @@ module Gitlab yielder.yield(value) end + rescue ::GRPC::BadStatus => err + set_gitaly_error_metadata(err) + raise err ensure store_timings end @@ -73,6 +78,15 @@ module Gitlab backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) ) end + + def set_gitaly_error_metadata(err) + err.metadata[::Gitlab::Git::BaseError::METADATA_KEY] = { + storage: @storage, + address: ::Gitlab::GitalyClient.address(@storage), + service: @service, + rpc: @rpc + } + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index aa25fd3589a..c10f780665c 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -531,14 +531,24 @@ module Gitlab request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) response = gitaly_client_call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) - signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } + signatures = Hash.new do |h, k| + h[k] = { + signature: +''.b, + signed_text: +''.b, + signer: :SIGNER_UNSPECIFIED + } + end + current_commit_id = nil response.each do |message| current_commit_id = message.commit_id if message.commit_id.present? - signatures[current_commit_id].first << message.signature - signatures[current_commit_id].last << message.signed_text + signatures[current_commit_id][:signature] << message.signature + signatures[current_commit_id][:signed_text] << message.signed_text + + # The actual value is send once. All the other chunks send SIGNER_UNSPECIFIED + signatures[current_commit_id][:signer] = message.signer unless message.signer == :SIGNER_UNSPECIFIED end signatures @@ -585,9 +595,7 @@ module Gitlab end def call_commit_diff(request_params, options = {}) - request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) - - if Feature.enabled?(:add_ignore_all_white_spaces) && (request_params[:ignore_whitespace_change]) + if options.fetch(:ignore_whitespace_change, false) request_params[:whitespace_changes] = WHITESPACE_CHANGES['ignore_all_spaces'] end @@ -641,10 +649,6 @@ module Gitlab def find_changed_paths_request(commits, merge_commit_diff_mode) diff_mode = MERGE_COMMIT_DIFF_MODES[merge_commit_diff_mode] if Feature.enabled?(:merge_commit_diff_modes) - if Feature.disabled?(:find_changed_paths_new_format) - return Gitaly::FindChangedPathsRequest.new(repository: @gitaly_repo, commits: commits, merge_commit_diff_mode: diff_mode) - end - commit_requests = commits.map do |commit| Gitaly::FindChangedPathsRequest::Request.new( commit_request: Gitaly::FindChangedPathsRequest::Request::CommitRequest.new(commit_revision: commit) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index bd6cc9105d9..67e135bb530 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -135,7 +135,7 @@ module Gitlab end end - def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:, allow_conflicts: false) + def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:) request = Gitaly::UserMergeToRefRequest.new( repository: @gitaly_repo, source_sha: source_sha, @@ -144,7 +144,6 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, message: encode_binary(message), first_parent_ref: encode_binary(first_parent_ref), - allow_conflicts: allow_conflicts, timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 93d58710b0c..b5b7d94b4d0 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -52,6 +52,12 @@ module Gitlab response.size end + def repository_info + request = Gitaly::RepositoryInfoRequest.new(repository: @gitaly_repo) + + gitaly_client_call(@storage, :repository_service, :repository_info, request, timeout: GitalyClient.long_timeout) + end + def get_object_directory_size request = Gitaly::GetObjectDirectorySizeRequest.new(repository: @gitaly_repo) response = gitaly_client_call(@storage, :repository_service, :get_object_directory_size, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 9556a9e98ba..24e77363e1b 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -8,12 +8,18 @@ module Gitlab def self.new_client_for(project, token: nil, host: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) - Client.new( - token_to_use, + token_pool = project.import_data&.credentials&.dig(:additional_access_tokens) + options = { host: host.presence || self.formatted_import_url(project), per_page: self.per_page(project), parallel: parallel - ) + } + + if token_pool + ClientPool.new(token_pool: token_pool, **options) + else + Client.new(token_to_use, **options) + end end # Returns the ID of the ghost user. diff --git a/lib/gitlab/github_import/client_pool.rb b/lib/gitlab/github_import/client_pool.rb new file mode 100644 index 00000000000..e8414942d1b --- /dev/null +++ b/lib/gitlab/github_import/client_pool.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ClientPool + delegate_missing_to :best_client + + def initialize(token_pool:, per_page:, parallel:, host: nil) + @token_pool = token_pool + @host = host + @per_page = per_page + @parallel = parallel + end + + # Returns the client with the most remaining requests, or the client with + # the closest rate limit reset time, if all clients are rate limited. + def best_client + clients_with_requests_remaining = clients.select(&:requests_remaining?) + + return clients_with_requests_remaining.max_by(&:remaining_requests) if clients_with_requests_remaining.any? + + clients.min_by(&:rate_limit_resets_in) + end + + private + + def clients + @clients ||= @token_pool.map do |token| + Client.new( + token, + host: @host, + per_page: @per_page, + parallel: @parallel + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index b477468d327..a537841ecf3 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -31,7 +31,7 @@ module Gitlab if (issue_id = create_issue) create_assignees(issue_id) issuable_finder.cache_database_id(issue_id) - update_search_data(issue_id) if Feature.enabled?(:issues_full_text_search) + update_search_data(issue_id) end end end diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index 0b883de8ed0..73a5f49a9e3 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -56,8 +56,16 @@ module Gitlab def write(user_settings) user_settings = user_settings.to_h.with_indifferent_access - optional_stages = fetch_stages_from_params(user_settings) - import_data = project.create_or_update_import_data(data: { optional_stages: optional_stages }) + optional_stages = fetch_stages_from_params(user_settings[:optional_stages]) + credentials = project.import_data&.credentials&.merge( + additional_access_tokens: user_settings[:additional_access_tokens] + ) + + import_data = project.create_or_update_import_data( + data: { optional_stages: optional_stages }, + credentials: credentials + ) + import_data.save! end @@ -74,6 +82,8 @@ module Gitlab attr_reader :project def fetch_stages_from_params(user_settings) + user_settings = user_settings.to_h.with_indifferent_access + OPTIONAL_STAGES.keys.to_h do |stage_name| enabled = Gitlab::Utils.to_boolean(user_settings[stage_name], default: false) [stage_name, enabled] diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index dd71edbd205..57365ebe206 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -28,9 +28,6 @@ module Gitlab EMAIL_FOR_USERNAME_CACHE_KEY = 'github-import/user-finder/email-for-username/%s' - # The base cache key to use for caching inexistence of GitHub usernames. - INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY = 'github-import/user-finder/inexistence-of-username/%s' - # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` def initialize(project, client) @@ -112,18 +109,24 @@ module Gitlab id_for_github_id(id) || id_for_github_email(email) end + # Find the public email of a given username in GitHub. The public email is cached to avoid multiple calls to + # GitHub. In case the username does not exist or the public email is nil, a blank value is cached to also prevent + # multiple calls to GitHub. + # + # @return [String] If public email is found + # @return [Nil] If public email or username does not exist def email_for_github_username(username) cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username email = Gitlab::Cache::Import::Caching.read(cache_key) - if email.blank? && !github_username_inexists?(username) + if email.nil? user = client.user(username) - email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email], timeout: timeout(user[:email])) if user + email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email].to_s, timeout: timeout(user[:email])) end - email + email.presence rescue ::Octokit::NotFound - cache_github_username_inexistence(username) + Gitlab::Cache::Import::Caching.write(cache_key, '') nil end @@ -196,18 +199,6 @@ module Gitlab Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT end end - - def github_username_inexists?(username) - cache_key = INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % username - - Gitlab::Cache::Import::Caching.read(cache_key) == 'true' - end - - def cache_github_username_inexistence(username) - cache_key = INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % username - - Gitlab::Cache::Import::Caching.write(cache_key, true) - end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9eeea7336b5..ff171c24549 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -75,6 +75,7 @@ module Gitlab # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) push_frontend_feature_flag(:gitlab_duo, current_user) + push_frontend_feature_flag(:custom_emoji) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index a03aeb9c293..1fc95181767 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -87,6 +87,7 @@ module Gitlab end def verification_status(gpg_key) + return :verified_system if verified_by_gitlab? return :multiple_signatures if multiple_signatures? return :unknown_key unless gpg_key return :unverified_key unless gpg_key.verified? @@ -101,6 +102,15 @@ module Gitlab end end + # If a commit is signed by Gitaly, the Gitaly returns `SIGNER_SYSTEM` as a signer + # In order to calculate it, the signature is Verified using the Gitaly's public key: + # https://gitlab.com/gitlab-org/gitaly/-/blob/v16.2.0-rc2/internal/gitaly/service/commit/commit_signatures.go#L63 + # + # It is safe to skip verification step if the commit has been signed by Gitaly + def verified_by_gitlab? + signer == :SIGNER_SYSTEM + end + def user_infos(gpg_key) gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end diff --git a/lib/gitlab/grape_logging/loggers/response_logger.rb b/lib/gitlab/grape_logging/loggers/response_logger.rb index 767c282d62e..b87566a62b0 100644 --- a/lib/gitlab/grape_logging/loggers/response_logger.rb +++ b/lib/gitlab/grape_logging/loggers/response_logger.rb @@ -5,8 +5,6 @@ module Gitlab module Loggers class ResponseLogger < ::GrapeLogging::Loggers::Base def parameters(_, response) - return {} unless Feature.enabled?(:log_response_length) - response_bytes = 0 case response diff --git a/lib/gitlab/graphql/generic_tracing.rb b/lib/gitlab/graphql/generic_tracing.rb deleted file mode 100644 index dc3f6574631..00000000000 --- a/lib/gitlab/graphql/generic_tracing.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -# This class is used as a hook to observe graphql runtime events. From this -# hook both gitlab metrics and opentracking measurements are generated - -module Gitlab - module Graphql - class GenericTracing < GraphQL::Tracing::PlatformTracing - self.platform_keys = { - 'lex' => 'graphql.lex', - 'parse' => 'graphql.parse', - 'validate' => 'graphql.validate', - 'analyze_query' => 'graphql.analyze', - 'analyze_multiplex' => 'graphql.analyze', - 'execute_multiplex' => 'graphql.execute', - 'execute_query' => 'graphql.execute', - 'execute_query_lazy' => 'graphql.execute', - 'execute_field' => 'graphql.execute', - 'execute_field_lazy' => 'graphql.execute' - } - - def platform_field_key(type, field) - "#{type.name}.#{field.name}" - end - - def platform_authorized_key(type) - "#{type.graphql_name}.authorized" - end - - def platform_resolve_type_key(type) - "#{type.graphql_name}.resolve_type" - end - - def platform_trace(platform_key, key, data, &block) - tags = { platform_key: platform_key, key: key } - start = Gitlab::Metrics::System.monotonic_time - - with_labkit_tracing(tags, &block) - ensure - duration = Gitlab::Metrics::System.monotonic_time - start - - graphql_duration_seconds.observe(tags, duration) unless deactivated? - end - - private - - def deactivated? - Feature.enabled?(:graphql_generic_tracing_metrics_deactivate) - end - - def with_labkit_tracing(tags, &block) - return yield unless Labkit::Tracing.enabled? - - name = "#{tags[:platform_key]}.#{tags[:key]}" - span_tags = { - 'component' => 'web', - 'span.kind' => 'server' - }.merge(tags.stringify_keys) - - Labkit::Tracing.with_tracing(operation_name: name, tags: span_tags, &block) - end - - def graphql_duration_seconds - @graphql_duration_seconds ||= Gitlab::Metrics.histogram( - :graphql_duration_seconds, - 'GraphQL execution time' - ) - end - end - end -end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index b112740c4ad..8ca88859b22 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -13,6 +13,7 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def users groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user) + groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/417455") members = GroupMember.where(group: groups).non_invite users = super diff --git a/lib/gitlab/hook_data/emoji_builder.rb b/lib/gitlab/hook_data/emoji_builder.rb new file mode 100644 index 00000000000..673eb516e43 --- /dev/null +++ b/lib/gitlab/hook_data/emoji_builder.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class EmojiBuilder < BaseBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + user_id + created_at + id + name + awardable_type + awardable_id + updated_at + ].freeze + + alias_method :award_emoji, :object + + def build + award_emoji + .attributes + .with_indifferent_access + .slice(*SAFE_HOOK_ATTRIBUTES) + end + end + end +end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 180ccf21264..fabc02af70a 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 31, - 'de' => 97, + 'da_DK' => 30, + 'de' => 99, 'en' => 100, 'eo' => 0, - 'es' => 30, + 'es' => 29, 'fil_PH' => 0, - 'fr' => 98, + 'fr' => 99, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 98, - 'ko' => 18, + 'ja' => 99, + 'ko' => 17, 'nb_NO' => 22, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 56, - 'ro_RO' => 82, + 'pt_BR' => 57, + 'ro_RO' => 80, 'ru' => 23, 'si_LK' => 10, 'tr_TR' => 9, 'uk' => 53, 'zh_CN' => 98, 'zh_HK' => 1, - 'zh_TW' => 99 + 'zh_TW' => 100 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index c2a1a1f8575..7a91cfb340a 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -7,12 +7,10 @@ tree: group: - :milestones - :badges - - labels: - - :priorities + - :labels - boards: - lists: - - label: - - :priorities + - :label - :board - members: - :user @@ -126,8 +124,7 @@ ee: - boards: - :board_assignee - :milestone - - labels: - - :priorities + - :labels - lists: - milestone: - events: diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index 1b8436c4ed9..664ef5358ef 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -6,7 +6,6 @@ module Gitlab class RelationFactory < Base::RelationFactory OVERRIDES = { labels: :group_labels, - priorities: :label_priorities, label: :group_label, parent: :epic, iterations_cadences: 'Iterations::Cadence' diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 410e918649b..5986c5de441 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -44,6 +44,7 @@ tree: - :zoom_meetings - :sentry_issue - :award_emoji + - :work_item_type - snippets: - :award_emoji - notes: @@ -771,6 +772,8 @@ included_attributes: - :source_commit - :close_after_error_tracking_resolve - :close_auto_resolve_prometheus_alert + work_item_type: + - :base_type # Do not include the following attributes for the models specified. excluded_attributes: @@ -1101,6 +1104,15 @@ excluded_attributes: - :roll_over - :description - :sequence + work_item_type: + - :id + - :cached_markdown_version + - :name + - :description + - :description_html + - :icon_name + - :namespace_id + - :updated_at methods: project: diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index ac28ae6bfe0..5534a0f2aa4 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -24,6 +24,7 @@ module Gitlab return if group_relation_without_group? return find_diff_commit_user if diff_commit_user? return find_diff_commit if diff_commit? + return find_work_item_type if work_item_type? super end @@ -142,6 +143,10 @@ module Gitlab klass == MergeRequestDiffCommit end + def work_item_type? + klass == ::WorkItems::Type + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -166,6 +171,18 @@ module Gitlab def group_level_object? epic? end + + def find_work_item_type + base_type = @attributes['base_type'] + + find_with_cache([::WorkItems::Type, base_type]) do + if ::WorkItems::Type.base_types.key?(base_type) + ::WorkItems::Type.default_by_type(base_type) + else + ::WorkItems::Type.default_issue_type + end + end + end end end end diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 8c673acdd1a..7af65235492 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -16,6 +16,8 @@ module Gitlab bridges: 'Ci::Bridge', runners: 'Ci::Runner', pipeline_metadata: 'Ci::PipelineMetadata', + external_pull_request: 'Ci::ExternalPullRequest', + external_pull_requests: 'Ci::ExternalPullRequest', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -39,7 +41,8 @@ module Gitlab metrics_setting: 'ProjectMetricsSetting', commit_author: 'MergeRequest::DiffCommitUser', committer: 'MergeRequest::DiffCommitUser', - merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze + merge_request_diff_commits: 'MergeRequestDiffCommit', + work_item_type: 'WorkItems::Type' }.freeze BUILD_MODELS = %i[Ci::Build Ci::Bridge commit_status generic_commit_status].freeze @@ -61,11 +64,11 @@ module Gitlab epic ProjectCiCdSetting container_expiration_policy - external_pull_request - external_pull_requests + Ci::ExternalPullRequest DesignManagement::Design MergeRequest::DiffCommitUser MergeRequestDiffCommit + WorkItems::Type ].freeze def create @@ -90,7 +93,7 @@ module Gitlab when :notes, :Note then setup_note when :'Ci::Pipeline' then setup_pipeline when *BUILD_MODELS then setup_build - when :issues then setup_issue + when :issues then setup_work_item when :'Ci::PipelineSchedule' then setup_pipeline_schedule when :'ProtectedBranch::MergeAccessLevel' then setup_protected_branch_access_level when :'ProtectedBranch::PushAccessLevel' then setup_protected_branch_access_level @@ -166,8 +169,11 @@ module Gitlab end end - def setup_issue + def setup_work_item @relation_hash['relative_position'] = compute_relative_position + + issue_type = @relation_hash.delete('issue_type') + @relation_hash['work_item_type'] ||= ::WorkItems::Type.default_by_type(issue_type) if issue_type end def setup_release diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb index cde83068de1..92bf2a826ff 100644 --- a/lib/gitlab/internal_events.rb +++ b/lib/gitlab/internal_events.rb @@ -2,21 +2,38 @@ module Gitlab module InternalEvents + UnknownEventError = Class.new(StandardError) + InvalidPropertyError = Class.new(StandardError) + InvalidMethodError = Class.new(StandardError) + class << self include Gitlab::Tracking::Helpers def track_event(event_name, **kwargs) - user_id = kwargs.delete(:user_id) - UsageDataCounters::HLLRedisCounter.track_event(event_name, values: user_id) + raise UnknownEventError, "Unknown event: #{event_name}" unless EventDefinitions.known_event?(event_name) + + unique_property = EventDefinitions.unique_property(event_name) + unique_method = :id + + unless kwargs.has_key?(unique_property) + raise InvalidPropertyError, "#{event_name} should be triggered with a named parameter '#{unique_property}'." + end + + unless kwargs[unique_property].respond_to?(unique_method) + raise InvalidMethodError, "'#{unique_property}' should have a '#{unique_method}' method." + end + + unique_value = kwargs[unique_property].public_send(unique_method) # rubocop:disable GitlabSecurity/PublicSend - project_id = kwargs.delete(:project_id) - namespace_id = kwargs.delete(:namespace_id) + UsageDataCounters::HLLRedisCounter.track_event(event_name, values: unique_value) - namespace = Namespace.find(namespace_id) if namespace_id + user = kwargs[:user] + project = kwargs[:project] + namespace = kwargs[:namespace] standard_context = Tracking::StandardContext.new( - project_id: project_id, - user_id: user_id, + project_id: project&.id, + user_id: user&.id, namespace_id: namespace&.id, plan_name: namespace&.actual_plan_name ).to_context @@ -27,6 +44,9 @@ module Gitlab ).to_context track_struct_event(event_name, contexts: [standard_context, service_ping_context]) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: kwargs) + nil end private diff --git a/lib/gitlab/internal_events/event_definitions.rb b/lib/gitlab/internal_events/event_definitions.rb new file mode 100644 index 00000000000..e1c9faa12de --- /dev/null +++ b/lib/gitlab/internal_events/event_definitions.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module InternalEvents + module EventDefinitions + InvalidMetricConfiguration = Class.new(StandardError) + + class << self + VALID_UNIQUE_VALUES = %w[user.id project.id namespace.id].freeze + + def clear_events + @events = nil + end + + def load_configurations + @events = load_metric_definitions + nil + end + + def unique_property(event_name) + unique_value = events[event_name]&.to_s + + raise(InvalidMetricConfiguration, "Unique property not defined for #{event_name}") unless unique_value + + unless VALID_UNIQUE_VALUES.include?(unique_value) + raise(InvalidMetricConfiguration, "Invalid unique value '#{unique_value}' for #{event_name}") + end + + unique_value.split('.').first.to_sym + end + + def known_event?(event_name) + events.key?(event_name) + end + + private + + def events + load_configurations if @events.nil? || Gitlab::Usage::MetricDefinition.metric_definitions_changed? + + @events + end + + def load_metric_definitions + all_events = {} + + Gitlab::Usage::MetricDefinition.all.each do |metric_definition| + next unless metric_definition.available? + + process_events(all_events, metric_definition.events) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + + all_events + end + + def process_events(all_events, metric_events) + metric_events.each do |event_name, event_unique_attribute| + unless all_events[event_name] + all_events[event_name] = event_unique_attribute + next + end + + next if event_unique_attribute.nil? || event_unique_attribute == all_events[event_name] + + raise InvalidMetricConfiguration, + "The same event cannot have several unique properties defined. " \ + "Event: #{event_name}, unique values: #{event_unique_attribute}, #{all_events[event_name]}" + end + end + end + end + end +end diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb index f1f6cc55a2b..12cc5f6e5dd 100644 --- a/lib/gitlab/issues/rebalancing/state.rb +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -53,7 +53,7 @@ module Gitlab end def can_start_rebalance? - rebalance_in_progress? || too_many_rebalances_running? + rebalance_in_progress? || concurrent_rebalance_within_limit? end def cache_issue_ids(issue_ids) @@ -100,11 +100,11 @@ module Gitlab def refresh_keys_expiration with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.multi do |multi| - multi.expire(issue_ids_key, REDIS_EXPIRY_TIME) - multi.expire(current_index_key, REDIS_EXPIRY_TIME) - multi.expire(current_project_key, REDIS_EXPIRY_TIME) - multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) + redis.pipelined do |pipeline| + pipeline.expire(issue_ids_key, REDIS_EXPIRY_TIME) + pipeline.expire(current_index_key, REDIS_EXPIRY_TIME) + pipeline.expire(current_project_key, REDIS_EXPIRY_TIME) + pipeline.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) end end end @@ -113,16 +113,20 @@ module Gitlab def cleanup_cache value = "#{rebalanced_container_type}/#{rebalanced_container_id}" + # The clean up is done sequentially to be compatible with Redis Cluster + # Do not use a pipeline as it fans-out in a Redis-Cluster setting and forego ordering guarantees with_redis do |redis| - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.multi do |multi| - multi.del(issue_ids_key) - multi.del(current_index_key) - multi.del(current_project_key) - multi.srem?(CONCURRENT_RUNNING_REBALANCES_KEY, value) - multi.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour) - end - end + # srem followed by .del(issue_ids_key) to ensure that any subsequent redis errors would + # result in a no-op job retry since current_index_key still exists + redis.srem?(CONCURRENT_RUNNING_REBALANCES_KEY, value) + redis.del(issue_ids_key) + + # delete current_index_key to ensure that subsequent redis errors would + # result in a fresh job retry + redis.del(current_index_key) + + # setting recently_finished_key last after job details is cleaned up + redis.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour) end end @@ -159,7 +163,7 @@ module Gitlab attr_accessor :root_namespace, :projects, :rebalanced_container_type, :rebalanced_container_id - def too_many_rebalances_running? + def concurrent_rebalance_within_limit? concurrent_running_rebalances_count <= MAX_NUMBER_OF_CONCURRENT_REBALANCES end diff --git a/lib/gitlab/jwt_authenticatable.rb b/lib/gitlab/jwt_authenticatable.rb index 7c36bbf3426..d7a341b3ba2 100644 --- a/lib/gitlab/jwt_authenticatable.rb +++ b/lib/gitlab/jwt_authenticatable.rb @@ -13,8 +13,8 @@ module Gitlab module ClassMethods include Gitlab::Utils::StrongMemoize - def decode_jwt(encoded_message, jwt_secret = secret, issuer: nil, iat_after: nil) - options = { algorithm: 'HS256' } + def decode_jwt(encoded_message, jwt_secret = secret, algorithm: 'HS256', issuer: nil, iat_after: nil) + options = { algorithm: algorithm } options = options.merge(iss: issuer, verify_iss: true) if issuer.present? options = options.merge(verify_iat: true) if iat_after.present? diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb index 43546d04087..fe244bd88a0 100644 --- a/lib/gitlab/kas/client.rb +++ b/lib/gitlab/kas/client.rb @@ -31,7 +31,7 @@ module Gitlab def list_agent_config_files(project:) request = Gitlab::Agent::ConfigurationProject::Rpc::ListAgentConfigFilesRequest.new( repository: repository(project), - gitaly_address: gitaly_address(project) + gitaly_info: gitaly_info(project) ) stub_for(:configuration_project) @@ -42,9 +42,11 @@ module Gitlab def send_git_push_event(project:) request = Gitlab::Agent::Notifications::Rpc::GitPushEventRequest.new( - project: Gitlab::Agent::Notifications::Rpc::Project.new( - id: project.id, - full_path: project.full_path + event: Gitlab::Agent::Event::GitPushEvent.new( + project: Gitlab::Agent::Event::Project.new( + id: project.id, + full_path: project.full_path + ) ) ) @@ -62,13 +64,15 @@ module Gitlab def repository(project) gitaly_repository = project.repository.gitaly_repository - Gitlab::Agent::Modserver::Repository.new(gitaly_repository.to_h) + Gitlab::Agent::Entity::GitalyRepository.new(gitaly_repository.to_h) end - def gitaly_address(project) + def gitaly_info(project) + gitaly_features = Feature::Gitaly.server_feature_flags connection_data = Gitlab::GitalyClient.connection_data(project.repository_storage) + .merge(features: gitaly_features) - Gitlab::Agent::Modserver::GitalyAddress.new(connection_data) + Gitlab::Agent::Entity::GitalyInfo.new(connection_data) end def kas_endpoint_url diff --git a/lib/gitlab/kas/user_access.rb b/lib/gitlab/kas/user_access.rb index 65ae399d826..587aa4803c6 100644 --- a/lib/gitlab/kas/user_access.rb +++ b/lib/gitlab/kas/user_access.rb @@ -9,11 +9,7 @@ module Gitlab class UserAccess class << self def enabled? - ::Gitlab::Kas.enabled? && ::Feature.enabled?(:kas_user_access) - end - - def enabled_for?(agent) - enabled? && ::Feature.enabled?(:kas_user_access_project, agent.project) + ::Gitlab::Kas.enabled? end def encrypt_public_session_id(data) diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index f8ec58cf217..9abad44b10e 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -36,10 +36,6 @@ module Gitlab payload[:feature_flag_states] = Feature.logged_states.map { |key, state| "#{key}:#{state ? 1 : 0}" } end - if Feature.disabled?(:log_response_length) - payload.delete(:response_bytes) - end - payload end end diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb index 3747431c6a7..81711be729e 100644 --- a/lib/gitlab/manifest_import/metadata.rb +++ b/lib/gitlab/manifest_import/metadata.rb @@ -4,6 +4,7 @@ module Gitlab module ManifestImport class Metadata EXPIRY_TIME = 1.week + KEY_PREFIX = 'manifest_import:metadata:user' attr_reader :user, :fallback @@ -14,11 +15,9 @@ module Gitlab def save(repositories, group_id) Gitlab::Redis::SharedState.with do |redis| - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.multi do |multi| - multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) - multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) - end + redis.multi do |multi| + multi.set(hashtag_key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + multi.set(hashtag_key_for('group_id'), group_id, ex: EXPIRY_TIME) end end end @@ -37,13 +36,17 @@ module Gitlab private + def hashtag_key_for(field) + "#{KEY_PREFIX}:{#{user.id}}:#{field}" + end + def key_for(field) - "manifest_import:metadata:user:#{user.id}:#{field}" + "#{KEY_PREFIX}:#{user.id}:#{field}" end def redis_get(field) Gitlab::Redis::SharedState.with do |redis| - redis.get(key_for(field)) + redis.get(hashtag_key_for(field)) || redis.get(key_for(field)) end end end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index f742cb82b8d..52260623c55 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -11,9 +11,11 @@ module Gitlab Gitlab::Redis::Cache.with do |r| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| - subjects.each do |subject| - results[subject.cache_key] = new(subject).read(pipeline) + r.with_readonly_pipeline do + Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| + subjects.each do |subject| + results[subject.cache_key] = new(subject).read(pipeline) + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb index 31d75225972..56a82d1df46 100644 --- a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb @@ -7,59 +7,14 @@ module Gitlab class ClusterEndpointInserter < BaseStage def transform! verify_params - - for_metrics do |metric| - metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) - end end private - def admin_url(metric) - Gitlab::Routing.url_helpers.prometheus_api_admin_cluster_path( - params[:cluster], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) - end - - def endpoint_for_metric(metric) - case params[:cluster_type] - when :admin - admin_url(metric) - when :group - error!(_('Group is required when cluster_type is :group')) unless params[:group] - group_url(metric) - when :project - error!(_('Project is required when cluster_type is :project')) unless project - project_url(metric) - else - error!(_('Unrecognized cluster type')) - end - end - def error!(message) raise Errors::DashboardProcessingError, message end - def group_url(metric) - Gitlab::Routing.url_helpers.prometheus_api_group_cluster_path( - params[:group], - params[:cluster], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) - end - - def project_url(metric) - Gitlab::Routing.url_helpers.prometheus_api_project_cluster_path( - project, - params[:cluster], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) - end - def query_type(metric) metric[:query] ? :query : :query_range end diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb index 622b6adec7e..03370ae7370 100644 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -53,8 +53,7 @@ module Gitlab { id: "#{metric[:legendFormat]}_#{idx}", query_range: format_query(metric), - label: replace_variables(metric[:legendFormat]), - prometheus_endpoint_path: prometheus_endpoint_for_metric(metric) + label: replace_variables(metric[:legendFormat]) }.compact end @@ -89,17 +88,6 @@ module Gitlab end end - # Endpoint which will return prometheus metric data - # for the metric - def prometheus_endpoint_for_metric(metric) - Gitlab::Routing.url_helpers.project_grafana_api_path( - project, - datasource_id: datasource[:id], - proxy_path: PROXY_PATH, - query: format_query(metric) - ) - end - # Reformats query for compatibility with prometheus api. def format_query(metric) expression = remove_new_lines(metric[:expr]) diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index bdd28744137..e7b901861ef 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -12,26 +12,6 @@ module Gitlab ANCHOR_PATTERN = '(?<anchor>\#[a-z0-9_-]+)?' DASH_PATTERN = '(?:/-)' - # Matches urls for a metrics dashboard. - # This regex needs to match the old metrics URL, the new metrics URL, - # and the dashboard URL (inline_metrics_redactor_filter.rb - # uses this regex to match against the dashboard URL.) - # - # EX - Old URL: https://<host>/<namespace>/<project>/environments/<env_id>/metrics - # OR - # New URL: https://<host>/<namespace>/<project>/-/metrics?environment=<env_id> - # OR - # dashboard URL: https://<host>/<namespace>/<project>/environments/<env_id>/metrics_dashboard - def metrics_regex - strong_memoize(:metrics_regex) do - regex_for_project_metrics( - %r{ - ( #{environment_metrics_regex} ) | ( #{non_environment_metrics_regex} ) - }x - ) - end - end - # Matches dashboard urls for a Grafana embed. # # EX - https://<host>/<namespace>/<project>/grafana/metrics_dashboard @@ -99,11 +79,6 @@ module Gitlab .symbolize_keys end - # Builds a metrics dashboard url based on the passed in arguments - def build_dashboard_url(...) - Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(...) - end - private def environment_metrics_regex diff --git a/lib/gitlab/metrics/sidekiq_slis.rb b/lib/gitlab/metrics/sidekiq_slis.rb index f28cf4ac967..748666f2200 100644 --- a/lib/gitlab/metrics/sidekiq_slis.rb +++ b/lib/gitlab/metrics/sidekiq_slis.rb @@ -8,16 +8,26 @@ module Gitlab "low" => 300, "throttled" => 300 }.freeze + QUEUEING_URGENCY_DURATIONS = { + "high" => 10, + "low" => 60, + "throttled" => Float::INFINITY # no queueing target duration for throttled urgency + }.freeze # workers without urgency attribute have "low" urgency by default in # WorkerAttributes.get_urgency, just mirroring it here DEFAULT_EXECUTION_URGENCY_DURATION = EXECUTION_URGENCY_DURATIONS["low"] + DEFAULT_QUEUEING_URGENCY_DURATION = QUEUEING_URGENCY_DURATIONS["low"] class << self - def initialize_slis!(possible_labels) + def initialize_execution_slis!(possible_labels) Gitlab::Metrics::Sli::Apdex.initialize_sli(:sidekiq_execution, possible_labels) Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:sidekiq_execution, possible_labels) end + def initialize_queueing_slis!(possible_labels) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:sidekiq_queueing, possible_labels) + end + def record_execution_apdex(labels, job_completion_duration) urgency_requirement = execution_duration_for_urgency(labels[:urgency]) Gitlab::Metrics::Sli::Apdex[:sidekiq_execution].increment( @@ -30,9 +40,21 @@ module Gitlab Gitlab::Metrics::Sli::ErrorRate[:sidekiq_execution].increment(labels: labels, error: error) end + def record_queueing_apdex(labels, queue_duration) + urgency_requirement = queueing_duration_for_urgency(labels[:urgency]) + Gitlab::Metrics::Sli::Apdex[:sidekiq_queueing].increment( + labels: labels, + success: queue_duration < urgency_requirement + ) + end + def execution_duration_for_urgency(urgency) EXECUTION_URGENCY_DURATIONS.fetch(urgency, DEFAULT_EXECUTION_URGENCY_DURATION) end + + def queueing_duration_for_urgency(urgency) + QUEUEING_URGENCY_DURATIONS.fetch(urgency, DEFAULT_QUEUEING_URGENCY_DURATION) + end end end end diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index a83cdbe15df..e7790fd77d0 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -21,7 +21,7 @@ module Gitlab href: href, view: view.to_s, css_class: css_class, - data: data || { qa_selector: 'menu_item_link', qa_title: title }, + data: data || { testid: 'menu_item_link', qa_title: title }, partial: partial, component: component } diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb index f7f65c91339..b500df86363 100644 --- a/lib/gitlab/observability.rb +++ b/lib/gitlab/observability.rb @@ -23,7 +23,22 @@ module Gitlab 'https://observe.gitlab.com' end - # Returns true if the Observability feature flag is enabled + def oauth_url + "#{Gitlab::Observability.observability_url}/v1/auth/start" + end + + def tracing_url(project) + "#{Gitlab::Observability.observability_url}/query/#{project.group.id}/#{project.id}/v1/traces" + end + + def provisioning_url(_project) + # TODO Change to correct endpoint when API is ready + Gitlab::Observability.observability_url.to_s + end + + # Returns true if the GitLab Observability UI (GOUI) feature flag is enabled + # + # @deprecated # def enabled?(group = nil) return Feature.enabled?(:observability_group_tab, group) if group @@ -31,6 +46,11 @@ module Gitlab Feature.enabled?(:observability_group_tab) end + # Returns true if Tracing UI is enabled + def tracing_enabled?(project) + Feature.enabled?(:observability_tracing, project) + end + # Returns the embeddable Observability URL of a given URL # # - Validates the URL diff --git a/lib/gitlab/pages/url_builder.rb b/lib/gitlab/pages/url_builder.rb new file mode 100644 index 00000000000..215154b7248 --- /dev/null +++ b/lib/gitlab/pages/url_builder.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class UrlBuilder + attr_reader :project_namespace + + ALLOWED_ARTIFACT_EXTENSIONS = %w[.html .htm .txt .json .xml .log].freeze + ARTIFACT_URL = "%{host}/-/%{project_path}/-/jobs/%{job_id}/artifacts/%{artifact_path}" + + def initialize(project) + @project = project + @project_namespace, _, @project_path = project.full_path.partition('/') + end + + def pages_url(with_unique_domain: false) + return unique_url if with_unique_domain && unique_domain_enabled? + + project_path_url = "#{config.protocol}://#{project_path}".downcase + + # If the project path is the same as host, we serve it as group page + # On development we ignore the URL port to make it work on GDK + return namespace_url if Rails.env.development? && portless(namespace_url) == project_path_url + # If the project path is the same as host, we serve it as group page + return namespace_url if namespace_url == project_path_url + + "#{namespace_url}/#{project_path}" + end + + def unique_host + return unless unique_domain_enabled? + + URI(unique_url).host + end + + def namespace_pages? + namespace_url == pages_url + end + + def artifact_url(artifact, job) + return unless artifact_url_available?(artifact, job) + + format( + ARTIFACT_URL, + host: namespace_url, + project_path: project_path, + job_id: job.id, + artifact_path: artifact.path) + end + + def artifact_url_available?(artifact, job) + config.enabled && + config.artifacts_server && + ALLOWED_ARTIFACT_EXTENSIONS.include?(File.extname(artifact.name)) && + (config.access_control || job.project.public?) + end + + private + + attr_reader :project, :project_path + + def namespace_url + @namespace_url ||= url_for(project_namespace) + end + + def unique_url + @unique_url ||= url_for(project.project_setting.pages_unique_domain) + end + + def url_for(subdomain) + URI(config.url) + .tap { |url| url.port = config.port } + .tap { |url| url.host.prepend("#{subdomain}.") } + .to_s + .downcase + end + + def portless(url) + URI(url) + .tap { |u| u.port = nil } + .to_s + end + + def unique_domain_enabled? + Feature.enabled?(:pages_unique_domain, project) && + project.project_setting.pages_unique_domain_enabled? + end + + def config + Gitlab.config.pages + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 8c0f082f61c..422839dcde1 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -67,7 +67,7 @@ module Gitlab .select(finder_strategy.final_projections) .where("count <> 0") # filter out the initializer row - model.from(q.arel.as(table_name)) + model.select(Arel.star).from(q.arel.as(table_name)) end private diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb index 4f79a3593f4..786ae282c88 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -14,6 +14,7 @@ module Gitlab @finder_query = finder_query @order_by_columns = order_by_columns @table_name = model.table_name + @model = model end def initializer_columns @@ -30,7 +31,11 @@ module Gitlab end def final_projections - ["(#{RECORDS_COLUMN}).*"] + if @model.default_select_columns.is_a?(Array) + @model.default_select_columns.map { |column| "(#{RECORDS_COLUMN}).#{column.name}" } + else + ["(#{RECORDS_COLUMN}).*"] + end end private diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 0d8e4ea6fee..a7faef2fdad 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -98,7 +98,7 @@ module Gitlab hash[column_definition.attribute_name] = if field_value.is_a?(Time) # use :inspect formatter to provide specific timezone info # eg 2022-07-05 21:57:56.041499000 +0800 - field_value.to_s(:inspect) + field_value.to_fs(:inspect) elsif field_value.nil? nil elsif lower_named_function?(column_definition) @@ -246,7 +246,8 @@ module Gitlab scopes = where_values.map do |where_value| scope.dup.where(where_value).reorder(self) # rubocop: disable CodeReuse/ActiveRecord end - scope.model.from_union(scopes, remove_duplicates: false, remove_order: false) + + scope.model.select(scope.select_values).from_union(scopes, remove_duplicates: false, remove_order: false) end def to_sql_literal(column_definitions) diff --git a/lib/gitlab/patch/action_cable_redis_listener.rb b/lib/gitlab/patch/action_cable_redis_listener.rb deleted file mode 100644 index b21bee45991..00000000000 --- a/lib/gitlab/patch/action_cable_redis_listener.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -# Modifies https://github.com/rails/rails/blob/v6.1.4.6/actioncable/lib/action_cable/subscription_adapter/redis.rb -# so that it is resilient to Redis connection errors. -# See https://github.com/rails/rails/issues/27659. - -# rubocop:disable Gitlab/ModuleWithInstanceVariables -module Gitlab - module Patch - module ActionCableRedisListener - private - - def ensure_listener_running - @thread ||= Thread.new do - Thread.current.abort_on_exception = true - - conn = @adapter.redis_connection_for_subscriptions - listen conn - rescue ::Redis::BaseConnectionError - @thread = @raw_client = nil - ::ActionCable.server.restart - end - end - end - end -end diff --git a/lib/gitlab/patch/redis_cache_store.rb b/lib/gitlab/patch/redis_cache_store.rb index 5279c4081b2..041cb2d44bd 100644 --- a/lib/gitlab/patch/redis_cache_store.rb +++ b/lib/gitlab/patch/redis_cache_store.rb @@ -43,7 +43,13 @@ module Gitlab keys = names.map { |name| normalize_key(name, options) } values = failsafe(:patched_read_multi_mget, returning: {}) do - redis.with { |c| pipeline_mget(c, keys) } + redis.with do |c| + if c.is_a?(Gitlab::Redis::MultiStore) + c.with_readonly_pipeline { pipeline_mget(c, keys) } + else + pipeline_mget(c, keys) + end + end end names.zip(values).each_with_object({}) do |(name, value), results| diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 28d195238ea..8a604c7d8a6 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -21,6 +21,9 @@ module Gitlab }, ci: { keys: [:skip, :variable] + }, + integrations: { + keys: [:skip_ci] } }).freeze diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index dfbc00ef847..1a61b33fd9e 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -61,7 +61,7 @@ module Gitlab # Example: # # explanation do |arguments| - # "Adds label(s) #{arguments.join(' ')}" + # "Adds labels #{arguments.join(' ')}" # end # command :command_key do |arguments| # # Awesome code block @@ -76,7 +76,7 @@ module Gitlab # Example: # # execution_message do |arguments| - # "Added label(s) #{arguments.join(' ')}" + # "Added labels #{arguments.join(' ')}" # end # command :command_key do |arguments| # # Awesome code block diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 96e3112f32f..57ed6c5c35e 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -69,7 +69,7 @@ module Gitlab @updates[:title] = title_param end - desc { _('Add label(s)') } + desc { _('Add labels') } explanation do |labels_param| labels = find_label_references(labels_param) @@ -88,7 +88,7 @@ module Gitlab run_label_command(labels: find_labels(labels_param), command: :label, updates_key: :add_label_ids) end - desc { _('Remove all or specific label(s)') } + desc { _('Remove all or specific labels') } explanation do |labels_param = nil| label_references = labels_param.present? ? find_label_references(labels_param) : [] if label_references.any? @@ -125,7 +125,7 @@ module Gitlab @execution_message[:unlabel] = remove_label_message(label_references) end - desc { _('Replace all label(s)') } + desc { _('Replace all labels') } explanation do |labels_param| labels = find_label_references(labels_param) "Replaces all labels with #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index d7e9e1a980b..ae79db723f2 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -259,7 +259,6 @@ module Gitlab current_user.can?(:"set_#{quick_action_target.issue_type}_metadata", quick_action_target) end command :promote_to_incident do - @updates[:issue_type] = :incident @updates[:work_item_type] = ::WorkItems::Type.default_by_type(:incident) end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index e549ee2e43a..e01be4e0604 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -44,7 +44,7 @@ module Gitlab desc do if quick_action_target.allows_multiple_assignees? - _('Remove all or specific assignee(s)') + _('Remove all or specific assignees') else _('Remove assignee') end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index c374593bf01..9798b0eca2c 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -202,7 +202,7 @@ module Gitlab desc do if quick_action_target.allows_multiple_reviewers? - _('Assign reviewer(s)') + _('Assign reviewers') else _('Assign reviewer') end @@ -244,7 +244,7 @@ module Gitlab desc do if quick_action_target.allows_multiple_reviewers? - _('Remove all or specific reviewer(s)') + _('Remove all or specific reviewers') else _('Remove reviewer') end diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb index 5664410f3ca..a5c3c6a56be 100644 --- a/lib/gitlab/quick_actions/work_item_actions.rb +++ b/lib/gitlab/quick_actions/work_item_actions.rb @@ -98,7 +98,7 @@ module Gitlab def success_msg { type: _('Type changed successfully.'), - promote_to: _("Work Item promoted successfully.") + promote_to: _("Work item promoted successfully.") } end end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index d36ef6b99ee..7f4d611a490 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -6,15 +6,15 @@ module Gitlab include Gitlab::Utils::StrongMemoize class PipelinedDiffError < StandardError - def initialize(result_primary, result_secondary) - @result_primary = result_primary - @result_secondary = result_secondary + def initialize(non_default_store_result, default_store_result) + @non_default_store_result = non_default_store_result + @default_store_result = default_store_result end def message "Pipelined command executed on both stores successfully but results differ between them. " \ - "Result from the primary: #{@result_primary.inspect}. " \ - "Result from the secondary: #{@result_secondary.inspect}." + "Result from the non-default store: #{@non_default_store_result.inspect}. " \ + "Result from the default store: #{@default_store_result.inspect}." end end @@ -24,11 +24,17 @@ module Gitlab end end + class NestedReadonlyPipelineError < StandardError + def message + 'Nested use of with_readonly_pipeline is detected.' + end + end + attr_reader :primary_store, :secondary_store, :instance_name FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis default_store.' - FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' - FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.' + FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis non_default_store.' + FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis non_default_store.' SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze @@ -100,6 +106,25 @@ module Gitlab validate_stores! end + # Pipelines are sent to both instances by default since + # they could execute both read and write commands. + # + # But for pipelines that only consists of read commands, this method + # can be used to scope the pipeline and send it only to the default store. + def with_readonly_pipeline + raise NestedReadonlyPipelineError if readonly_pipeline? + + Thread.current[:readonly_pipeline] = true + + yield + ensure + Thread.current[:readonly_pipeline] = false + end + + def readonly_pipeline? + Thread.current[:readonly_pipeline].present? + end + # rubocop:disable GitlabSecurity/PublicSend READ_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| @@ -123,7 +148,7 @@ module Gitlab PIPELINED_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| - if use_primary_and_secondary_stores? + if use_primary_and_secondary_stores? && !readonly_pipeline? pipelined_both(name, *args, **kwargs, &block) else send_command(default_store, name, *args, **kwargs, &block) @@ -192,7 +217,7 @@ module Gitlab use_primary_store_as_default? ? primary_store : secondary_store end - def fallback_store + def non_default_store use_primary_store_as_default? ? secondary_store : primary_store end @@ -252,36 +277,39 @@ module Gitlab end def write_both(command_name, *args, **kwargs, &block) + result = send_command(default_store, command_name, *args, **kwargs, &block) + + # write to the non-default store only if write on default store is successful begin - send_command(primary_store, command_name, *args, **kwargs, &block) + send_command(non_default_store, command_name, *args, **kwargs, &block) rescue StandardError => e log_error(e, command_name, multi_store_error_message: FAILED_TO_WRITE_ERROR_MESSAGE) end - send_command(secondary_store, command_name, *args, **kwargs, &block) + result end # Run the entire pipeline on both stores. We assume that `&block` is idempotent. def pipelined_both(command_name, *args, **kwargs, &block) + result_default = send_command(default_store, command_name, *args, **kwargs, &block) + begin - result_primary = send_command(primary_store, command_name, *args, **kwargs, &block) + result_non_default = send_command(non_default_store, command_name, *args, **kwargs, &block) rescue StandardError => e log_error(e, command_name, multi_store_error_message: FAILED_TO_RUN_PIPELINE) end - result_secondary = send_command(secondary_store, command_name, *args, **kwargs, &block) - # Pipelined commands return an array with all results. If they differ, log an error - if result_primary && result_primary != result_secondary - error = PipelinedDiffError.new(result_primary, result_secondary) + if result_non_default && result_non_default != result_default + error = PipelinedDiffError.new(result_non_default, result_default) error.set_backtrace(Thread.current.backtrace[1..]) # Manually set backtrace, since the error is not `raise`d log_error(error, command_name) increment_pipelined_command_error_count(command_name) end - result_secondary + result_default end def same_redis_store? diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 26ca9d2547c..4e666dbaf77 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -78,6 +78,10 @@ module Gitlab @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z}o end + def npm_package_name_regex_message + 'should be a valid NPM package name: https://github.com/npm/validate-npm-package-name#naming-rules.' + end + def nuget_package_name_regex @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}.freeze end @@ -177,6 +181,10 @@ module Gitlab @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options).freeze end + def semver_regex_message + 'should follow SemVer: https://semver.org' + end + # These partial semver regexes are intended for use in composing other # regexes rather than being used alone. def _semver_major_minor_patch_regex diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index 79d6cfc84a3..c9051b6a5ff 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include BlobActiveModel - attr_reader :project, :content_match, :blob_path, :highlight_line, :matched_lines_count + attr_reader :project, :content_match, :blob_path, :highlight_line, :matched_lines_count, :group_level_blob, :group PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze @@ -31,14 +31,17 @@ module Gitlab @binary_data = opts.fetch(:data, nil) @per_page = opts.fetch(:per_page, 20) @project = opts.fetch(:project, nil) + @group = opts.fetch(:group, nil) # Some callers (e.g. Elasticsearch) do not have the Project object, # yet they can trigger many calls in one go, # causing duplicated queries. # Allow those to just pass project_id instead. @project_id = opts.fetch(:project_id, nil) + @group_id = opts.fetch(:group_id, nil) @content_match = opts.fetch(:content_match, nil) @blob_path = opts.fetch(:blob_path, nil) @repository = opts.fetch(:repository, nil) + @group_level_blob = opts.fetch(:group_level_blob, false) end def id diff --git a/lib/gitlab/search/found_wiki_page.rb b/lib/gitlab/search/found_wiki_page.rb index 99ca6a79fe2..650bae2af4d 100644 --- a/lib/gitlab/search/found_wiki_page.rb +++ b/lib/gitlab/search/found_wiki_page.rb @@ -14,7 +14,8 @@ module Gitlab # @param found_blob [Gitlab::Search::FoundBlob] def initialize(found_blob) super - @wiki = found_blob.project.wiki + + @wiki ||= found_blob.project.wiki end def to_ability_name @@ -23,3 +24,5 @@ module Gitlab end end end + +Gitlab::Search::FoundWikiPage.prepend_mod_with('Gitlab::Search::FoundWikiPage') diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index a733dca6a56..4fedc450f9b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -107,11 +107,7 @@ module Gitlab def users return User.none unless Ability.allowed?(current_user, :read_users_list) - if Feature.enabled?(:autocomplete_users_use_search_service) - UsersFinder.new(current_user, { search: query, use_minimum_char_limit: false }).execute - else - UsersFinder.new(current_user, search: query).execute - end + UsersFinder.new(current_user, { search: query, use_minimum_char_limit: false }).execute end # highlighting is only performed by Elasticsearch backed results diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index ec514adafc8..d5d9b794cd9 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -126,12 +126,12 @@ module Gitlab end def self.without_statement_timeout - Gitlab::Database::EachDatabase.each_database_connection do |connection| + Gitlab::Database::EachDatabase.each_connection do |connection| connection.execute('SET statement_timeout=0') end yield ensure - Gitlab::Database::EachDatabase.each_database_connection do |connection| + Gitlab::Database::EachDatabase.each_connection do |connection| connection.execute('RESET statement_timeout') end end diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb index c77db02061c..2cd9afc5bdc 100644 --- a/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb +++ b/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb @@ -169,7 +169,10 @@ module Gitlab } logger.info(message: 'Creating build', **build_attrs) - ::Ci::Build.new(importing: true, **build_attrs).tap(&:save!) + ::Ci::Build.transaction do + build = ::Ci::Build.new(importing: true, **build_attrs).tap(&:save!) + ::Ci::RunningBuild.upsert_shared_runner_build!(build) if build.running? && build.shared_runner_build? + end end def random_pipeline_status diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index c4566a6dc2a..56762c0fb4b 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -78,6 +78,8 @@ module Gitlab job_status = if job_exception 'fail' + elsif job['dropped'] + 'dropped' elsif job['deferred'] 'deferred' else @@ -87,12 +89,19 @@ module Gitlab payload['message'] = "#{message}: #{job_status}: #{payload['duration_s']} sec" payload['job_status'] = job_status payload['job_deferred_by'] = job['deferred_by'] if job['deferred'] + payload['deferred_count'] = job['deferred_count'] if job['deferred'] Gitlab::ExceptionLogFormatter.format!(job_exception, payload) if job_exception db_duration = ActiveRecord::LogSubscriber.runtime payload['db_duration_s'] = Gitlab::Utils.ms_to_round_sec(db_duration) + job_urgency = payload['class'].safe_constantize&.get_urgency.to_s + unless job_urgency.empty? + payload['urgency'] = job_urgency + payload['target_duration_s'] = Gitlab::Metrics::SidekiqSlis.execution_duration_for_urgency(job_urgency) + end + payload end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index ec2a6472809..614cd11421e 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -7,7 +7,7 @@ module Gitlab # The result of this method should be passed to # Sidekiq's `config.server_middleware` method # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` - def self.server_configurator(metrics: true, arguments_logger: true, defer_jobs: true) + def self.server_configurator(metrics: true, arguments_logger: true, skip_jobs: true) lambda do |chain| # Size limiter should be placed at the top chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server @@ -40,7 +40,7 @@ module Gitlab # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware - chain.add ::Gitlab::SidekiqMiddleware::DeferJobs if defer_jobs + chain.add ::Gitlab::SidekiqMiddleware::SkipJobs if skip_jobs end end diff --git a/lib/gitlab/sidekiq_middleware/defer_jobs.rb b/lib/gitlab/sidekiq_middleware/defer_jobs.rb deleted file mode 100644 index 0a12667865c..00000000000 --- a/lib/gitlab/sidekiq_middleware/defer_jobs.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqMiddleware - class DeferJobs - DELAY = ENV.fetch("SIDEKIQ_DEFER_JOBS_DELAY", 5.minutes) - FEATURE_FLAG_PREFIX = "defer_sidekiq_jobs" - - DatabaseHealthStatusChecker = Struct.new(:id, :job_class_name) - - # There are 2 scenarios under which this middleware defers a job - # 1. defer_sidekiq_jobs_#{worker_name} FF, jobs are deferred indefinitely until this feature flag - # is turned off or when Feature.enabled? returns false by chance while using `percentage of time` value. - # 2. Gitlab::Database::HealthStatus, on evaluating the db health status if it returns any indicator - # with stop signal, the jobs will be delayed by 'x' seconds (set in worker). - def call(worker, job, _queue) - # ActiveJobs have wrapped class stored in 'wrapped' key - resolved_class = job['wrapped']&.safe_constantize || worker.class - defer_job, delay, deferred_by = defer_job_info(resolved_class, job) - - if !!defer_job - # Referred in job_logger's 'log_job_done' method to compute proper 'job_status' - job['deferred'] = true - job['deferred_by'] = deferred_by - - worker.class.perform_in(delay, *job['args']) - counter.increment({ worker: worker.class.name }) - - # This breaks the middleware chain and return - return - end - - yield - end - - private - - def defer_job_info(worker_class, job) - if defer_job_by_ff?(worker_class) - [true, DELAY, :feature_flag] - elsif defer_job_by_database_health_signal?(job, worker_class) - [true, worker_class.database_health_check_attrs[:delay_by], :database_health_check] - end - end - - def defer_job_by_ff?(worker_class) - Feature.enabled?( - :"#{FEATURE_FLAG_PREFIX}_#{worker_class.name}", - type: :worker, - default_enabled_if_undefined: false - ) - end - - def defer_job_by_database_health_signal?(job, worker_class) - unless worker_class.respond_to?(:defer_on_database_health_signal?) && - worker_class.defer_on_database_health_signal? - return false - end - - health_check_attrs = worker_class.database_health_check_attrs - job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first - - health_context = Gitlab::Database::HealthStatus::Context.new( - DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), - job_base_model.connection, - health_check_attrs[:gitlab_schema], - health_check_attrs[:tables] - ) - - Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) - end - - def counter - @counter ||= Gitlab::Metrics.counter(:sidekiq_jobs_deferred_total, 'The number of jobs deferred') - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 3ed9c1743ed..46939d70c9e 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -97,13 +97,13 @@ module Gitlab local connection = ARGV[i] local current_offset = cookie.offsets[connection] local new_offset = tonumber(ARGV[i+1]) - if not current_offset or current_offset < new_offset then + if not current_offset or (new_offset and current_offset < new_offset) then cookie.offsets[connection] = new_offset cookie.wal_locations[connection] = ARGV[i+2] end end - redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1])) + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "keepttl") LUA def latest_wal_locations @@ -147,10 +147,7 @@ module Gitlab end local cookie = cmsgpack.unpack(cookie_msgpack) cookie.deduplicated = "1" - local ttl = redis.call("ttl", KEYS[1]) - if ttl > 0 then - redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", ttl) - end + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "keepttl") LUA def should_reschedule? diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index b3c3c94a0a3..058c23178f8 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -18,7 +18,7 @@ module Gitlab SIDEKIQ_QUEUE_DURATION_BUCKETS = [10, 60].freeze # These labels from Gitlab::SidekiqMiddleware::MetricsHelper are included in SLI metrics - SIDEKIQ_SLI_LABELS = [:worker, :feature_category, :urgency].freeze + SIDEKIQ_SLI_LABELS = [:worker, :feature_category, :urgency, :external_dependencies].freeze class << self include ::Gitlab::SidekiqMiddleware::MetricsHelper @@ -64,7 +64,8 @@ module Gitlab end end - Gitlab::Metrics::SidekiqSlis.initialize_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_execution_application_slis) + Gitlab::Metrics::SidekiqSlis.initialize_execution_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_execution_application_slis) + Gitlab::Metrics::SidekiqSlis.initialize_queueing_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_queueing_application_slis) end end @@ -147,6 +148,11 @@ module Gitlab Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) end + + if ::Feature.enabled?(:sidekiq_queueing_application_slis) + sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) + Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration + end end end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index acc3e1712ab..b19cc994d32 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -33,7 +33,8 @@ module Gitlab EXEMPT_WORKER_NAMES = %w[BackgroundMigrationWorker BackgroundMigration::CiDatabaseWorker Database::BatchedBackgroundMigrationWorker - Database::BatchedBackgroundMigration::CiDatabaseWorker].to_set + Database::BatchedBackgroundMigration::CiDatabaseWorker + RedisMigrationWorker].to_set JOB_STATUS_KEY = 'size_limiter' diff --git a/lib/gitlab/sidekiq_middleware/skip_jobs.rb b/lib/gitlab/sidekiq_middleware/skip_jobs.rb new file mode 100644 index 00000000000..8932607df52 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/skip_jobs.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class SkipJobs + DELAY = ENV.fetch("SIDEKIQ_DEFER_JOBS_DELAY", 5.minutes) + RUN_FEATURE_FLAG_PREFIX = "run_sidekiq_jobs" + DROP_FEATURE_FLAG_PREFIX = "drop_sidekiq_jobs" + + DatabaseHealthStatusChecker = Struct.new(:id, :job_class_name) + + COUNTER = :sidekiq_jobs_skipped_total + + def initialize + @metrics = init_metrics + end + + # This middleware decides whether a job is dropped, deferred or runs normally. + # In short: + # - `drop_sidekiq_jobs_#{worker_name}` FF enabled (disabled by default) --> drops the job + # - `run_sidekiq_jobs_#{worker_name}` FF disabled (enabled by default) --> defers the job + # + # DROPPING JOBS + # A job is dropped when `drop_sidekiq_jobs_#{worker_name}` FF is enabled. This FF is disabled by default for + # all workers. Dropped jobs are completely ignored and not requeued for future processing. + # + # DEFERRING JOBS + # Deferred jobs are rescheduled to perform in the future. + # There are 2 scenarios under which this middleware defers a job: + # 1. When run_sidekiq_jobs_#{worker_name} FF is disabled. This FF is enabled by default + # for all workers. + # 2. Gitlab::Database::HealthStatus, on evaluating the db health status if it returns any indicator + # with stop signal, the jobs will be delayed by 'x' seconds (set in worker). + # + # Dropping jobs takes higher priority over deferring jobs. For example, when `drop_sidekiq_jobs` is enabled and + # `run_sidekiq_jobs` is disabled, it results to jobs being dropped. + def call(worker, job, _queue) + # ActiveJobs have wrapped class stored in 'wrapped' key + resolved_class = job['wrapped']&.safe_constantize || worker.class + if drop_job?(resolved_class) + # no-op, drop the job entirely + drop_job!(job, worker) + return + elsif !!defer_job?(resolved_class, job) + defer_job!(job, worker) + return + end + + yield + end + + private + + def defer_job?(worker_class, job) + if !run_job_by_ff?(worker_class) + @delay = DELAY + @deferred_by = :feature_flag + true + elsif defer_job_by_database_health_signal?(job, worker_class) + @delay = worker_class.database_health_check_attrs[:delay_by] + @deferred_by = :database_health_check + true + end + end + + def run_job_by_ff?(worker_class) + # always returns true by default for all workers unless the FF is specifically disabled, e.g. during an incident + Feature.enabled?( + :"#{RUN_FEATURE_FLAG_PREFIX}_#{worker_class.name}", + type: :worker, + default_enabled_if_undefined: true + ) + end + + def defer_job_by_database_health_signal?(job, worker_class) + unless worker_class.respond_to?(:defer_on_database_health_signal?) && + worker_class.defer_on_database_health_signal? + return false + end + + health_check_attrs = worker_class.database_health_check_attrs + job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first + + health_context = Gitlab::Database::HealthStatus::Context.new( + DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), + job_base_model.connection, + health_check_attrs[:gitlab_schema], + health_check_attrs[:tables] + ) + + Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) + end + + def drop_job?(worker_class) + Feature.enabled?( + :"#{DROP_FEATURE_FLAG_PREFIX}_#{worker_class.name}", + type: :worker, + default_enabled_if_undefined: false + ) + end + + def drop_job!(job, worker) + job['dropped'] = true + @metrics.fetch(COUNTER).increment({ worker: worker.class.name, action: "dropped" }) + end + + def defer_job!(job, worker) + # Referred in job_logger's 'log_job_done' method to compute proper 'job_status' + job['deferred'] = true + job['deferred_by'] = @deferred_by + job['deferred_count'] ||= 0 + job['deferred_count'] += 1 + + worker.class.perform_in(@delay, *job['args']) + @metrics.fetch(COUNTER).increment({ worker: worker.class.name, action: "deferred" }) + end + + def init_metrics + { + COUNTER => Gitlab::Metrics.counter(COUNTER, 'The number of skipped jobs') + } + end + end + end +end diff --git a/lib/gitlab/signed_commit.rb b/lib/gitlab/signed_commit.rb index 410e71f51a1..be6592dd231 100644 --- a/lib/gitlab/signed_commit.rb +++ b/lib/gitlab/signed_commit.rb @@ -34,13 +34,19 @@ module Gitlab def signature_text strong_memoize(:signature_text) do - @signature_data.itself ? @signature_data[0] : nil + @signature_data.itself ? @signature_data[:signature] : nil end end def signed_text strong_memoize(:signed_text) do - @signature_data.itself ? @signature_data[1] : nil + @signature_data.itself ? @signature_data[:signed_text] : nil + end + end + + def signer + strong_memoize(:signer) do + @signature_data.itself ? @signature_data[:signer] : nil end end diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index c9c5c6da3bf..e098762f290 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -21,8 +21,8 @@ module Gitlab def deactivated ephemeral_response(text: <<~MESSAGE) - You are not allowed to perform the given chatops command since - your account has been deactivated by your administrator. + You are not allowed to perform the given ChatOps command. Most likely + your #{Gitlab.config.gitlab.url} account needs to be reactivated. Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url} MESSAGE diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 0afaf46fa9b..d13c3be0a09 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -58,7 +58,7 @@ module Gitlab pb.title = spammable.spam_title || '' if pb.respond_to?(:title) pb.description = spammable.spam_description || '' if pb.respond_to?(:description) pb.text = spammable.spammable_text || '' if pb.respond_to?(:text) - pb.type = spammable.spammable_entity_type if pb.respond_to?(:type) + pb.type = spammable.to_ability_name if pb.respond_to?(:type) pb.created_at = convert_to_pb_timestamp(spammable.created_at) if spammable.created_at pb.updated_at = convert_to_pb_timestamp(spammable.updated_at) if spammable.updated_at pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) diff --git a/lib/gitlab/ssh/commit.rb b/lib/gitlab/ssh/commit.rb index d9ac8c1b881..7d7cc529b1a 100644 --- a/lib/gitlab/ssh/commit.rb +++ b/lib/gitlab/ssh/commit.rb @@ -10,7 +10,7 @@ module Gitlab end def attributes - signature = ::Gitlab::Ssh::Signature.new(signature_text, signed_text, @commit.committer_email) + signature = ::Gitlab::Ssh::Signature.new(signature_text, signed_text, signer, @commit.committer_email) { commit_sha: @commit.sha, diff --git a/lib/gitlab/ssh/signature.rb b/lib/gitlab/ssh/signature.rb index 763d89116f1..6b0cab75557 100644 --- a/lib/gitlab/ssh/signature.rb +++ b/lib/gitlab/ssh/signature.rb @@ -11,15 +11,17 @@ module Gitlab GIT_NAMESPACE = 'git' - def initialize(signature_text, signed_text, committer_email) + def initialize(signature_text, signed_text, signer, committer_email) @signature_text = signature_text @signed_text = signed_text + @signer = signer @committer_email = committer_email end def verification_status strong_memoize(:verification_status) do next :unverified unless all_attributes_present? + next :verified_system if verified_by_gitlab? next :unverified unless valid_signature_blob? next :unknown_key unless signed_by_key next :other_user unless committer @@ -81,6 +83,15 @@ module Gitlab nil end end + + # If a commit is signed by Gitaly, the Gitaly returns `SIGNER_SYSTEM` as a signer + # In order to calculate it, the signature is Verified using the Gitaly's public key: + # https://gitlab.com/gitlab-org/gitaly/-/blob/v16.2.0-rc2/internal/gitaly/service/commit/commit_signatures.go#L63 + # + # It is safe to skip verification step if the commit has been signed by Gitaly + def verified_by_gitlab? + @signer == :SIGNER_SYSTEM + end end end end diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 1d9ecb624b2..bbcefabcb40 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -21,6 +21,14 @@ module Gitlab def self.renewal_service_email 'renewals-service@customers.gitlab.com' end + + def self.default_staging_customer_portal_url + 'https://customers.staging.gitlab.com' + end + + def self.default_production_customer_portal_url + 'https://customers.gitlab.com' + end end end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index b9800a4db73..f756d229ba1 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'rainbow/ext/string' -require_relative 'utils/strong_memoize' +require 'gitlab/utils/all' # rubocop:disable Rails/Output module Gitlab diff --git a/lib/gitlab/testing/action_cable_blocker.rb b/lib/gitlab/testing/action_cable_blocker.rb new file mode 100644 index 00000000000..aebb0732035 --- /dev/null +++ b/lib/gitlab/testing/action_cable_blocker.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# rubocop:disable Style/ClassVars + +# This is inspired by http://www.salsify.com/blog/engineering/tearing-capybara-ajax-tests +# Rack middleware that keeps track of the number of active requests and can block new requests. +module Gitlab + module Testing + class ActionCableBlocker + @@num_active_requests = Concurrent::AtomicFixnum.new(0) + @@block_requests = Concurrent::AtomicBoolean.new(false) + + # Returns the number of requests the server is currently processing. + def self.num_active_requests + @@num_active_requests.value + end + + # Prevents the server from accepting new requests. Any new requests will be skipped. + def self.block_requests! + @@block_requests.value = true + end + + # Allows the server to accept requests again. + def self.allow_requests! + @@block_requests.value = false + end + + def self.install + ::ActionCable::Server::Worker.set_callback :work, :around do |_, inner| + @@num_active_requests.increment + + inner.call if @@block_requests.false? + ensure + @@num_active_requests.decrement + end + end + end + end +end +# rubocop:enable Style/ClassVars diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 065ede75c60..bd42586731e 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -22,6 +22,10 @@ module Gitlab key_path end + def events + events_from_new_structure || events_from_old_structure || {} + end + def to_h attributes end @@ -44,7 +48,7 @@ module Gitlab def validate! unless skip_validation? - self.class.schemer.validate(attributes.stringify_keys).each do |error| + self.class.schemer.validate(attributes.deep_stringify_keys).each do |error| error_message = <<~ERROR_MSG Error type: #{error['type']} Data: #{error['data']} @@ -102,6 +106,19 @@ module Gitlab @metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml end + def metric_definitions_changed? + return false unless Rails.env.development? + + return false if @last_change_check && @last_change_check > 3.seconds.ago + + @last_change_check = Time.current + + last_change = Dir.glob(paths).map { |f| File.mtime(f) }.max + did_change = @last_metric_update != last_change + @last_metric_update = last_change + did_change + end + private def load_all! @@ -146,6 +163,20 @@ module Gitlab def skip_validation? !!attributes[:skip_validation] || @skip_validation || attributes[:status] == SKIP_VALIDATION_STATUS end + + def events_from_new_structure + events = attributes[:events] + return unless events + + events.to_h { |event| [event[:name], event[:unique].to_sym] } + end + + def events_from_old_structure + options_events = attributes.dig(:options, :events) + return unless options_events + + options_events.index_with { nil } + end end end end diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb index 4b38809dde4..0edd9f7914a 100644 --- a/lib/gitlab/usage/metrics/aggregates.rb +++ b/lib/gitlab/usage/metrics/aggregates.rb @@ -15,10 +15,14 @@ module Gitlab DATABASE_SOURCE = 'database' REDIS_SOURCE = 'redis_hll' + INTERNAL_EVENTS_SOURCE = 'internal_events' SOURCES = { DATABASE_SOURCE => Sources::PostgresHll, - REDIS_SOURCE => Sources::RedisHll + REDIS_SOURCE => Sources::RedisHll, + # Same strategy as RedisHLL, since they are a part of internal events + # and should get counted together with other RedisHLL-based aggregations + INTERNAL_EVENTS_SOURCE => Sources::RedisHll }.freeze end end diff --git a/lib/gitlab/usage/metrics/instrumentations/batched_background_migrations_metric.rb b/lib/gitlab/usage/metrics/instrumentations/batched_background_migrations_metric.rb new file mode 100644 index 00000000000..f5529b96678 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/batched_background_migrations_metric.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class BatchedBackgroundMigrationsMetric < DatabaseMetric + relation { Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:finished) } + + timestamp_column(:finished_at) + + operation :count + + def value + relation.map do |batched_migration| + { + job_class_name: batched_migration.job_class_name, + elapsed_time: batched_migration.finished_at.to_i - batched_migration.started_at.to_i + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_jira_dvcs_integration_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_jira_dvcs_integration_metric.rb new file mode 100644 index 00000000000..25a45a259e2 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_jira_dvcs_integration_metric.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithJiraDvcsIntegrationMetric < DatabaseMetric + operation :count + + def initialize(metric_definition) + super + + raise ArgumentError, "option 'cloud' must be a boolean" unless [true, false].include?(options[:cloud]) + end + + relation do |options| + ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: options[:cloud]) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_gbp_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_gbp_metric.rb new file mode 100644 index 00000000000..0a796c9fae9 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_gbp_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountSlackAppInstallationsGbpMetric < DatabaseMetric + operation :count + + relation { SlackIntegration.with_bot } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_metric.rb new file mode 100644 index 00000000000..af9cf957dab --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountSlackAppInstallationsMetric < DatabaseMetric + operation :count + + relation { SlackIntegration } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 7c646281598..d57dd7eac20 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -32,9 +32,9 @@ module Gitlab super(metric_definition.reverse_merge(time_frame: 'none')) end - def value + def value(...) alt_usage_data(fallback: self.class.fallback) do - self.class.metric_value.call + self.class.metric_value.call(...) end end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitaly_apdex_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitaly_apdex_metric.rb new file mode 100644 index 00000000000..ae1d076af19 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitaly_apdex_metric.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitalyApdexMetric < PrometheusMetric + value do |client| + result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first + + break FALLBACK unless result + + result['value'].last.to_f + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb index 409027925d1..2ce7e95ce77 100644 --- a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class IndexInconsistenciesMetric < GenericMetric value do - runner = Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database, validators: validators) + runner = Gitlab::Schema::Validation::Runner.new(structure_sql, database, validators: validators) inconsistencies = runner.execute @@ -23,19 +23,19 @@ module Gitlab def database database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] - Gitlab::Database::SchemaValidation::Database.new(database_model.connection) + Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) end def structure_sql stucture_sql_path = Rails.root.join('db/structure.sql') - Gitlab::Database::SchemaValidation::StructureSql.new(stucture_sql_path) + Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) end def validators [ - Gitlab::Database::SchemaValidation::Validators::MissingIndexes, - Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, - Gitlab::Database::SchemaValidation::Validators::ExtraIndexes + Gitlab::Schema::Validation::Validators::MissingIndexes, + Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes, + Gitlab::Schema::Validation::Validators::ExtraIndexes ] end end diff --git a/lib/gitlab/usage/metrics/instrumentations/ldap_encrypted_secrets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/ldap_encrypted_secrets_metric.rb new file mode 100644 index 00000000000..7667cff06e0 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/ldap_encrypted_secrets_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class LdapEncryptedSecretsMetric < GenericMetric + value do + Gitlab::Auth::Ldap::Config.encrypted_secrets.active? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/operating_system_metric.rb b/lib/gitlab/usage/metrics/instrumentations/operating_system_metric.rb new file mode 100644 index 00000000000..9bfe12d8ead --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/operating_system_metric.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class OperatingSystemMetric < GenericMetric + value do + ohai_data = Ohai::System.new.tap do |oh| + oh.all_plugins(['platform']) + end.data + + platform = ohai_data['platform'] + if ohai_data['platform'] == 'debian' && ohai_data['kernel']['machine']&.include?('armv') + platform = 'raspbian' + end + + "#{platform}-#{ohai_data['platform_version']}" + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb b/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb new file mode 100644 index 00000000000..ab1298b63c3 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class PrometheusMetric < GenericMetric + # Usage example + # + # class GitalyApdexMetric < PrometheusMetric + # value do + # result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first + # + # break FALLBACK unless result + # + # result['value'].last.to_f + # end + # end + def value + with_prometheus_client(verify: false, fallback: FALLBACK) do |client| + super(client) + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb new file mode 100644 index 00000000000..a481f7a5682 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class SchemaInconsistenciesMetric < GenericMetric + MAX_INCONSISTENCIES = 150 # Limit the number of inconsistencies reported to avoid large payloads + + value do + runner = Gitlab::Schema::Validation::Runner.new(structure_sql, database, validators: validators) + + inconsistencies = runner.execute + + inconsistencies.take(MAX_INCONSISTENCIES).map do |inconsistency| + { + object_name: inconsistency.object_name, + inconsistency_type: inconsistency.type, + object_type: inconsistency.object_type + } + end + end + + class << self + private + + def validators + Gitlab::Schema::Validation::Validators::Base.all_validators + end + + def database + database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] + Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) + end + + def structure_sql + stucture_sql_path = Rails.root.join('db/structure.sql') + Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/smtp_encrypted_secrets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/smtp_encrypted_secrets_metric.rb new file mode 100644 index 00000000000..1e1925f9933 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/smtp_encrypted_secrets_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class SmtpEncryptedSecretsMetric < GenericMetric + value do + Gitlab::Email::SmtpConfig.encrypted_secrets.active? + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 72168bce782..ab041a31bde 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -152,22 +152,6 @@ module Gitlab } end - def system_usage_data_settings - { - settings: { - ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, - smtp_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::SmtpConfig.encrypted_secrets.active? }, - operating_system: alt_usage_data(fallback: nil) { operating_system }, - gitaly_apdex: alt_usage_data { gitaly_apdex }, - collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none'), - service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none'), - snowplow_enabled: add_metric('SnowplowEnabledMetric', time_frame: 'none'), - snowplow_configured_to_gitlab_collector: add_metric('SnowplowConfiguredToGitlabCollectorMetric', time_frame: 'none'), - certificate_based_clusters_ff: add_metric('CertBasedClustersFfMetric') - } - } - end - def system_usage_data_weekly { counts_weekly: {} @@ -286,16 +270,9 @@ module Gitlab response[:"instances_#{name}_active"] = count(Integration.active.where(instance: true, type: type)) response[:"projects_inheriting_#{name}_active"] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: type)) response[:"groups_inheriting_#{name}_active"] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: type)) - end.merge(jira_usage, jira_import_usage) + end.merge(jira_import_usage) # rubocop: enable UsageData/LargeTable: end - - def jira_usage - { - projects_jira_dvcs_cloud_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled), - projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) - } - end # rubocop: enable CodeReuse/ActiveRecord def jira_import_usage @@ -328,17 +305,6 @@ module Gitlab } end - def operating_system - ohai_data = Ohai::System.new.tap do |oh| - oh.all_plugins(['platform']) - end.data - - platform = ohai_data['platform'] - platform = 'raspbian' if ohai_data['platform'] == 'debian' && ohai_data['kernel']['machine']&.include?('armv') - - "#{platform}-#{ohai_data['platform_version']}" - end - # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) { @@ -371,7 +337,11 @@ module Gitlab group_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.group_type, time_period), group_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.group_type, time_period), project_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.project_type, time_period), - project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period) + project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period), + # These two `projects_slack_x` metrics are owned by the Manage stage, but are in this method as their key paths can't change. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123442#note_1427961339. + projects_slack_notifications_active: distinct_count(::Project.with_slack_integration.where(time_period), :creator_id), + projects_slack_slash_active: distinct_count(::Project.with_slack_slash_commands_integration.where(time_period), :creator_id) } end # rubocop: enable UsageData/LargeTable @@ -527,7 +497,6 @@ module Gitlab def usage_data_metrics system_usage_data_license - .merge(system_usage_data_settings) .merge(system_usage_data) .merge(system_usage_data_monthly) .merge(system_usage_data_weekly) @@ -543,16 +512,6 @@ module Gitlab time_period.present? ? '28d' : 'none' end - def gitaly_apdex - with_prometheus_client(verify: false, fallback: FALLBACK) do |client| - result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first - - break FALLBACK unless result - - result['value'].last.to_f - end - end - def distinct_count_service_desk_enabled_projects(time_period) project_creator_id_start = minimum_id(User) project_creator_id_finish = maximum_id(User) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index eaa4bf15fe1..e71061c4522 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -15,14 +15,7 @@ module Gitlab # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # - # All events should be added to known_events yml files lib/gitlab/usage_data_counters/known_events/ - # - # Event example: - # - # - name: g_compliance_dashboard # Unique event name - # # Usage: - # # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event('g_compliance_dashboard', values: user_id) # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) class << self @@ -119,8 +112,18 @@ module Gitlab end def load_events(wildcard) - Dir[wildcard].each_with_object([]) do |path, events| - events.push(*load_yaml_from_path(path)) + if Feature.enabled?(:use_metric_definitions_for_events_list) + events = Gitlab::Usage::MetricDefinition.not_removed.values.map do |d| + d.attributes[:options] && d.attributes[:options][:events] + end.flatten.compact.uniq + + events.map do |e| + { name: e }.with_indifferent_access + end + else + Dir[wildcard].each_with_object([]) do |path, events| + events.push(*load_yaml_from_path(path)) + end end end @@ -129,7 +132,7 @@ module Gitlab end def known_events_names - known_events.map { |event| event[:name] } + @known_events_names ||= known_events.map { |event| event[:name] } end def event_for(event_name) diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 31f090e0f51..54464b63fce 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -38,8 +38,7 @@ module Gitlab class << self def track_issue_created_action(author:, namespace:) - track_snowplow_action(ISSUE_CREATED, author, namespace) - track_unique_action(ISSUE_CREATED, author) + track_internal_action(ISSUE_CREATED, author, namespace) end def track_issue_title_changed_action(author:, project:) @@ -180,14 +179,7 @@ module Gitlab private def track_snowplow_action(event_name, author, container) - namespace, project = case container - when Project - [container.namespace, container] - when Namespaces::ProjectNamespace - [container.parent, container.project] - else - [container, nil] - end + namespace, project = get_params_from_container(container) return unless author @@ -208,6 +200,30 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id) end + + def track_internal_action(event_name, author, container) + return unless author + + namespace, project = get_params_from_container(container) + + Gitlab::InternalEvents.track_event( + event_name, + user: author, + project: project, + namespace: namespace + ) + end + + def get_params_from_container(container) + case container + when Project + [container.namespace, container] + when Namespaces::ProjectNamespace + [container.parent, container.project] + else + [container, nil] + end + end end end end diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml index b3d1c51c0e7..fe779a9a25f 100644 --- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml +++ b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml @@ -1,2 +1,22 @@ - name: agent_users_using_ci_tunnel aggregation: weekly +- name: k8s_api_proxy_requests_unique_users_via_ci_access + aggregation: weekly +- name: k8s_api_proxy_requests_unique_users_via_ci_access + aggregation: monthly +- name: k8s_api_proxy_requests_unique_agents_via_ci_access + aggregation: weekly +- name: k8s_api_proxy_requests_unique_agents_via_ci_access + aggregation: monthly +- name: k8s_api_proxy_requests_unique_users_via_user_access + aggregation: weekly +- name: k8s_api_proxy_requests_unique_users_via_user_access + aggregation: monthly +- name: k8s_api_proxy_requests_unique_agents_via_user_access + aggregation: weekly +- name: k8s_api_proxy_requests_unique_agents_via_user_access + aggregation: monthly +- name: flux_git_push_notified_unique_projects + aggregation: weekly +- name: flux_git_push_notified_unique_projects + aggregation: monthly diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index ece2ffea83b..9e8c207a19a 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -4,7 +4,13 @@ module Gitlab module UsageDataCounters class KubernetesAgentCounter < BaseCounter PREFIX = 'kubernetes_agent' - KNOWN_EVENTS = %w[gitops_sync k8s_api_proxy_request flux_git_push_notifications_total].freeze + KNOWN_EVENTS = %w[ + gitops_sync + k8s_api_proxy_request + flux_git_push_notifications_total + k8s_api_proxy_requests_via_ci_access + k8s_api_proxy_requests_via_user_access + ].freeze class << self def increment_event_counts(events) diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 1ed2e891a1f..d26b7ce951d 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -70,9 +70,9 @@ module Gitlab Gitlab::InternalEvents.track_event( MR_USER_CREATE_ACTION, - user_id: user.id, - project_id: project.id, - namespace_id: project.namespace_id + user: user, + project: project, + namespace: project.namespace ) end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb deleted file mode 100644 index dc0112c14d6..00000000000 --- a/lib/gitlab/utils.rb +++ /dev/null @@ -1,259 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Utils - extend self - DoubleEncodingError ||= Class.new(StandardError) - - def allowlisted?(absolute_path, allowlist) - path = absolute_path.downcase - - allowlist.map(&:downcase).any? do |allowed_path| - path.start_with?(allowed_path) - end - end - - def decode_path(encoded_path) - decoded = CGI.unescape(encoded_path) - if decoded != CGI.unescape(decoded) - raise DoubleEncodingError, "path #{encoded_path} is not allowed" - end - - decoded - end - - def force_utf8(str) - str.dup.force_encoding(Encoding::UTF_8) - end - - def ensure_utf8_size(str, bytes:) - raise ArgumentError, 'Empty string provided!' if str.empty? - raise ArgumentError, 'Negative string size provided!' if bytes < 0 - - truncated = str.each_char.each_with_object(+'') do |char, object| - if object.bytesize + char.bytesize > bytes - break object - else - object.concat(char) - end - end - - truncated + ('0' * (bytes - truncated.bytesize)) - end - - # Append path to host, making sure there's one single / in between - def append_path(host, path) - "#{host.to_s.sub(%r{\/+$}, '')}/#{remove_leading_slashes(path)}" - end - - def remove_leading_slashes(str) - str.to_s.sub(%r{^/+}, '') - end - - # A slugified version of the string, suitable for inclusion in URLs and - # domain names. Rules: - # - # * Lowercased - # * Anything not matching [a-z0-9-] is replaced with a - - # * Maximum length is 63 bytes - # * First/Last Character is not a hyphen - def slugify(str) - str.downcase - .gsub(/[^a-z0-9]/, '-')[0..62] - .gsub(/(\A-+|-+\z)/, '') - end - - # Converts newlines into HTML line break elements - def nlbr(str) - ActionView::Base.full_sanitizer.sanitize(+str, tags: []).gsub(/\r?\n/, '<br>').html_safe - end - - def remove_line_breaks(str) - str.gsub(/\r?\n/, '') - end - - def to_boolean(value, default: nil) - value = value.to_s if [0, 1].include?(value) - - return value if [true, false].include?(value) - return true if value =~ /^(true|t|yes|y|1|on)$/i - return false if value =~ /^(false|f|no|n|0|off)$/i - - default - end - - def boolean_to_yes_no(bool) - if bool - 'Yes' - else - 'No' - end - end - - # Behaves like `which` on Linux machines: given PATH, try to resolve the given - # executable name to an absolute path, or return nil. - # - # which('ruby') #=> /usr/bin/ruby - def which(filename) - ENV['PATH']&.split(File::PATH_SEPARATOR)&.each do |path| - full_path = File.join(path, filename) - return full_path if File.executable?(full_path) - end - - nil - end - - def try_megabytes_to_bytes(size) - Integer(size).megabytes - rescue ArgumentError - size - end - - def bytes_to_megabytes(bytes) - bytes.to_f / Numeric::MEGABYTE - end - - def ms_to_round_sec(ms) - (ms.to_f / 1000).round(6) - end - - # Used in EE - # Accepts either an Array or a String and returns an array - def ensure_array_from_string(string_or_array) - return string_or_array if string_or_array.is_a?(Array) - - string_or_array.split(',').map(&:strip) - end - - def deep_indifferent_access(data) - case data - when Array - data.map(&method(:deep_indifferent_access)) - when Hash - data.with_indifferent_access - else - data - end - end - - def deep_symbolized_access(data) - case data - when Array - data.map(&method(:deep_symbolized_access)) - when Hash - data.deep_symbolize_keys - else - data - end - end - - def string_to_ip_object(str) - return unless str - - IPAddr.new(str) - rescue IPAddr::InvalidAddressError - end - - # A safe alternative to String#downcase! - # - # This will make copies of frozen strings but downcase unfrozen - # strings in place, reducing allocations. - def safe_downcase!(str) - if str.frozen? - str.downcase - else - str.downcase! || str - end - end - - # Converts a string to an Addressable::URI object. - # If the string is not a valid URI, it returns nil. - # Param uri_string should be a String object. - # This method returns an Addressable::URI object or nil. - def parse_url(uri_string) - Addressable::URI.parse(uri_string) - rescue Addressable::URI::InvalidURIError, TypeError - end - - def add_url_parameters(url, params) - uri = parse_url(url.to_s) - uri.query_values = uri.query_values.to_h.merge(params.to_h.stringify_keys) - uri.query_values = nil if uri.query_values.empty? - uri.to_s - end - - def removes_sensitive_data_from_url(uri_string) - uri = parse_url(uri_string) - - return unless uri - return uri_string unless uri.fragment - - stripped_params = CGI.parse(uri.fragment) - if stripped_params['access_token'] - stripped_params['access_token'] = 'filtered' - filtered_query = Addressable::URI.new - filtered_query.query_values = stripped_params - - uri.fragment = filtered_query.query - end - - uri.to_s - end - - # Invert a hash, collecting all keys that map to a given value in an array. - # - # Unlike `Hash#invert`, where the last encountered pair wins, and which has the - # type `Hash[k, v] => Hash[v, k]`, `multiple_key_invert` does not lose any - # information, has the type `Hash[k, v] => Hash[v, Array[k]]`, and the original - # hash can always be reconstructed. - # - # example: - # - # multiple_key_invert({ a: 1, b: 2, c: 1 }) - # # => { 1 => [:a, :c], 2 => [:b] } - # - def multiple_key_invert(hash) - hash.flat_map { |k, v| Array.wrap(v).zip([k].cycle) } - .group_by(&:first) - .transform_values { |kvs| kvs.map(&:last) } - end - - # This sort is stable (see https://en.wikipedia.org/wiki/Sorting_algorithm#Stability) - # contrary to the bare Ruby sort_by method. Using just sort_by leads to - # instability across different platforms (e.g., x86_64-linux and x86_64-darwin18) - # which in turn leads to different sorting results for the equal elements across - # these platforms. - # This method uses a list item's original index position to break ties. - def stable_sort_by(list) - list.sort_by.with_index { |x, idx| [yield(x), idx] } - end - - # Check for valid brackets (`[` and `]`) in a string using this aspects: - # * open brackets count == closed brackets count - # * (optionally) reject nested brackets via `allow_nested: false` - # * open / close brackets coherence, eg. ][[] -> invalid - def valid_brackets?(string = '', allow_nested: true) - # remove everything except brackets - brackets = string.remove(/[^\[\]]/) - - return true if brackets.empty? - # balanced counts check - return false if brackets.size.odd? - - unless allow_nested - # nested brackets check - return false if brackets.include?('[[') || brackets.include?(']]') - end - - # open / close brackets coherence check - untrimmed = brackets - loop do - trimmed = untrimmed.gsub('[]', '') - return true if trimmed.empty? - return false if trimmed == untrimmed - - untrimmed = trimmed - end - end - end -end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 1d02bcbb2d2..10370811bb5 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative '../utils' +require 'gitlab/utils/all' require_relative '../environment' module Gitlab diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb deleted file mode 100644 index 2b3841b8f09..00000000000 --- a/lib/gitlab/utils/strong_memoize.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Utils - module StrongMemoize - # Instead of writing patterns like this: - # - # def trigger_from_token - # return @trigger if defined?(@trigger) - # - # @trigger = Ci::Trigger.find_by_token(params[:token].to_s) - # end - # - # We could write it like: - # - # include Gitlab::Utils::StrongMemoize - # - # def trigger_from_token - # Ci::Trigger.find_by_token(params[:token].to_s) - # end - # strong_memoize_attr :trigger_from_token - # - # def enabled? - # Feature.enabled?(:some_feature) - # end - # strong_memoize_attr :enabled? - # - def strong_memoize(name) - key = ivar(name) - - if instance_variable_defined?(key) - instance_variable_get(key) - else - instance_variable_set(key, yield) - end - end - - # Works the same way as "strong_memoize" but takes - # a second argument - expire_in. This allows invalidate - # the data after specified number of seconds - def strong_memoize_with_expiration(name, expire_in) - key = ivar(name) - expiration_key = "#{key}_expired_at" - - if instance_variable_defined?(expiration_key) - expire_at = instance_variable_get(expiration_key) - clear_memoization(name) if Time.current > expire_at - end - - if instance_variable_defined?(key) - instance_variable_get(key) - else - value = instance_variable_set(key, yield) - instance_variable_set(expiration_key, Time.current + expire_in) - value - end - end - - def strong_memoize_with(name, *args) - container = strong_memoize(name) { {} } - - if container.key?(args) - container[args] - else - container[args] = yield - end - end - - def strong_memoized?(name) - key = ivar(StrongMemoize.normalize_key(name)) - instance_variable_defined?(key) - end - - def clear_memoization(name) - key = ivar(StrongMemoize.normalize_key(name)) - remove_instance_variable(key) if instance_variable_defined?(key) - end - - module StrongMemoizeClassMethods - def strong_memoize_attr(method_name) - member_name = StrongMemoize.normalize_key(method_name) - - StrongMemoize.send(:do_strong_memoize, self, method_name, member_name) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def self.included(base) - base.singleton_class.prepend(StrongMemoizeClassMethods) - end - - private - - # Convert `"name"`/`:name` into `:@name` - # - # Depending on a type ensure that there's a single memory allocation - def ivar(name) - case name - when Symbol - name.to_s.prepend("@").to_sym - when String - :"@#{name}" - else - raise ArgumentError, "Invalid type of '#{name}'" - end - end - - class << self - def normalize_key(key) - return key unless key.end_with?('!', '?') - - # Replace invalid chars like `!` and `?` with allowed Unicode codeparts. - key.to_s.tr('!?', "\uFF01\uFF1F") - end - - private - - def do_strong_memoize(klass, method_name, member_name) - method = klass.instance_method(method_name) - - unless method.arity == 0 - raise <<~ERROR - Using `strong_memoize_attr` on methods with parameters is not supported. - - Use `strong_memoize_with` instead. - See https://docs.gitlab.com/ee/development/utilities.html#strongmemoize - ERROR - end - - # Methods defined within a class method are already public by default, so we don't need to - # explicitly make them public. - scope = %i[private protected].find do |scope| - klass.send("#{scope}_instance_methods") # rubocop:disable GitlabSecurity/PublicSend - .include? method_name - end - - klass.define_method(method_name) do |&block| - strong_memoize(member_name) do - method.bind_call(self, &block) - end - end - - klass.send(scope, method_name) if scope # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end -end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 4106084b301..1e482901929 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -240,7 +240,7 @@ module Gitlab yield.merge(key => Time.current) end - # @param event_name [String] the event name + # @param event_name [String, Symbol] the event name # @param values [Array|String] the values counted def track_usage_event(event_name, values) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb index 016c25eb94b..a3abe90a412 100644 --- a/lib/gitlab/uuid.rb +++ b/lib/gitlab/uuid.rb @@ -10,8 +10,6 @@ module Gitlab }.freeze UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/.freeze - NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN" class << self def v5(name, namespace_id: default_namespace_id) @@ -25,12 +23,7 @@ module Gitlab private def default_namespace_id - @default_namespace_id ||= begin - namespace_uuid = NAMESPACE_IDS.fetch(Rails.env.to_sym) - # Digest::UUID is broken when using a UUID as a namespace_id - # https://github.com/rails/rails/issues/37681#issue-520718028 - namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN) - end + NAMESPACE_IDS.fetch(Rails.env.to_sym) end end end diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb deleted file mode 100644 index 0351c9b30b3..00000000000 --- a/lib/gitlab/version_info.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class VersionInfo - include Comparable - - attr_reader :major, :minor, :patch - - VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze - # To mitigate ReDoS, limit the length of the version string we're - # willing to check - MAX_VERSION_LENGTH = 128 - - def self.parse(str, parse_suffix: false) - if str.is_a?(self) - str - elsif str && str.length <= MAX_VERSION_LENGTH && m = str.match(VERSION_REGEX) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil) - else - VersionInfo.new - end - end - - def initialize(major = 0, minor = 0, patch = 0, suffix = nil) - @major = major - @minor = minor - @patch = patch - @suffix_s = suffix.to_s - end - - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def <=>(other) - return unless other.is_a? VersionInfo - return unless valid? && other.valid? - - if other.major < @major - 1 - elsif @major < other.major - -1 - elsif other.minor < @minor - 1 - elsif @minor < other.minor - -1 - elsif other.patch < @patch - 1 - elsif @patch < other.patch - -1 - elsif @suffix_s.empty? && other.suffix.present? - 1 - elsif other.suffix.empty? && @suffix_s.present? - -1 - else - suffix <=> other.suffix - end - end - # rubocop:enable Metrics/CyclomaticComplexity - # rubocop:enable Metrics/PerceivedComplexity - - def to_s - if valid? - "%d.%d.%d%s" % [@major, @minor, @patch, @suffix_s] - else - 'Unknown' - end - end - - def to_json(*_args) - { major: @major, minor: @minor, patch: @patch }.to_json - end - - def suffix - @suffix ||= @suffix_s.strip.gsub('-', '.pre.').scan(/\d+|[a-z]+/i).map do |s| - /^\d+$/ =~ s ? s.to_i : s - end.freeze - end - - def valid? - @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 - end - - def hash - [self.class, to_s].hash - end - - def eql?(other) - (self <=> other) == 0 - end - - def same_minor_version?(other) - @major == other.major && @minor == other.minor - end - - def without_patch - self.class.new(@major, @minor, 0) - end - end -end diff --git a/lib/gitlab/web_hooks.rb b/lib/gitlab/web_hooks.rb index 8c6de56292a..031f69f3679 100644 --- a/lib/gitlab/web_hooks.rb +++ b/lib/gitlab/web_hooks.rb @@ -4,5 +4,6 @@ module Gitlab module WebHooks GITLAB_EVENT_HEADER = 'X-Gitlab-Event' GITLAB_INSTANCE_HEADER = 'X-Gitlab-Instance' + GITLAB_UUID_HEADER = 'X-Gitlab-Webhook-UUID' end end diff --git a/lib/gitlab_settings/options.rb b/lib/gitlab_settings/options.rb index 077c1aa944a..68555794436 100644 --- a/lib/gitlab_settings/options.rb +++ b/lib/gitlab_settings/options.rb @@ -1,7 +1,42 @@ # frozen_string_literal: true +require 'forwardable' + module GitlabSettings class Options + extend Forwardable + + def_delegators :@options, + :count, + :deep_stringify_keys, + :deep_symbolize_keys, + :default_proc, + :dig, + :each_key, + :each_pair, + :each_value, + :each, + :empty?, + :fetch_values, + :fetch, + :filter, + :keys, + :length, + :map, + :member?, + :merge, + :reject, + :select, + :size, + :slice, + :stringify_keys, + :symbolize_keys, + :transform_keys, + :transform_values, + :value?, + :values_at, + :values + # Recursively build GitlabSettings::Options def self.build(obj) case obj @@ -26,22 +61,25 @@ module GitlabSettings @options[key.to_s] = self.class.build(value) end - def key?(name) - @options.key?(name.to_s) || @options.key?(name.to_sym) + def key?(key) + @options.key?(key.to_s) end alias_method :has_key?, :key? - def to_hash - @options.deep_transform_values do |option| - case option - when self.class - option.to_hash - else - option - end - end + # Some configurations use the 'default' key, like: + # https://gitlab.com/gitlab-org/gitlab/-/blob/c4d5c77c87494bb320fa7fdf19b0e4d7d52af1d1/spec/support/helpers/stub_configuration.rb#L96 + # But since `default` is also a method in Hash, this can be confusing and + # raise an exception instead of returning nil, as expected in some places. + # To avoid that, we use #default always as a possible internal key + def default + @options['default'] + end + + # For backward compatibility, like: + # https://gitlab.com/gitlab-org/gitlab/-/blob/adf67e90428670aaa955731f3bdeafb8b3a874cd/lib/gitlab/database/health_status/indicators/patroni_apdex.rb#L58 + def with_indifferent_access + to_hash.with_indifferent_access end - alias_method :to_h, :to_hash def dup self.class.build(to_hash) @@ -51,16 +89,56 @@ module GitlabSettings self.class.build(to_hash.merge(other.deep_stringify_keys)) end + def merge!(other) + @options = to_hash.merge(other.deep_stringify_keys) + end + def deep_merge(other) self.class.build(to_hash.deep_merge(other.deep_stringify_keys)) end + def deep_merge!(other) + @options = to_hash.deep_merge(other.deep_stringify_keys) + end + def is_a?(klass) return true if klass == Hash super(klass) end + def to_hash + @options.deep_transform_values do |option| + case option + when self.class + option.to_hash + else + option + end + end + end + alias_method :to_h, :to_hash + + # Don't alter the internal keys + def stringify_keys! + error_msg = "Warning: Do not mutate #{self.class} objects: `#{__method__}`" + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(RuntimeError.new(error_msg), method: __method__) + + to_hash.deep_stringify_keys + end + alias_method :deep_stringify_keys!, :stringify_keys! + + # Don't alter the internal keys + def symbolize_keys! + error_msg = "Warning: Do not mutate #{self.class} objects: `#{__method__}`" + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(RuntimeError.new(error_msg), method: __method__) + + to_hash.deep_symbolize_keys + end + alias_method :deep_symbolize_keys!, :symbolize_keys! + def method_missing(name, *args, &block) name_string = +name.to_s @@ -70,7 +148,13 @@ module GitlabSettings return self[name_string] end - return @options.public_send(name, *args, &block) if @options.respond_to?(name) # rubocop: disable GitlabSecurity/PublicSend + if @options.respond_to?(name) + error_msg = "Calling a hash method on #{self.class}: `#{name}`" + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(RuntimeError.new(error_msg), method: name) + + return @options.public_send(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end raise ::GitlabSettings::MissingSetting, "option '#{name}' not defined" end diff --git a/lib/result.rb b/lib/result.rb new file mode 100644 index 00000000000..5e72b3f13cb --- /dev/null +++ b/lib/result.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +# A (partial) implementation of the functional Result type, with naming conventions based on the +# Rust implementation (https://doc.rust-lang.org/std/result/index.html) +# +# Modern Ruby 3+ destructuring and pattern matching are supported. +# +# - See "Railway Oriented Programming and the Result Class" in `ee/lib/remote_development/README.md` for details +# and example usage. +# - See `spec/lib/result_spec.rb` for detailed executable example usage. +# - See https://en.wikipedia.org/wiki/Result_type for a general description of the Result pattern. +# - See https://fsharpforfunandprofit.com/rop/ for how this can be used with Railway Oriented Programming (ROP) +# to improve design and architecture +# - See https://doc.rust-lang.org/std/result/ for the Rust implementation. + +# NOTE: This class is intentionally not namespaced to allow for more concise, readable, and explicit usage. +# It it a generic reusable implementation of the Result type, and is not specific to any domain +# rubocop:disable Gitlab/NamespacedClass +class Result + # The .ok and .err factory class methods are the only way to create a Result + # + # "self.ok" corresponds to Ok(T) in Rust: https://doc.rust-lang.org/std/result/enum.Result.html#variant.Ok + # + # @param [Object, #new] ok_value + # @return [Result] + def self.ok(ok_value) + new(ok_value: ok_value) + end + + # "self.err" corresponds to Err(E) in Rust: https://doc.rust-lang.org/std/result/enum.Result.html#variant.Err + # + # @param [Object, #new] ok_value + # @return [Result] + def self.err(err_value) + new(err_value: err_value) + end + + # "#unwrap" corresponds to "unwrap" in Rust. + # + # @return [Object] + # @raise [RuntimeError] if called on an "err" Result + def unwrap + ok? ? value : raise("Called Result#unwrap on an 'err' Result") + end + + # "#unwrap" corresponds to "unwrap" in Rust. + # + # @return [Object] + # @raise [RuntimeError] if called on an "ok" Result + def unwrap_err + err? ? value : raise("Called Result#unwrap_err on an 'ok' Result") + end + + # The `ok?` attribute will be true if the Result was constructed with .ok, and false if it was constructed with .err + # + # "#ok?" corresponds to "is_ok" in Rust. + # @return [Boolean] + def ok? + # We don't make `@ok` an attr_reader, because we don't want to confusingly shadow the class method `.ok` + @ok + end + + # The `err?` attribute will be false if the Result was constructed with .ok, and true if it was constructed with .err + # "#err?" corresponds to "is_err" in Rust. + # + # @return [Boolean] + def err? + !ok? + end + + # `and_then` is a functional way to chain together operations which may succeed or have errors. It is passed + # a lambda or class (singleton) method object, and must return a Result object representing "ok" + # or "err". + # + # If the Result object it is called on is "ok", then the passed lambda or singleton method + # is called with the value contained in the Result. + # + # If the Result object it is called on is "err", then it is returned without calling the passed + # lambda or method. + # + # It only supports being passed a lambda, or a class (singleton) method object + # which responds to `call` with a single argument (arity of 1). If multiple values are needed, + # pass a hash or array. Note that passing `Proc` objects is NOT supported, even though the YARD + # annotation contains `Proc` (because the type of a lambda is also `Proc`). + # + # Passing instance methods to `and_then` is not supported, because the methods in the chain should be + # stateless "pure functions", and should not be persisting or referencing any instance state anyway. + # + # "#and_then" corresponds to "and_then" in Rust: https://doc.rust-lang.org/std/result/enum.Result.html#method.and_then + # + # @param [Proc, Method] lambda_or_singleton_method + # @return [Result] + # @raise [TypeError] + def and_then(lambda_or_singleton_method) + validate_lambda_or_singleton_method(lambda_or_singleton_method) + + # Return/passthough the Result itself if it is an err + return self if err? + + # If the Result is ok, call the lambda or singleton method with the contained value + result = lambda_or_singleton_method.call(value) + + unless result.is_a?(Result) + err_msg = "'Result##{__method__}' expects a lambda or singleton method object which returns a 'Result' type " \ + ", but instead received '#{lambda_or_singleton_method.inspect}' which returned '#{result.class}'. " \ + "Check that the previous method calls in the '#and_then' chain are correct." + raise(TypeError, err_msg) + end + + result + end + + # `map` is similar to `and_then`, but it is used for "single track" methods which always succeed, + # and have no possibility of returning an error (but they may still raise exceptions, + # which is unrelated to the Result handling). The passed lambda or singleton method must return + # a value, not a Result. + # + # If the Result object it is called on is "ok", then the passed lambda or singleton method + # is called with the value contained in the Result. + # + # If the Result object it is called on is "err", then it is returned without calling the passed + # lambda or method. + # + # "#map" corresponds to "map" in Rust: https://doc.rust-lang.org/std/result/enum.Result.html#method.map + # + # @param [Proc, Method] lambda_or_singleton_method + # @return [Result] + # @raise [TypeError] + def map(lambda_or_singleton_method) + validate_lambda_or_singleton_method(lambda_or_singleton_method) + + # Return/passthrough the Result itself if it is an err + return self if err? + + # If the Result is ok, call the lambda or singleton method with the contained value + mapped_value = lambda_or_singleton_method.call(value) + + if mapped_value.is_a?(Result) + err_msg = "'Result##{__method__}' expects a lambda or singleton method object which returns an unwrapped " \ + "value, not a 'Result', but instead received '#{lambda_or_singleton_method.inspect}' which returned " \ + "a 'Result'." + raise(TypeError, err_msg) + end + + # wrap the returned mapped_value in an "ok" Result. + Result.ok(mapped_value) + end + + # `to_h` supports destructuring of a result object, for example: `result => { ok: }; puts ok` + # + # @return [Hash] + def to_h + ok? ? { ok: value } : { err: value } + end + + # `deconstruct_keys` supports pattern matching on a Result object with a `case` statement. See specs for examples. + # + # @param [Array] keys + # @return [Hash] + # @raise [ArgumentError] + def deconstruct_keys(keys) + raise(ArgumentError, 'Use either :ok or :err for pattern matching') unless [[:ok], [:err]].include?(keys) + + to_h + end + + # @return [Boolean] + def ==(other) + # NOTE: The underlying `@ok` instance variable is a boolean, so we only need to check `ok?`, not `err?` too + self.class == other.class && other.ok? == ok? && other.instance_variable_get(:@value) == value + end + + private + + # The `value` attribute will contain either the ok_value or the err_value + attr_reader :value + + def initialize(ok_value: nil, err_value: nil) + if (!ok_value.nil? && !err_value.nil?) || (ok_value.nil? && err_value.nil?) + raise(ArgumentError, 'Do not directly use private constructor, use Result.ok or Result.err') + end + + @ok = err_value.nil? + @value = ok? ? ok_value : err_value + end + + # @param [Proc, Method] lambda_or_singleton_method + # @return [void] + # @raise [TypeError] + def validate_lambda_or_singleton_method(lambda_or_singleton_method) + is_lambda = lambda_or_singleton_method.is_a?(Proc) && lambda_or_singleton_method.lambda? + is_singleton_method = lambda_or_singleton_method.is_a?(Method) && lambda_or_singleton_method.owner.singleton_class? + unless is_lambda || is_singleton_method + err_msg = "'Result##{__method__}' expects a lambda or singleton method object, " \ + "but instead received '#{lambda_or_singleton_method.inspect}'." + raise(TypeError, err_msg) + end + + arity = lambda_or_singleton_method.arity + + return if arity == 1 + + err_msg = "'Result##{__method__}' expects a lambda or singleton method object with a single argument " \ + "(arity of 1), but instead received '#{lambda_or_singleton_method.inspect}' with an arity of #{arity}." + raise(ArgumentError, err_msg) + end +end + +# rubocop:enable Gitlab/NamespacedClass diff --git a/lib/search/navigation.rb b/lib/search/navigation.rb new file mode 100644 index 00000000000..3594ac0dc30 --- /dev/null +++ b/lib/search/navigation.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +module Search + class Navigation + include Gitlab::Allowable + + def initialize(user:, project: nil, group: nil, options: {}) + @user = user + @project = project + @group = group + @options = options + end + + def tab_enabled_for_project?(tab) + return false unless project.present? + + abilities = Array(search_tab_ability_map[tab]) + Array.wrap(project).any? { |p| abilities.any? { |ability| can?(user, ability, p) } } + end + + def tabs + { + projects: { + sort: 1, + label: _("Projects"), + data: { qa_selector: 'projects_tab' }, + condition: project.nil? + }, + blobs: { + sort: 2, + label: _("Code"), + data: { qa_selector: 'code_tab' }, + condition: show_code_search_tab? + }, + # sort: 3 is reserved for EE items + issues: { + sort: 4, + label: _("Issues"), + condition: show_issues_search_tab? + }, + merge_requests: { + sort: 5, + label: _("Merge requests"), + condition: show_merge_requests_search_tab? + }, + wiki_blobs: { + sort: 6, + label: _("Wiki"), + condition: show_wiki_search_tab? + }, + commits: { + sort: 7, + label: _("Commits"), + condition: show_commits_search_tab? + }, + notes: { + sort: 8, + label: _("Comments"), + condition: show_comments_search_tab? + }, + milestones: { + sort: 9, label: _("Milestones"), + condition: show_milestones_search_tab? + }, + users: { + sort: 10, + label: _("Users"), + condition: show_user_search_tab? + }, + snippet_titles: { + sort: 11, + label: _("Snippets"), + search: { snippets: true, group_id: nil, project_id: nil }, + condition: show_snippets_search_tab? + } + } + end + + private + + attr_reader :user, :project, :group, :options + + def show_elasticsearch_tabs? + !!options[:show_elasticsearch_tabs] + end + + def search_tab_ability_map + { + milestones: :read_milestone, + snippets: :read_snippet, + issues: :read_issue, + blobs: :read_code, + commits: :read_code, + merge_requests: :read_merge_request, + notes: [:read_merge_request, :read_code, :read_issue, :read_snippet], + users: :read_project_member, + wiki_blobs: :read_wiki + } + end + + def show_user_search_tab? + return true if tab_enabled_for_project?(:users) + return false unless can?(user, :read_users_list) + + project.nil? && feature_flag_tab_enabled?(:global_search_users_tab) + end + + def show_code_search_tab? + return true if tab_enabled_for_project?(:blobs) + + project.nil? && show_elasticsearch_tabs? && feature_flag_tab_enabled?(:global_search_code_tab) + end + + def show_wiki_search_tab? + return true if tab_enabled_for_project?(:wiki_blobs) + + project.nil? && show_elasticsearch_tabs? && feature_flag_tab_enabled?(:global_search_wiki_tab) + end + + def show_commits_search_tab? + return true if tab_enabled_for_project?(:commits) + + project.nil? && show_elasticsearch_tabs? && feature_flag_tab_enabled?(:global_search_commits_tab) + end + + def show_issues_search_tab? + return true if tab_enabled_for_project?(:issues) + + project.nil? && feature_flag_tab_enabled?(:global_search_issues_tab) + end + + def show_merge_requests_search_tab? + return true if tab_enabled_for_project?(:merge_requests) + + project.nil? && feature_flag_tab_enabled?(:global_search_merge_requests_tab) + end + + def show_comments_search_tab? + return true if tab_enabled_for_project?(:notes) + + project.nil? && show_elasticsearch_tabs? + end + + def show_snippets_search_tab? + !!options[:show_snippets] && project.nil? && feature_flag_tab_enabled?(:global_search_snippet_titles_tab) + end + + def show_milestones_search_tab? + project.nil? || tab_enabled_for_project?(:milestones) + end + + def feature_flag_tab_enabled?(flag) + group.present? || Feature.enabled?(flag, user, type: :ops) + end + end +end + +Search::Navigation.prepend_mod diff --git a/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb b/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb index 65393336797..a053288ccea 100644 --- a/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb +++ b/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb @@ -17,6 +17,8 @@ module Sidebars override :configure_menu_items def configure_menu_items [ + :analytics_dashboards, + :dashboards_analytics, :cycle_analytics, :ci_cd_analytics, :contribution_analytics, diff --git a/lib/sidebars/organizations/menus/manage_menu.rb b/lib/sidebars/organizations/menus/manage_menu.rb new file mode 100644 index 00000000000..0df716cdd3f --- /dev/null +++ b/lib/sidebars/organizations/menus/manage_menu.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Sidebars + module Organizations + module Menus + class ManageMenu < ::Sidebars::Menu + override :title + def title + s_('Navigation|Manage') + end + + override :sprite_icon + def sprite_icon + 'users' + end + + override :pick_into_super_sidebar? + def pick_into_super_sidebar? + true + end + + override :configure_menu_items + def configure_menu_items + add_item( + ::Sidebars::MenuItem.new( + title: _('Groups and projects'), + link: groups_and_projects_organization_path(context.container), + super_sidebar_parent: ::Sidebars::Organizations::Menus::ManageMenu, + active_routes: { path: 'organizations/organizations#groups_and_projects' }, + item_id: :organization_groups_and_projects + ) + ) + end + end + end + end +end diff --git a/lib/sidebars/organizations/menus/scope_menu.rb b/lib/sidebars/organizations/menus/scope_menu.rb new file mode 100644 index 00000000000..86f0a083731 --- /dev/null +++ b/lib/sidebars/organizations/menus/scope_menu.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Sidebars + module Organizations + module Menus + class ScopeMenu < ::Sidebars::Menu + override :link + def link + organization_path(context.container) + end + + override :title + def title + context.container.name + end + + override :active_routes + def active_routes + { path: 'organizations/organizations#show' } + end + + override :render? + def render? + true + end + + override :extra_nav_link_html_options + def extra_nav_link_html_options + { + class: 'context-header' + } + end + + override :serialize_as_menu_item_args + def serialize_as_menu_item_args + super.merge({ + title: s_('Organization|Organization overview'), + sprite_icon: 'organization', + super_sidebar_parent: ::Sidebars::StaticMenu, + item_id: :organization_overview + }) + end + end + end + end +end diff --git a/lib/sidebars/organizations/panel.rb b/lib/sidebars/organizations/panel.rb new file mode 100644 index 00000000000..159ccb6cbe9 --- /dev/null +++ b/lib/sidebars/organizations/panel.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Sidebars + module Organizations + class Panel < ::Sidebars::Panel + include ::Sidebars::Concerns::SuperSidebarPanel + + override :aria_label + def aria_label + s_('Organization|Organization navigation') + end + + override :configure_menus + def configure_menus + set_scope_menu(Sidebars::Organizations::Menus::ScopeMenu.new(context)) + add_menu(Sidebars::StaticMenu.new(context)) + add_menu(Sidebars::Organizations::Menus::ManageMenu.new(context)) + end + end + end +end diff --git a/lib/sidebars/organizations/super_sidebar_panel.rb b/lib/sidebars/organizations/super_sidebar_panel.rb new file mode 100644 index 00000000000..acc3d9a0ddf --- /dev/null +++ b/lib/sidebars/organizations/super_sidebar_panel.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Sidebars + module Organizations + class SuperSidebarPanel < ::Sidebars::Organizations::Panel + include ::Sidebars::Concerns::SuperSidebarPanel + extend ::Gitlab::Utils::Override + + override :configure_menus + def configure_menus + super + old_menus = @menus + @menus = [] + + add_menu(Sidebars::StaticMenu.new(context)) + + # Pick old menus, will be obsolete once everything is in their own + # super sidebar menu + pick_from_old_menus(old_menus) + + transform_old_menus(@menus, @scope_menu, *old_menus) + end + + override :super_sidebar_context_header + def super_sidebar_context_header + { + title: context.container.name, + id: context.container.id + } + end + end + end +end diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb index 0a411f075b7..ff2f833763a 100644 --- a/lib/sidebars/projects/menus/deployments_menu.rb +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -9,10 +9,7 @@ module Sidebars add_item(environments_menu_item) add_item(feature_flags_menu_item) add_item(releases_menu_item) - - if Feature.enabled?(:show_pages_in_deployments_menu, context.current_user, type: :experiment) - add_item(pages_menu_item) - end + add_item(pages_menu_item) true end @@ -95,8 +92,8 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Pages'), link: project_pages_path(context.project), - super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu, - active_routes: { path: 'pages#show' }, + super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::DeployMenu, + active_routes: { path: %w[pages#new pages#show] }, item_id: :pages ) end diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index a74448d0bdc..87b09e42fe1 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -8,6 +8,7 @@ module Sidebars def configure_menu_items return false unless feature_enabled? + add_item(tracing_menu_item) add_item(error_tracking_menu_item) add_item(alert_management_menu_item) add_item(incidents_menu_item) @@ -62,6 +63,20 @@ module Sidebars ) end + def tracing_menu_item + unless Gitlab::Observability.tracing_enabled?(context.project) + return ::Sidebars::NilMenuItem.new(item_id: :tracing) + end + + ::Sidebars::MenuItem.new( + title: _('Tracing'), + link: project_tracing_index_path(context.project), + super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::MonitorMenu, + active_routes: { controller: :tracing }, + item_id: :tracing + ) + end + def alert_management_menu_item unless can?(context.current_user, :read_alert_management_alert, context.project) return ::Sidebars::NilMenuItem.new(item_id: :alert_management) diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index f41b7ce1a73..053ce5e82fd 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -98,7 +98,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Model experiments'), link: project_ml_experiments_path(context.project), - super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::DeployMenu, + super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu, active_routes: { controller: %w[projects/ml/experiments projects/ml/candidates] }, item_id: :model_experiments ) diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 9ed142f7f60..9219312ede8 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -16,11 +16,6 @@ module Sidebars add_item(merge_requests_menu_item) add_item(ci_cd_menu_item) add_item(packages_and_registries_menu_item) - - if Feature.disabled?(:show_pages_in_deployments_menu, context.current_user, type: :experiment) - add_item(pages_menu_item) - end - add_item(monitor_menu_item) add_item(usage_quotas_menu_item) @@ -131,19 +126,6 @@ module Sidebars ) end - def pages_menu_item - unless context.project.pages_available? - return ::Sidebars::NilMenuItem.new(item_id: :pages) - end - - ::Sidebars::MenuItem.new( - title: _('Pages'), - link: project_pages_path(context.project), - active_routes: { path: 'pages#show' }, - item_id: :pages - ) - end - def monitor_menu_item if context.project.archived? || !can?(context.current_user, :admin_operations, context.project) return ::Sidebars::NilMenuItem.new(item_id: :monitor) diff --git a/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb b/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb index 2c5dc8a08e7..58b231a269c 100644 --- a/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/analyze_menu.rb @@ -25,7 +25,8 @@ module Sidebars :code_review, :merge_request_analytics, :issues, - :insights + :insights, + :model_experiments ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } end end diff --git a/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb b/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb index 49aa6a23a0e..9f667466d1c 100644 --- a/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb @@ -20,8 +20,7 @@ module Sidebars :releases, :feature_flags, :packages_registry, - :container_registry, - :model_experiments + :container_registry ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } end end diff --git a/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb b/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb index 6e64ac01ffa..0441d3b4a03 100644 --- a/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb @@ -17,6 +17,7 @@ module Sidebars override :configure_menu_items def configure_menu_items [ + :tracing, :error_tracking, :alert_management, :incidents, diff --git a/lib/sidebars/user_settings/menus/access_tokens_menu.rb b/lib/sidebars/user_settings/menus/access_tokens_menu.rb index f52be22e044..ed39b5d6720 100644 --- a/lib/sidebars/user_settings/menus/access_tokens_menu.rb +++ b/lib/sidebars/user_settings/menus/access_tokens_menu.rb @@ -31,7 +31,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - { 'data-qa-selector': 'access_token_link' } + { 'data-testid': 'access_token_link' } end end end diff --git a/lib/sidebars/user_settings/menus/account_menu.rb b/lib/sidebars/user_settings/menus/account_menu.rb index a26dee83da3..53e8b60ea4d 100644 --- a/lib/sidebars/user_settings/menus/account_menu.rb +++ b/lib/sidebars/user_settings/menus/account_menu.rb @@ -28,7 +28,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - { 'data-qa-selector': 'profile_account_link' } + { 'data-testid': 'profile_account_link' } end end end diff --git a/lib/sidebars/user_settings/menus/emails_menu.rb b/lib/sidebars/user_settings/menus/emails_menu.rb index 3b6b4ae1daf..7df7a76238e 100644 --- a/lib/sidebars/user_settings/menus/emails_menu.rb +++ b/lib/sidebars/user_settings/menus/emails_menu.rb @@ -28,7 +28,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - { 'data-qa-selector': 'profile_emails_link' } + { 'data-testid': 'profile_emails_link' } end end end diff --git a/lib/sidebars/user_settings/menus/password_menu.rb b/lib/sidebars/user_settings/menus/password_menu.rb index 9a53e0c727e..e518e1f8bf7 100644 --- a/lib/sidebars/user_settings/menus/password_menu.rb +++ b/lib/sidebars/user_settings/menus/password_menu.rb @@ -31,7 +31,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - { 'data-qa-selector': 'profile_password_link' } + { 'data-testid': 'profile_password_link' } end end end diff --git a/lib/sidebars/user_settings/menus/ssh_keys_menu.rb b/lib/sidebars/user_settings/menus/ssh_keys_menu.rb index 8d92db0147a..bd3cbe30e64 100644 --- a/lib/sidebars/user_settings/menus/ssh_keys_menu.rb +++ b/lib/sidebars/user_settings/menus/ssh_keys_menu.rb @@ -28,7 +28,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - { 'data-qa-selector': 'ssh_keys_link' } + { 'data-testid': 'ssh_keys_link' } end end end diff --git a/lib/slack/manifest.rb b/lib/slack/manifest.rb new file mode 100644 index 00000000000..de189f3bdf5 --- /dev/null +++ b/lib/slack/manifest.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Slack + module Manifest + class << self + delegate :to_json, to: :to_h + + def share_url + "https://api.slack.com/apps?new_app=1&manifest_json=#{ERB::Util.url_encode(to_json)}" + end + + def to_h + { + display_information: display_information, + features: features, + oauth_config: oauth_config, + settings: settings + } + end + + private + + def display_information + { + name: "GitLab (#{Gitlab.config.gitlab.host.first(26)})", + description: s_('SlackIntegration|Interact with GitLab without leaving your Slack workspace!'), + background_color: '#171321', + # Each element in this array will become a paragraph joined with `\r\n\r\n'. + long_description: [ + format( + s_( + 'SlackIntegration|Generated for %{host} by GitLab %{version}.' + ), + host: Gitlab.config.gitlab.host, + version: Gitlab::VERSION + ), + s_( + 'SlackIntegration|- *Notifications:* Get notifications to your team\'s Slack channel about events ' \ + 'happening inside your GitLab projects.' + ), + format( + s_( + 'SlackIntegration|- *Slash commands:* Quickly open, access, or close issues from Slack using the ' \ + '`%{slash_command}` command. Streamline your GitLab deployments with ChatOps.' + ), + slash_command: '/gitlab' + ) + ].join("\r\n\r\n") + } + end + + def features + { + app_home: { + home_tab_enabled: true, + messages_tab_enabled: false, + messages_tab_read_only_enabled: true + }, + bot_user: { + display_name: 'GitLab', + always_online: true + }, + slash_commands: [ + { + command: '/gitlab', + url: api_v4('slack/trigger'), + description: s_('SlackIntegration|GitLab slash commands'), + usage_hint: s_('SlackIntegration|your-project-name-or-alias command'), + should_escape: false + } + ] + } + end + + def oauth_config + { + redirect_urls: [ + Gitlab.config.gitlab.url + ], + scopes: { + bot: %w[ + commands + chat:write + chat:write.public + ] + } + } + end + + def settings + { + event_subscriptions: { + request_url: api_v4('integrations/slack/events'), + bot_events: %w[ + app_home_opened + ] + }, + interactivity: { + is_enabled: true, + request_url: api_v4('integrations/slack/interactions'), + message_menu_options_url: api_v4('integrations/slack/options') + }, + org_deploy_enabled: false, + socket_mode_enabled: false, + token_rotation_enabled: false + } + end + + def api_v4(path) + "#{Gitlab.config.gitlab.url}/api/v4/#{path}" + end + end + end +end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 22ca5d9039c..e1799617ea0 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -8,7 +8,7 @@ namespace :dev do ENV['force'] = 'yes' Rake::Task["gitlab:setup"].invoke - Gitlab::Database::EachDatabase.each_database_connection do |connection| + Gitlab::Database::EachDatabase.each_connection do |connection| # Make sure DB statistics are up to date. # gitlab:setup task can insert quite a bit of data, especially with MASS_INSERT=1 # so ANALYZE can take more than default 15s statement timeout. This being a dev task, @@ -61,7 +61,7 @@ namespace :dev do AND pid <> pg_backend_pid(); SQL - Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection| + Gitlab::Database::EachDatabase.each_connection(include_shared: false) do |connection| connection.execute(cmd) rescue ActiveRecord::NoDatabaseError end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 825388461bc..1a659a930ab 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -42,7 +42,7 @@ namespace :gettext do desc 'Lint all po files in `locale/' task lint: :environment do require 'simple_po_parser' - require 'gitlab/utils' + require 'gitlab/utils/all' require 'parallel' FastGettext.silence_errors diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 026cb39a92f..546f5621515 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -33,7 +33,7 @@ namespace :gitlab do exit 1 end - Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |connection, name| + Gitlab::Database::EachDatabase.each_connection(only: only_on) do |connection, name| connection.execute("INSERT INTO schema_migrations (version) VALUES (#{connection.quote(version)})") puts "Successfully marked '#{version}' as complete on database #{name}".color(:green) @@ -57,7 +57,7 @@ namespace :gitlab do end def drop_tables(only_on: nil) - Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |connection, name| + Gitlab::Database::EachDatabase.each_connection(only: only_on) do |connection, name| # In PostgreSQLAdapter, data_sources returns both views and tables, so use tables instead tables = connection.tables @@ -142,7 +142,7 @@ namespace :gitlab do desc 'This adjusts and cleans db/structure.sql - it runs after db:schema:dump' task :clean_structure_sql do |task_name| ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config| - structure_file = ActiveRecord::Tasks::DatabaseTasks.dump_filename(db_config.name) + structure_file = ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config) schema = File.read(structure_file) @@ -292,7 +292,7 @@ namespace :gitlab do exit end - Gitlab::Database::EachDatabase.each_database_connection(only: database_name) do + Gitlab::Database::EachDatabase.each_connection(only: database_name) do Gitlab::Database::AsyncIndexes.execute_pending_actions!(how_many: args[:pick].to_i) end end @@ -322,7 +322,7 @@ namespace :gitlab do exit end - Gitlab::Database::EachDatabase.each_database_connection(only: database_name) do + Gitlab::Database::EachDatabase.each_connection(only: database_name) do Gitlab::Database::AsyncConstraints.validate_pending_entries!(how_many: args[:pick].to_i) end end @@ -413,7 +413,7 @@ namespace :gitlab do desc 'Run all pending batched migrations' task execute_batched_migrations: :environment do - Gitlab::Database::EachDatabase.each_database_connection do |connection, name| + Gitlab::Database::EachDatabase.each_connection do |connection, name| Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:active).queue_order.each do |migration| Gitlab::AppLogger.info("Executing batched migration #{migration.id} on database #{name} inline") Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new(connection: connection).run_entire_migration(migration) @@ -457,26 +457,20 @@ namespace :gitlab do desc 'Checks schema inconsistencies' task run: :environment do database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] - database = Gitlab::Database::SchemaValidation::Database.new(database_model.connection) + database = Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) stucture_sql_path = Rails.root.join('db/structure.sql') - structure_sql = Gitlab::Database::SchemaValidation::StructureSql.new(stucture_sql_path) + structure_sql = Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) filter = Gitlab::Database::SchemaValidation::InconsistencyFilter.new(IGNORED_TABLES, IGNORED_TRIGGERS) - inconsistencies = - Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database).execute.filter_map(&filter) + validators = Gitlab::Schema::Validation::Validators::Base.all_validators - gitlab_url = 'gitlab-org/gitlab' + inconsistencies = + Gitlab::Schema::Validation::Runner.new(structure_sql, database, validators: validators).execute.filter_map(&filter) inconsistencies.each do |inconsistency| - Gitlab::Database::SchemaValidation::TrackInconsistency.new( - inconsistency, - Project.find_by_full_path(gitlab_url), - User.automation_bot - ).execute - - puts inconsistency.inspect + puts inconsistency.display end end end diff --git a/lib/tasks/gitlab/db/migration_fix_15_11.rake b/lib/tasks/gitlab/db/migration_fix_15_11.rake index fbfee856abb..4716ac5fe9f 100644 --- a/lib/tasks/gitlab/db/migration_fix_15_11.rake +++ b/lib/tasks/gitlab/db/migration_fix_15_11.rake @@ -5,7 +5,7 @@ task migration_fix_15_11: [:environment] do next if Gitlab.com? only_on = %i[main ci].select { |db| Gitlab::Database.has_database?(db) } - Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |conn, database| + Gitlab::Database::EachDatabase.each_connection(only: only_on) do |conn, database| begin first_migration = conn.execute('SELECT * FROM schema_migrations ORDER BY version ASC LIMIT 1') rescue ActiveRecord::StatementInvalid diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake index b3c98e91d17..f42d30e9817 100644 --- a/lib/tasks/gitlab/db/validate_config.rake +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -25,10 +25,7 @@ namespace :gitlab do task validate_config: :environment do original_db_config = ActiveRecord::Base.connection_db_config # rubocop:disable Database/MultipleDatabases - # The include_replicas: is a legacy name to fetch all hidden entries (replica: true or database_tasks: false) - # Once we upgrade to Rails 7.x this should be changed to `include_hidden: true` - # Ref.: https://github.com/rails/rails/blob/f2d9316ba965e150ad04596085ee10eea4f58d3e/activerecord/lib/active_record/database_configurations.rb#L48 - db_configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, include_replicas: true) + db_configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, include_hidden: true) db_configs = db_configs.reject(&:replica?) # The `pg_control_system()` is not enough to properly discover matching database systems diff --git a/lib/tasks/gitlab/docs/compile_deprecations.rake b/lib/tasks/gitlab/docs/compile_deprecations.rake index f7821315f82..f6d8944c5a1 100644 --- a/lib/tasks/gitlab/docs/compile_deprecations.rake +++ b/lib/tasks/gitlab/docs/compile_deprecations.rake @@ -35,34 +35,5 @@ namespace :gitlab do abort end end - - desc "Generate removal list from individual files" - task :compile_removals do - require_relative '../../../../tooling/docs/deprecation_handling' - path = Rails.root.join("doc/update/removals.md") - File.write(path, Docs::DeprecationHandling.new('removal').render) - puts "#{COLOR_CODE_GREEN}INFO: Removals compiled to #{path}.#{COLOR_CODE_RESET}" - end - - desc "Check that the removal documentation is up to date" - task :check_removals do - require_relative '../../../../tooling/docs/deprecation_handling' - path = Rails.root.join("doc/update/removals.md") - contents = Docs::DeprecationHandling.new('removal').render - doc = File.read(path) - - if doc == contents - puts "#{COLOR_CODE_GREEN}INFO: Removals documentation is up to date.#{COLOR_CODE_RESET}" - else - warn <<~EOS - #{COLOR_CODE_RED}ERROR: Removals documentation is outdated!#{COLOR_CODE_RESET} - To update the removals documentation, either: - - - Run `bin/rake gitlab:docs:compile_removals` and commit the changes to this branch. - - Have a technical writer resolve the issue. - EOS - abort - end - end end end diff --git a/lib/tasks/gitlab/metrics_exporter.rake b/lib/tasks/gitlab/metrics_exporter.rake deleted file mode 100644 index 70719648fc5..00000000000 --- a/lib/tasks/gitlab/metrics_exporter.rake +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -namespace :gitlab do - require_relative Rails.root.join('metrics_server', 'dependencies') - require_relative Rails.root.join('metrics_server', 'metrics_server') - - namespace :metrics_exporter do - REPO = 'https://gitlab.com/gitlab-org/gitlab-metrics-exporter.git' - - desc "GitLab | Metrics Exporter | Install or upgrade gitlab-metrics-exporter" - task :install, [:dir] => :gitlab_environment do |t, args| - unless args.dir.present? - abort %(Please specify the directory where you want to install the exporter -Usage: rake "gitlab:metrics_exporter:install[/installation/dir]") - end - - version = ENV['GITLAB_METRICS_EXPORTER_VERSION'] || MetricsServer.version - make = Gitlab::Utils.which('gmake') || Gitlab::Utils.which('make') - - abort "Couldn't find a 'make' binary" unless make - - checkout_or_clone_version(version: version, repo: REPO, target_dir: args.dir, clone_opts: %w[--depth 1]) - - Dir.chdir(args.dir) { run_command!([make]) } - end - end -end diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake deleted file mode 100644 index b5dfd163dba..00000000000 --- a/lib/tasks/gitlab/packages/events.rake +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -desc "GitLab | Packages | Events | Generate hll counter events file for packages" -namespace :gitlab do - namespace :packages do - namespace :events do - task generate: :environment do - Rake::Task["gitlab:packages:events:generate_counts"].invoke - Rake::Task["gitlab:packages:events:generate_unique"].invoke - rescue StandardError => e - logger.error("Error building events list: #{e}") - end - - task generate_counts: :environment do - logger = Logger.new($stdout) - logger.info('Building list of package events...') - - path = Gitlab::UsageDataCounters::PackageEventCounter::KNOWN_EVENTS_PATH - File.open(path, "w") { |file| file << counter_events_list.to_yaml } - - logger.info("Events file `#{path}` generated successfully") - rescue StandardError => e - logger.error("Error building events list: #{e}") - end - - task generate_unique: :environment do - logger = Logger.new($stdout) - logger.info('Building list of package events...') - - path = File.join(File.dirname(Gitlab::UsageDataCounters::HLLRedisCounter::KNOWN_EVENTS_PATH), 'package_events.yml') - File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml } - - logger.info("Events file `#{path}` generated successfully") - rescue StandardError => e - logger.error("Error building events list: #{e}") - end - - private - - def event_pairs - Packages::Event::EVENT_TYPES.product(Packages::Event::EVENT_SCOPES.keys) - end - - def generate_unique_events_list - events = event_pairs.each_with_object([]) do |(event_type, event_scope), events| - Packages::Event::ORIGINATOR_TYPES.excluding(:guest).each do |originator_type| - events_definition = Packages::Event.unique_counters_for(event_scope, event_type, originator_type).map do |event_name| - { "name" => event_name } - end - - events.concat(events_definition) - end - end - - events.sort_by { |event| event["name"] }.uniq - end - - def counter_events_list - counters = event_pairs.flat_map do |event_type, event_scope| - Packages::Event::ORIGINATOR_TYPES.flat_map do |originator_type| - Packages::Event.counters_for(event_scope, event_type, originator_type) - end - end - - counters.compact.sort.uniq - end - end - end -end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index a5dcb23450f..8b305d68c68 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -44,7 +44,7 @@ namespace :gitlab do desc "GitLab | Shell | Setup gitlab-shell" task setup: :gitlab_environment do - setup + setup_gitlab_shell end desc "GitLab | Shell | Build missing projects" @@ -63,10 +63,13 @@ namespace :gitlab do end end - def setup - warn_user_is_not_gitlab + def setup_gitlab_shell + unless Gitlab::CurrentSettings.authorized_keys_enabled? + puts 'The "Write to authorized_keys" setting is disabled. Skipping rebuilding the authorized_keys file...' + return + end - ensure_write_to_authorized_keys_is_enabled + warn_user_is_not_gitlab unless ENV['force'] == 'yes' puts "This task will now rebuild the authorized_keys file." @@ -89,44 +92,4 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end - - def ensure_write_to_authorized_keys_is_enabled - return if Gitlab::CurrentSettings.authorized_keys_enabled? - - puts authorized_keys_is_disabled_warning - - unless ENV['force'] == 'yes' - puts 'Do you want to permanently enable the "Write to authorized_keys file" setting now?' - ask_to_continue - end - - puts 'Enabling the "Write to authorized_keys file" setting...' - Gitlab::CurrentSettings.update!(authorized_keys_enabled: true) - - puts 'Successfully enabled "Write to authorized_keys file"!' - puts '' - end - - def authorized_keys_is_disabled_warning - <<-MSG.strip_heredoc - WARNING - - The "Write to authorized_keys file" setting is disabled, which prevents - the file from being rebuilt! - - It should be enabled for most GitLab installations. Large installations - may wish to disable it as part of speeding up SSH operations. - - See https://docs.gitlab.com/ee/administration/operations/fast_ssh_key_lookup.html - - If you did not intentionally disable this option in Admin Area > Settings, - then you may have been affected by the 9.3.0 bug in which the new setting - was disabled by default. - - https://gitlab.com/gitlab-org/gitlab/issues/2738 - - It was reverted in 9.3.1 and fixed in 9.3.3, however, if Settings were - saved while the setting was unchecked, then it is still disabled. - MSG - end end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index f5bf1a266e5..1cd72ee6a1b 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -34,21 +34,6 @@ namespace :gitlab do puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data) end - desc 'GitLab | UsageDataMetrics | Generate known_events/ci_templates.yml based on template definitions' - task generate_ci_template_events: :environment do - banner = <<~BANNER - # This file is generated automatically by - # bin/rake gitlab:usage_data:generate_ci_template_events - # - # Do not edit it manually! - BANNER - - all_includes = explicit_template_includes + implicit_auto_devops_includes - yaml = banner + YAML.dump(all_includes).gsub(/ *$/m, '') - - File.write(Gitlab::UsageDataCounters::CiTemplateUniqueCounter::KNOWN_EVENTS_FILE_PATH, yaml) - end - desc 'GitLab | UsageDataMetrics | Generate raw SQL metrics queries for RSpec' task generate_sql_metrics_queries: :environment do require 'active_support/testing/time_helpers' |