diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-14 11:41:52 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-14 11:41:52 +0300 |
| commit | 585826cb22ecea5998a2c2a4675735c94bdeedac (patch) | |
| tree | 5b05f0b30d33cef48963609e8a18a4dff260eab3 /lib | |
| parent | df221d036e5d0c6c0ee4d55b9c97f481ee05dee8 (diff) | |
Add latest changes from gitlab-org/gitlab@16-6-stable-eev16.6.0-rc42
Diffstat (limited to 'lib')
296 files changed, 11655 insertions, 2046 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 8a26ae7e6f6..43a21c11dbc 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -387,16 +387,7 @@ module API mount ::API::Internal::MailRoom mount ::API::Internal::ContainerRegistry::Migration mount ::API::Internal::Workhorse - - version 'v3', using: :path do - # Although the following endpoints are kept behind V3 namespace, - # they're not deprecated neither should be removed when V3 get - # removed. They're needed as a layer to integrate with Jira - # Development Panel. - namespace '/', requirements: ::API::V3::Github::ENDPOINT_REQUIREMENTS do - mount ::API::V3::Github - end - end + mount ::API::Internal::Shellhorse route :any, '*path', feature_category: :not_owned do # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned error!('404 Not Found', 404) diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index 9bcc16cf211..9dc0e5bae9b 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -214,6 +214,23 @@ module API get ':import_id/entities/:entity_id' do present bulk_import_entity, with: Entities::BulkImports::Entity end + + desc 'Get GitLab Migration entity failures' do + detail 'This feature was introduced in GitLab 16.6' + success code: 200, model: Entities::BulkImports::EntityFailure + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + end + params do + requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" + requires :entity_id, type: Integer, desc: "The ID of GitLab Migration entity" + end + get ':import_id/entities/:entity_id/failures' do + present paginate(bulk_import_entity.failures), with: Entities::BulkImports::EntityFailure + end end end end diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index 6f0a2ff7f62..250fe249489 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -57,7 +57,7 @@ module API builds = filter_builds(builds, params[:scope]) builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project) - present paginate_with_strategies(builds, paginator_params: { without_count: true }), with: Entities::Ci::Job + present paginate_with_strategies(builds, user_project, paginator_params: { without_count: true }), with: Entities::Ci::Job end # rubocop: enable CodeReuse/ActiveRecord @@ -122,10 +122,10 @@ module API requires :job_id, type: Integer, desc: 'The ID of a job', documentation: { example: 88 } end post ':id/jobs/:job_id/cancel', urgency: :low, feature_category: :continuous_integration do - authorize_update_builds! + authorize_cancel_builds! build = find_build!(params[:job_id]) - authorize!(:update_build, build) + authorize!(:cancel_build, build) build.cancel diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index bd5c04f401b..b5123ab49dc 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -288,6 +288,33 @@ module API end end + desc 'Updates pipeline metadata' do + detail 'This feature was introduced in GitLab 16.6' + success status: 200, model: Entities::Ci::PipelineWithMetadata + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } + requires :name, type: String, desc: 'The name of the pipeline', documentation: { example: 'Deployment to production' } + end + route_setting :authentication, job_token_allowed: true + put ':id/pipelines/:pipeline_id/metadata', urgency: :low, feature_category: :continuous_integration do + authorize! :update_pipeline, pipeline + + response = ::Ci::Pipelines::UpdateMetadataService.new(pipeline, params.slice(:name)).execute + + if response.success? + present response.payload, with: Entities::Ci::PipelineWithMetadata + else + render_api_error_with_reason!(response.reason, response.message, response.payload.join(', ')) + end + end + desc 'Retry builds in the pipeline' do detail 'This feature was introduced in GitLab 8.11.' success status: 201, model: Entities::Ci::Pipeline @@ -325,7 +352,7 @@ module API requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } end post ':id/pipelines/:pipeline_id/cancel', urgency: :low, feature_category: :continuous_integration do - authorize! :update_pipeline, pipeline + authorize! :cancel_pipeline, pipeline # TODO: inconsistent behavior: when pipeline is not cancelable we should return an error ::Ci::CancelPipelineService.new(pipeline: pipeline, current_user: current_user).execute diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 42817c782f4..17bee275c51 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -24,6 +24,9 @@ module API desc: 'The status of runners to return' optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'A list of runner tags', documentation: { example: "['macos', 'shell']" } + optional :version_prefix, type: String, desc: 'The version prefix of runners to return', documentation: { example: "'15.1.' or '16.'" }, + regexp: /^[\d+.]+/ + use :pagination end @@ -46,6 +49,7 @@ module API runners = filter_runners(runners, params[:type], allowed_scopes: ::Ci::Runner::AVAILABLE_TYPES) runners = filter_runners(runners, params[:status], allowed_scopes: ::Ci::Runner::AVAILABLE_STATUSES) runners = filter_runners(runners, params[:paused] ? 'paused' : 'active', allowed_scopes: %w[paused active]) if params.include?(:paused) + runners = runners.with_version_prefix(params[:version_prefix]) if params[:version_prefix] runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] runners diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index acb64cd0d3a..62b2885f955 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -94,37 +94,6 @@ module API # rubocop: enable CodeReuse/ActiveRecord helpers do - def commit - strong_memoize(:commit) do - user_project.commit(params[:sha]) - end - end - - def all_matching_pipelines - pipelines = user_project.ci_pipelines.newest_first(sha: commit.sha) - pipelines = pipelines.for_ref(params[:ref]) if params[:ref] - pipelines = pipelines.id_in(params[:pipeline_id]) if params[:pipeline_id] - pipelines - end - - def apply_job_state!(job) - case params[:state] - when 'pending' - job.enqueue! - when 'running' - job.enqueue - job.run! - when 'success' - job.success! - when 'failed' - job.drop!(:api_failure) - when 'canceled' - job.cancel! - else - render_api_error!('invalid state', 400) - end - end - def optional_commit_status_params updatable_optional_attributes = %w[target_url description coverage] attributes_for_keys(updatable_optional_attributes) diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index bfaba5c4d7a..19d63a39242 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -202,7 +202,8 @@ module API get '*package_name', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do package_name = params[:package_name] available_packages = - if Feature.enabled?(:npm_allow_packages_in_multiple_projects) + if endpoint_scope != :project && + Feature.enabled?(:npm_allow_packages_in_multiple_projects, group_or_namespace) finder_for_endpoint_scope(package_name).execute else ::Packages::Npm::PackageFinder.new(package_name, project: project_or_nil) @@ -218,9 +219,8 @@ module API target: project_or_nil, package_name: package_name ) do - if endpoint_scope == :project || Feature.disabled?(:npm_allow_packages_in_multiple_projects) - authorize_read_package!(project) - elsif Feature.enabled?(:npm_allow_packages_in_multiple_projects) + if endpoint_scope != :project && + Feature.enabled?(:npm_allow_packages_in_multiple_projects, group_or_namespace) available_packages_to_user = ::Packages::Npm::PackagesForUserFinder.new( current_user, group_or_namespace, @@ -232,6 +232,8 @@ module API end available_packages = available_packages_to_user + else + authorize_read_package!(project) end not_found!('Packages') if available_packages.empty? diff --git a/lib/api/entities/bulk_imports/entity_failure.rb b/lib/api/entities/bulk_imports/entity_failure.rb index 3e69e7fa2aa..08708a7c961 100644 --- a/lib/api/entities/bulk_imports/entity_failure.rb +++ b/lib/api/entities/bulk_imports/entity_failure.rb @@ -4,18 +4,14 @@ module API module Entities module BulkImports class EntityFailure < Grape::Entity - expose :relation, documentation: { type: 'string', example: 'group' } - expose :pipeline_step, as: :step, documentation: { type: 'string', example: 'extractor' } + expose :relation, documentation: { type: 'string', example: 'label' } expose :exception_message, documentation: { type: 'string', example: 'error message' } do |failure| ::Projects::ImportErrorFilter.filter_message(failure.exception_message.truncate(72)) end expose :exception_class, documentation: { type: 'string', example: 'Exception' } expose :correlation_id_value, documentation: { type: 'string', example: 'dfcf583058ed4508e4c7c617bd7f0edd' } - expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } - expose :pipeline_class, documentation: { - type: 'string', example: 'BulkImports::Groups::Pipelines::GroupPipeline' - } - expose :pipeline_step, documentation: { type: 'string', example: 'extractor' } + expose :source_url, documentation: { type: 'string', example: 'https://source.gitlab.com/group/-/epics/1' } + expose :source_title, documentation: { type: 'string', example: 'title' } end end end diff --git a/lib/api/entities/commit_signature.rb b/lib/api/entities/commit_signature.rb index 9c30c3c59ea..cdd63df77f0 100644 --- a/lib/api/entities/commit_signature.rb +++ b/lib/api/entities/commit_signature.rb @@ -6,27 +6,24 @@ module API expose :signature_type, documentation: { type: 'string', example: 'PGP' } expose :signature, merge: true do |commit, options| - if commit.signature.is_a?(::CommitSignatures::GpgSignature) || commit.raw_commit_from_rugged? + case commit.signature + when ::CommitSignatures::GpgSignature ::API::Entities::GpgCommitSignature.represent commit_signature(commit), options - elsif commit.signature.is_a?(::CommitSignatures::X509CommitSignature) + when ::CommitSignatures::X509CommitSignature ::API::Entities::X509Signature.represent commit.signature, options - elsif commit.signature.is_a?(::CommitSignatures::SshSignature) + when ::CommitSignatures::SshSignature ::API::Entities::SshSignature.represent(commit.signature, options) end end - expose :commit_source, documentation: { type: 'string', example: 'gitaly' } do |commit, _| - commit.raw_commit_from_rugged? ? "rugged" : "gitaly" + expose :commit_source, documentation: { type: 'string', example: 'gitaly' } do |_commit, _| + "gitaly" end private def commit_signature(commit) - if commit.raw_commit_from_rugged? - commit.gpg_commit.signature - else - commit.signature - end + commit.signature end end end diff --git a/lib/api/entities/group.rb b/lib/api/entities/group.rb index d18a29ce4d4..1a1765c2e0a 100644 --- a/lib/api/entities/group.rb +++ b/lib/api/entities/group.rb @@ -10,7 +10,8 @@ module API expose :project_creation_level_str, as: :project_creation_level expose :auto_devops_enabled expose :subgroup_creation_level_str, as: :subgroup_creation_level - expose :emails_disabled + expose(:emails_disabled, documentation: { type: 'boolean' }) { |group, options| group.emails_disabled? } + expose :emails_enabled, documentation: { type: 'boolean' } expose :mentions_disabled expose :lfs_enabled?, as: :lfs_enabled expose :default_branch_protection diff --git a/lib/api/entities/ml/mlflow/model_versions/responses/get.rb b/lib/api/entities/ml/mlflow/model_versions/responses/get.rb new file mode 100644 index 00000000000..14baae03644 --- /dev/null +++ b/lib/api/entities/ml/mlflow/model_versions/responses/get.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + module ModelVersions + module Responses + class Get < Grape::Entity + expose :model_version, with: Types::ModelVersion + end + end + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/model_versions/types/model_version.rb b/lib/api/entities/ml/mlflow/model_versions/types/model_version.rb new file mode 100644 index 00000000000..407158521f7 --- /dev/null +++ b/lib/api/entities/ml/mlflow/model_versions/types/model_version.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + module ModelVersions + module Types + class ModelVersion < Grape::Entity + expose :name + expose :version + expose :creation_timestamp, documentation: { type: Integer } + expose :last_updated_timestamp, documentation: { type: Integer } + expose :user_id + expose :current_stage + expose :description + expose :source + expose :run_id + expose :status + expose :status_message + expose :metadata + expose :run_link + expose :aliases, documentation: { is_array: true, type: String } + + private + + def name + object.model.name + end + + def creation_timestamp + object.created_at.to_i + end + + def last_updated_timestamp + object.updated_at.to_i + end + + def user_id + nil + end + + def current_stage + "development" + end + + def description + "" + end + + def source + model_name = object.model.name + "api/v4/projects/(id)/packages/ml_models/#{model_name}/model_version/" + end + + def run_id + "" + end + + def status + "READY" + end + + def status_message + "" + end + + def metadata + [] + end + + def run_link + "" + end + + def aliases + [] + end + end + end + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/model_versions/types/model_version_tag.rb b/lib/api/entities/ml/mlflow/model_versions/types/model_version_tag.rb new file mode 100644 index 00000000000..f5ad3bf3fb9 --- /dev/null +++ b/lib/api/entities/ml/mlflow/model_versions/types/model_version_tag.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + module ModelVersions + module Types + class ModelVersionTag < Grape::Entity + expose :key + expose :value + end + end + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/registered_model.rb b/lib/api/entities/ml/mlflow/registered_model.rb new file mode 100644 index 00000000000..1ff983e1611 --- /dev/null +++ b/lib/api/entities/ml/mlflow/registered_model.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class RegisteredModel < Grape::Entity + expose :name + expose :created_at, as: :creation_timestamp + expose :updated_at, as: :last_updated_timestamp + expose :description + expose(:user_id) { |model| model.user_id.to_s } + expose :metadata, as: :tags, using: KeyValue + end + end + end + end +end diff --git a/lib/api/entities/wiki_page.rb b/lib/api/entities/wiki_page.rb index 0f3fdd586a3..8b2c951ecf2 100644 --- a/lib/api/entities/wiki_page.rb +++ b/lib/api/entities/wiki_page.rb @@ -22,6 +22,10 @@ module API expose :encoding, documentation: { type: 'string', example: 'UTF-8' } do |wiki_page| wiki_page.content.encoding.name end + + expose :front_matter, documentation: { type: 'Hash', example: { title: "deploy" } }, if: ->(wiki_page) { + ::Feature.enabled?(:wiki_front_matter_title, wiki_page.container) + } end end end diff --git a/lib/api/github/entities.rb b/lib/api/github/entities.rb deleted file mode 100644 index 125985f0e23..00000000000 --- a/lib/api/github/entities.rb +++ /dev/null @@ -1,219 +0,0 @@ -# frozen_string_literal: true - -# Simplified version of Github API entities. -# It's mainly used to mimic Github API and integrate with Jira Development Panel. -# -module API - module Github - module Entities - class Repository < Grape::Entity - expose :id - expose :owner do |project, options| - root_namespace = options[:root_namespace] || project.root_namespace - - { login: root_namespace.path } - end - expose :name do |project, options| - ::Gitlab::Jira::Dvcs.encode_project_name(project) - end - end - - class BranchCommit < Grape::Entity - expose :id, as: :sha - expose :type do |_| - 'commit' - end - end - - class RepoCommit < Grape::Entity - expose :id, as: :sha - expose :author do |commit| - { - login: commit.author&.username, - email: commit.author_email - } - end - expose :committer do |commit| - { - login: commit.author&.username, - email: commit.committer_email - } - end - expose :commit do |commit| - { - author: { - name: commit.author_name, - email: commit.author_email, - date: commit.authored_date.iso8601, - type: 'User' - }, - committer: { - name: commit.committer_name, - email: commit.committer_email, - date: commit.committed_date.iso8601, - type: 'User' - }, - message: commit.safe_message - } - end - expose :parents do |commit| - commit.parent_ids.map { |id| { sha: id } } - end - expose :files do |_commit, options| - options[:diff_files].flat_map do |diff| - additions = diff.added_lines - deletions = diff.removed_lines - - if diff.new_file? - { - status: 'added', - filename: diff.new_path, - additions: additions, - changes: additions - } - elsif diff.deleted_file? - { - status: 'removed', - filename: diff.old_path, - deletions: deletions, - changes: deletions - } - elsif diff.renamed_file? - [ - { - status: 'removed', - filename: diff.old_path, - deletions: deletions, - changes: deletions - }, - { - status: 'added', - filename: diff.new_path, - additions: additions, - changes: additions - } - ] - else - { - status: 'modified', - filename: diff.new_path, - additions: additions, - deletions: deletions, - changes: (additions + deletions) - } - end - end - end - end - - class Branch < Grape::Entity - expose :name - - expose :commit, using: BranchCommit do |repo_branch, options| - options[:project].repository.commit(repo_branch.dereferenced_target) - end - end - - class User < Grape::Entity - expose :id - expose :username, as: :login - expose :user_url, as: :url - expose :user_url, as: :html_url - expose :avatar_url do |user| - user.avatar_url(only_path: false) - end - - private - - def user_url - Gitlab::Routing.url_helpers.user_url(object) - end - end - - class NoteableComment < Grape::Entity - expose :id - expose :author, as: :user, using: User - expose :note, as: :body - expose :created_at - end - - class PullRequest < Grape::Entity - expose :title - expose :assignee, using: User do |merge_request| - merge_request.assignee - end - expose :author, as: :user, using: User - expose :created_at - expose :description, as: :body - # Since Jira service requests `/repos/-/jira/pulls` (without project - # scope), we need to make it work with ID instead IID. - expose :id, as: :number - # GitHub doesn't have a "merged" or "closed" state. It's just "open" or - # "closed". - expose :state do |merge_request| - case merge_request.state - when 'opened', 'locked' - 'open' - when 'merged' - 'closed' - else - merge_request.state - end - end - expose :merged?, as: :merged - expose :merged_at do |merge_request| - merge_request.metrics&.merged_at - end - expose :closed_at do |merge_request| - merge_request.metrics&.latest_closed_at - end - expose :updated_at - expose :html_url do |merge_request| - Gitlab::UrlBuilder.build(merge_request) - end - expose :head do - expose :source_branch, as: :label - expose :source_branch, as: :ref - expose :source_project, as: :repo, using: Repository - end - expose :base do - expose :target_branch, as: :label - expose :target_branch, as: :ref - expose :target_project, as: :repo, using: Repository - end - end - - class PullRequestPayload < Grape::Entity - expose :action do |merge_request| - case merge_request.state - when 'merged', 'closed' - 'closed' - else - 'opened' - end - end - - expose :id - expose :pull_request, using: PullRequest do |merge_request| - merge_request - end - end - - class PullRequestEvent < Grape::Entity - expose :id do |merge_request| - updated_at = merge_request.updated_at.to_i - "#{merge_request.id}-#{updated_at}" - end - expose :type do |_merge_request| - 'PullRequestEvent' - end - expose :updated_at, as: :created_at - expose :payload, using: PullRequestPayload do |merge_request| - # The merge request data is used by PullRequestPayload and PullRequest, so we just provide it - # here. Otherwise Grape::Entity would try to access a field "payload" on Merge Request. - merge_request - end - end - end - end -end diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb index c2b4cbf732f..b363f59b7ad 100644 --- a/lib/api/group_packages.rb +++ b/lib/api/group_packages.rb @@ -47,6 +47,9 @@ module API optional :package_name, type: String, desc: 'Return packages with this name' + optional :package_version, + type: String, + desc: 'Return packages with this version' optional :include_versionless, type: Boolean, desc: 'Returns packages without a version' @@ -60,7 +63,8 @@ module API current_user, user_group, declared(params).slice( - :exclude_subgroups, :order_by, :sort, :package_type, :package_name, :include_versionless, :status + :exclude_subgroups, :order_by, :sort, :package_type, :package_name, + :package_version, :include_versionless, :status ) ).execute diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 2efdfe109f7..1ff64cd2ffd 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -254,6 +254,7 @@ module API group = find_group!(params[:id]) group.preload_shared_group_links + mark_throttle! :update_namespace_name, scope: group if params.key?(:name) && params[:name].present? authorize! :admin_group, group group.remove_avatar! if params.key?(:avatar) && params[:avatar].nil? diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index 8260d8a88f8..c811f47cb5b 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -75,6 +75,8 @@ module API requires :file_name, type: String, desc: 'Helm package file name', documentation: { example: 'mychart' } end get ":channel/charts/:file_name.tgz" do + not_found!("Format #{params[:format]}") unless params[:format].nil? + project = authorized_user_project(action: :read_package) authorize_read_package!(project) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 56b157f662a..bb94d5d14d0 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -141,7 +141,7 @@ module API def find_project(id) return unless id - projects = Project.without_deleted.not_hidden + projects = find_project_scopes if id.is_a?(Integer) || id =~ INTEGER_ID_REGEX projects.find_by(id: id) @@ -151,6 +151,11 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + # Can be overriden by API endpoints + def find_project_scopes + Project.without_deleted.not_hidden + end + def find_project!(id) project = find_project(id) @@ -337,6 +342,12 @@ module API unauthorized! end + def authenticate_by_gitlab_shell_or_workhorse_token! + return require_gitlab_workhorse! unless headers[GITLAB_SHELL_API_HEADER].present? + + authenticate_by_gitlab_shell_token! + end + def authenticated_with_can_read_all_resources! authenticate! forbidden! unless current_user.can_read_all_resources? @@ -391,6 +402,10 @@ module API authorize! :update_build, user_project end + def authorize_cancel_builds! + authorize! :cancel_build, user_project + end + def require_repository_enabled!(subject = :global) not_found!("Repository") unless user_project.feature_available?(:repository, current_user) end @@ -758,6 +773,7 @@ module API finder_params[:id_before] = sanitize_id_param(params[:id_before]) if params[:id_before] finder_params[:updated_after] = declared_params[:updated_after] if declared_params[:updated_after] finder_params[:updated_before] = declared_params[:updated_before] if declared_params[:updated_before] + finder_params[:include_pending_delete] = declared_params[:include_pending_delete] if declared_params[:include_pending_delete] finder_params end @@ -891,7 +907,7 @@ module API def project_moved?(id, project) return false unless Feature.enabled?(:api_redirect_moved_projects) return false unless id.is_a?(String) && id.include?('/') - return false if project.blank? || id == project.full_path + return false if project.blank? || project.full_path.casecmp?(id) return false unless params[:id] == id true diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index f7802938d8b..fbe13bfe8f7 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -18,7 +18,8 @@ module API optional :project_creation_level, type: String, values: ::Gitlab::Access.project_creation_string_values, desc: 'Determine if developers can create projects in the group', as: :project_creation_level_str optional :auto_devops_enabled, type: Boolean, desc: 'Default to Auto DevOps pipeline for all projects within this group' optional :subgroup_creation_level, type: String, values: ::Gitlab::Access.subgroup_creation_string_values, desc: 'Allowed to create subgroups', as: :subgroup_creation_level_str - optional :emails_disabled, type: Boolean, desc: 'Disable email notifications' + optional :emails_disabled, type: Boolean, desc: '_(Deprecated)_ Disable email notifications. Use: emails_enabled' + optional :emails_enabled, type: Boolean, desc: 'Enable email notifications' optional :mentions_disabled, type: Boolean, desc: 'Disable a group from getting mentioned' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index f66f899c98b..0c5b12d48e9 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -123,6 +123,10 @@ module API # Defined in EE end + def need_git_audit_event? + false + end + private def repository_path diff --git a/lib/api/helpers/kubernetes/agent_helpers.rb b/lib/api/helpers/kubernetes/agent_helpers.rb index 50a8c2a5aed..aa4f4310e1d 100644 --- a/lib/api/helpers/kubernetes/agent_helpers.rb +++ b/lib/api/helpers/kubernetes/agent_helpers.rb @@ -41,7 +41,7 @@ module API end def agent_has_access_to_project?(project) - Guest.can?(:download_code, project) || agent.has_access_to?(project) + ::Users::Anonymous.can?(:download_code, project) || agent.has_access_to?(project) end def increment_unique_events diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index ef3da055b19..c91eef0c4b0 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -64,7 +64,7 @@ module API package_name = params[:package_name] namespace = - if Feature.enabled?(:npm_allow_packages_in_multiple_projects) + if Feature.enabled?(:npm_allow_packages_in_multiple_projects, top_namespace_from(package_name)) top_namespace_from(package_name) else namespace_path = ::Packages::Npm.scope_of(package_name) @@ -94,10 +94,12 @@ module API private def top_namespace_from(package_name) - namespace_path = ::Packages::Npm.scope_of(package_name) - return unless namespace_path + strong_memoize_with(:top_namespace_from, package_name) do + namespace_path = ::Packages::Npm.scope_of(package_name) + next unless namespace_path - Namespace.top_most.by_path(namespace_path) + Namespace.top_most.by_path(namespace_path) + end end def group diff --git a/lib/api/helpers/pagination_strategies.rb b/lib/api/helpers/pagination_strategies.rb index 5fbc3081ee8..4353ba0e99a 100644 --- a/lib/api/helpers/pagination_strategies.rb +++ b/lib/api/helpers/pagination_strategies.rb @@ -50,7 +50,7 @@ module API offset_limit = limit_for_scope(request_scope) if (Gitlab::Pagination::Keyset.available_for_type?(relation) || cursor_based_keyset_pagination_supported?(relation)) && - cursor_based_keyset_pagination_enforced?(relation) && + cursor_based_keyset_pagination_enforced?(request_scope, relation) && offset_limit_exceeded?(offset_limit) return error!("Offset pagination has a maximum allowed offset of #{offset_limit} " \ @@ -65,8 +65,8 @@ module API Gitlab::Pagination::CursorBasedKeyset.available_for_type?(relation) end - def cursor_based_keyset_pagination_enforced?(relation) - Gitlab::Pagination::CursorBasedKeyset.enforced_for_type?(relation) + def cursor_based_keyset_pagination_enforced?(request_scope, relation) + Gitlab::Pagination::CursorBasedKeyset.enforced_for_type?(request_scope, relation) end def keyset_pagination_enabled? diff --git a/lib/api/helpers/rate_limiter.rb b/lib/api/helpers/rate_limiter.rb index be92277c25a..39940d86fbf 100644 --- a/lib/api/helpers/rate_limiter.rb +++ b/lib/api/helpers/rate_limiter.rb @@ -18,6 +18,10 @@ module API render_api_error!({ error: _('This endpoint has been requested too many times. Try again later.') }, 429) end + + def mark_throttle!(key, scope:) + Gitlab::ApplicationRateLimiter.throttled?(key, scope: scope) + end end end end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index f9dc888fbeb..87b3838fb85 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -4,12 +4,18 @@ module API # Internal access API module Internal class Base < ::API::Base + include Gitlab::RackLoadBalancingHelpers + before { authenticate_by_gitlab_shell_token! } before do api_endpoint = env['api.endpoint'] feature_category = api_endpoint.options[:for].try(:feature_category_for_app, api_endpoint).to_s + if actor.user + load_balancer_stick_request(::User, :user, actor.user.id) + end + Gitlab::ApplicationContext.push( user: -> { actor&.user }, project: -> { project }, @@ -49,6 +55,11 @@ module API env = parse_env Gitlab::Git::HookEnv.set(gl_repository, env) if container + # Snapshot repositories have different relative path than the main repository. For access + # checks that need quarantined objects the relative path in also sent with Gitaly RPCs + # calls as a header. + populate_relative_path(params[:relative_path]) + actor.update_last_used_at! check_result = access_check_result @@ -66,7 +77,8 @@ module API git_config_options: ["uploadpack.allowFilter=true", "uploadpack.allowAnySHA1InWant=true"], gitaly: gitaly_payload(params[:action]), - gl_console_messages: check_result.console_messages + gl_console_messages: check_result.console_messages, + need_audit: need_git_audit_event? }.merge!(actor.key_details) # Custom option for git-receive-pack command @@ -77,7 +89,9 @@ module API payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" end - send_git_audit_streaming_event(protocol: params[:protocol], action: params[:action]) + unless Feature.enabled?(:log_git_streaming_audit_events, project) + send_git_audit_streaming_event(protocol: params[:protocol], action: params[:action]) + end response_with_status(**payload) when ::Gitlab::GitAccessResult::CustomAction @@ -88,6 +102,12 @@ module API end # rubocop: enable Metrics/AbcSize + def populate_relative_path(relative_path) + return unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore[:gitlab_git_relative_path] = relative_path + end + def validate_actor(actor) return 'Could not find the given key' unless actor.key @@ -112,6 +132,7 @@ module API # username - user name for Git over SSH in keyless SSH cert mode # protocol - Git access protocol being used, e.g. HTTP or SSH # project - project full_path (not path on disk) + # relative_path - relative path of repository having access checks performed. # action - git action (git-upload-pack or git-receive-pack) # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList # check_ip - optional, only in EE version, may limit access to diff --git a/lib/api/internal/shellhorse.rb b/lib/api/internal/shellhorse.rb new file mode 100644 index 00000000000..89210c8a78a --- /dev/null +++ b/lib/api/internal/shellhorse.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module API + module Internal + class Shellhorse < ::API::Base + before { authenticate_by_gitlab_shell_or_workhorse_token! } + + helpers ::API::Helpers::InternalHelpers + + COMMANDS_TO_AUDIT = %w[git-upload-pack git-receive-pack].freeze + + helpers do + def check_clone_or_pull_or_push_verb(params) + return 'push' if params[:action] == 'git-receive-pack' + + # we must set the default value for wants/haves because + # gitlab shell/workhorse will trim the whole posted params + # json key if its value is 0 + wants = haves = 0 + if params.key?(:packfile_stats) + wants = Integer(params[:packfile_stats][:wants]) if params[:packfile_stats][:wants].present? + haves = Integer(params[:packfile_stats][:haves]) if params[:packfile_stats][:haves].present? + end + + wants > 0 && haves == 0 ? 'clone' : 'pull' + end + end + + namespace 'internal' do + namespace 'shellhorse' do + params do + requires :action, type: String + requires :protocol, type: String + requires :gl_repository, type: String # repository identifier, such as project-7 + optional :packfile_stats, type: Hash do + # wants is the number of objects the client announced it wants. + optional :wants, type: Integer + # haves is the number of objects the client announced it has. + optional :haves, type: Integer + end + end + + post '/git_audit_event', feature_category: :source_code_management do + unless COMMANDS_TO_AUDIT.include?(params[:action]) + break response_with_status(code: 400, success: false, message: "No valid action specified") + end + + check_result = access_check_result + break check_result if unsuccessful_response?(check_result) + + unless need_git_audit_event? + break response_with_status(code: 200, success: false, message: "No git audit event needed") + end + + unless check_result.is_a?(::Gitlab::GitAccessResult::Success) + break response_with_status(code: 500, success: false, + message: ::API::Helpers::InternalHelpers::UNKNOWN_CHECK_RESULT_ERROR) + end + + msg = { + protocol: params[:protocol], + action: params[:action], + verb: check_clone_or_pull_or_push_verb(params) + } + send_git_audit_streaming_event(msg) + response_with_status(message: msg) + end + end + end + end + end +end + +API::Internal::Shellhorse.prepend_mod_with('API::Internal::Shellhorse') diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 34f9538b047..d625b2c0fe6 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -10,6 +10,12 @@ module API helpers ::API::Helpers::MembersHelpers + helpers do + params :invitation_params_ee do + # Overriden in EE + end + end + %w[group project].each do |source_type| params do requires :id, type: String, desc: "The #{source_type} ID" @@ -26,6 +32,8 @@ module API optional :user_id, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api' + + use :invitation_params_ee end post ":id/invitations", urgency: :low do ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/354016') @@ -34,11 +42,7 @@ module API source = find_source(source_type, params[:id]) - if ::Feature.enabled?(:admin_group_member, source) - authorize_admin_source_member!(source_type, source) - else - authorize_admin_source!(source_type, source) - end + authorize_admin_source_member!(source_type, source) create_service_params = params.merge(source: source) @@ -61,11 +65,7 @@ module API source = find_source(source_type, params[:id]) query = params[:query] - if ::Feature.enabled?(:admin_group_member, source) - authorize_admin_source_member!(source_type, source) - else - authorize_admin_source!(source_type, source) - end + authorize_admin_source_member!(source_type, source) invitations = paginate(retrieve_member_invitations(source, query)) @@ -80,16 +80,14 @@ module API requires :email, type: String, desc: 'The email address of the invitation' optional :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' optional :expires_at, type: DateTime, desc: 'Date string in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`)' + + use :invitation_params_ee end put ":id/invitations/:email", requirements: { email: %r{[^/]+} } do source = find_source(source_type, params.delete(:id)) invite_email = params[:email] - if ::Feature.enabled?(:admin_group_member, source) - authorize_admin_source_member!(source_type, source) - else - authorize_admin_source!(source_type, source) - end + authorize_admin_source_member!(source_type, source) invite = retrieve_member_invitations(source, invite_email).first not_found! unless invite @@ -127,11 +125,7 @@ module API source = find_source(source_type, params[:id]) invite_email = params[:email] - if ::Feature.enabled?(:admin_group_member, source) - authorize_admin_source_member!(source_type, source) - else - authorize_admin_source!(source_type, source) - end + authorize_admin_source_member!(source_type, source) invite = retrieve_member_invitations(source, invite_email).first not_found! unless invite @@ -145,3 +139,5 @@ module API end end end + +API::Members.prepend_mod diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 26619e6924f..b2f0f54e380 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -29,7 +29,7 @@ module API not_found! 'Commit' unless user_project.commit(sha).present? - content = user_project.repository.gitlab_ci_yml_for(sha, user_project.ci_config_path_or_default) + content = user_project.repository.blob_data_at(sha, user_project.ci_config_path_or_default) result = Gitlab::Ci::Lint .new(project: user_project, current_user: current_user, sha: sha) .validate(content, dry_run: params[:dry_run], ref: params[:ref] || user_project.default_branch) diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 517de98a148..14c3fccee32 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -228,7 +228,7 @@ module API requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.maven_file_name_regex, documentation: { example: 'mypkg-1.0-SNAPSHOT.pom' } end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true put ':id/packages/maven/*path/:file_name/authorize', requirements: MAVEN_ENDPOINT_REQUIREMENTS do authorize_upload! @@ -254,7 +254,7 @@ module API requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.maven_file_name_regex, documentation: { example: 'mypkg-1.0-SNAPSHOT.pom' } requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, basic_auth_personal_access_token: true put ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do unprocessable_entity! if Gitlab::FIPS.enabled? && params[:file].md5 authorize_upload! diff --git a/lib/api/members.rb b/lib/api/members.rb index bdbdea70da0..56a15c41e1c 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -118,11 +118,8 @@ module API post ":id/members", feature_category: feature_category do source = find_source(source_type, params[:id]) - if ::Feature.enabled?(:admin_group_member, source) - authorize_admin_source_member!(source_type, source) - else - authorize_admin_source!(source_type, source) - end + + authorize_admin_source_member!(source_type, source) create_service_params = params.merge(source: source) @@ -148,11 +145,7 @@ module API source = find_source(source_type, params.delete(:id)) member = source_members(source).find_by!(user_id: params[:user_id]) - if ::Feature.enabled?(:admin_group_member, source) - authorize_update_source_member!(source_type, member) - else - authorize_admin_source!(source_type, source) - end + authorize_update_source_member!(source_type, member) result = ::Members::UpdateService .new(current_user, declared_params(include_missing: false)) diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb index 35fdcfe3ab0..d0c9400039a 100644 --- a/lib/api/merge_request_approvals.rb +++ b/lib/api/merge_request_approvals.rb @@ -86,6 +86,10 @@ module API not_found! unless success + ::MergeRequests::UpdateReviewerStateService + .new(project: user_project, current_user: current_user) + .execute(merge_request, "unreviewed") + present_approval(merge_request) end diff --git a/lib/api/ml/mlflow/api_helpers.rb b/lib/api/ml/mlflow/api_helpers.rb index 19ac0dbba1b..aefa156717c 100644 --- a/lib/api/ml/mlflow/api_helpers.rb +++ b/lib/api/ml/mlflow/api_helpers.rb @@ -12,6 +12,10 @@ module API unauthorized! unless can?(current_user, :write_model_experiments, user_project) end + def check_api_model_registry_read! + not_found! unless can?(current_user, :read_model_registry, user_project) + end + def resource_not_found! render_structured_api_error!({ error_code: 'RESOURCE_DOES_NOT_EXIST' }, 404) end @@ -79,6 +83,10 @@ module API candidate_repository.by_eid(eid) || resource_not_found! end + def find_model(project, name) + ::Ml::FindModelService.new(project, name).execute || resource_not_found! + end + def packages_url path = api_v4_projects_packages_generic_package_version_path( id: user_project.id, package_name: '', file_name: '' diff --git a/lib/api/ml/mlflow/entrypoint.rb b/lib/api/ml/mlflow/entrypoint.rb index 3e0cb723580..7157d2a03f6 100644 --- a/lib/api/ml/mlflow/entrypoint.rb +++ b/lib/api/ml/mlflow/entrypoint.rb @@ -26,9 +26,6 @@ module API status 200 authenticate! - - check_api_read! - check_api_write! unless request.get? || request.head? end rescue_from ActiveRecord::ActiveRecordError do |e| @@ -44,7 +41,9 @@ module API end namespace MLFLOW_API_PREFIX do mount ::API::Ml::Mlflow::Experiments + mount ::API::Ml::Mlflow::ModelVersions mount ::API::Ml::Mlflow::Runs + mount ::API::Ml::Mlflow::RegisteredModels end end end diff --git a/lib/api/ml/mlflow/experiments.rb b/lib/api/ml/mlflow/experiments.rb index 614112f703b..1a501291941 100644 --- a/lib/api/ml/mlflow/experiments.rb +++ b/lib/api/ml/mlflow/experiments.rb @@ -9,6 +9,11 @@ module API class Experiments < ::API::Base feature_category :mlops + before do + check_api_read! + check_api_write! unless request.get? || request.head? + end + resource :experiments do desc 'Fetch experiment by experiment_id' do success Entities::Ml::Mlflow::GetExperiment diff --git a/lib/api/ml/mlflow/model_versions.rb b/lib/api/ml/mlflow/model_versions.rb new file mode 100644 index 00000000000..989b79e5774 --- /dev/null +++ b/lib/api/ml/mlflow/model_versions.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module API + module Ml + module Mlflow + class ModelVersions < ::API::Base + feature_category :mlops + + resource :model_versions do + desc 'Fetch model version by name and version' do + success Entities::Ml::Mlflow::ModelVersions::Responses::Get + detail 'https://mlflow.org/docs/2.6.0/rest-api.html#get-modelversion' + end + params do + requires :name, type: String, desc: 'Model version name' + requires :version, type: String, desc: 'Model version number' + end + get 'get', urgency: :low do + check_api_model_registry_read! + resource_not_found! unless params[:name] && params[:version] + model_version = ::Ml::ModelVersions::GetModelVersionService.new( + user_project, params[:name], params[:version] + ).execute + resource_not_found! unless model_version + response = { model_version: model_version } + present response, with: Entities::Ml::Mlflow::ModelVersions::Responses::Get + end + end + end + end + end +end diff --git a/lib/api/ml/mlflow/registered_models.rb b/lib/api/ml/mlflow/registered_models.rb new file mode 100644 index 00000000000..18b705ad214 --- /dev/null +++ b/lib/api/ml/mlflow/registered_models.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'mime/types' + +module API + # MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api + module Ml + module Mlflow + class RegisteredModels < ::API::Base + feature_category :mlops + + before do + check_api_read! + check_api_write! unless request.get? || request.head? + check_api_model_registry_read! + end + + resource 'registered-models' do + desc 'Creates a Registered Model.' do + success Entities::Ml::Mlflow::RegisteredModel + detail 'MLFlow Registered Models map to GitLab Models. https://mlflow.org/docs/2.6.0/rest-api.html#create-registeredmodel' + end + params do + requires :name, type: String, + desc: 'Register models under this name.' + optional :description, type: String, + desc: 'Optional description for registered model.' + optional :tags, type: Array, desc: 'Additional metadata for registered model.' + end + post 'create', urgency: :low do + present ::Ml::CreateModelService.new( + user_project, + params[:name], + current_user, + params[:description], + params[:tags] + ).execute, + with: Entities::Ml::Mlflow::RegisteredModel, + root: :registered_model + rescue ActiveRecord::RecordInvalid + resource_already_exists! + end + + desc 'Fetch a Registered Model by Name' do + success Entities::Ml::Mlflow::RegisteredModel + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-registeredmodel' + end + params do + # The name param is actually required, however it is listed as optional here + # we can send a custom error response required by MLFlow + optional :name, type: String, default: '', + desc: 'Registered model unique name identifier, in reference to the project' + end + get 'get', urgency: :low do + present find_model(user_project, params[:name]), with: Entities::Ml::Mlflow::RegisteredModel, + root: :registered_model + end + + desc 'Update a Registered Model by Name' do + success Entities::Ml::Mlflow::RegisteredModel + detail 'https://mlflow.org/docs/2.6.0/rest-api.html#update-registeredmodel' + end + params do + # The name param is actually required, however it is listed as optional here + # we can send a custom error response required by MLFlow + optional :name, type: String, + desc: 'Registered model unique name identifier, in reference to the project' + optional :description, type: String, + desc: 'Optional description for registered model.' + end + patch 'update', urgency: :low do + present ::Ml::UpdateModelService.new(find_model(user_project, params[:name]), params[:description]).execute, + with: Entities::Ml::Mlflow::RegisteredModel, root: :registered_model + end + + desc 'Fetch the latest Model Version for the given Registered Model Name' do + success Entities::Ml::Mlflow::ModelVersions::Types::ModelVersion + detail 'https://mlflow.org/docs/2.6.0/rest-api.html#get-latest-modelversions' + end + params do + # The name param is actually required, however it is listed as optional here + # we can send a custom error response required by MLFlow + optional :name, type: String, + desc: 'Registered model unique name identifier, in reference to the project' + end + post 'get-latest-versions', urgency: :low do + model = find_model(user_project, params[:name]) + + present [model.latest_version], with: Entities::Ml::Mlflow::ModelVersions::Types::ModelVersion, + root: :model_versions + end + end + end + end + end +end diff --git a/lib/api/ml/mlflow/runs.rb b/lib/api/ml/mlflow/runs.rb index ac052d8bff5..6716db21407 100644 --- a/lib/api/ml/mlflow/runs.rb +++ b/lib/api/ml/mlflow/runs.rb @@ -9,6 +9,11 @@ module API class Runs < ::API::Base feature_category :mlops + before do + check_api_read! + check_api_write! unless request.get? || request.head? + end + resource :runs do desc 'Creates a Run.' do success Entities::Ml::Mlflow::Run diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 46b388a2fda..b061876b997 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -102,7 +102,7 @@ module API end def check_duplicate(file_params, symbol_package) - return if symbol_package || Feature.disabled?(:nuget_duplicates_option, project_or_group.namespace) + return if symbol_package service_params = file_params.merge(remote_url: params['package.remote_url']) response = ::Packages::Nuget::CheckDuplicatesService.new(project_or_group, current_user, service_params).execute diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb index 9d234ca0593..de00b66ead3 100644 --- a/lib/api/personal_access_tokens.rb +++ b/lib/api/personal_access_tokens.rb @@ -72,11 +72,17 @@ module API detail 'Roates a personal access token.' success Entities::PersonalAccessTokenWithToken end + params do + optional :expires_at, + type: Date, + desc: "The expiration date of the token", + documentation: { example: '2021-01-31' } + end post ':id/rotate' do token = PersonalAccessToken.find_by_id(params[:id]) if Ability.allowed?(current_user, :manage_user_personal_access_token, token&.user) - response = ::PersonalAccessTokens::RotateService.new(current_user, token).execute + response = ::PersonalAccessTokens::RotateService.new(current_user, token).execute(declared_params) if response.success? status :ok diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 6b2ba41f013..7f531525870 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -46,6 +46,8 @@ module API desc: 'Return packages of a certain type' optional :package_name, type: String, desc: 'Return packages with this name' + optional :package_version, type: String, + desc: 'Return packages with this version' optional :include_versionless, type: Boolean, desc: 'Returns packages without a version' optional :status, type: String, values: Packages::Package.statuses.keys, @@ -55,7 +57,7 @@ module API get ':id/packages' do packages = ::Packages::PackagesFinder.new( user_project, - declared_params.slice(:order_by, :sort, :package_type, :package_name, :include_versionless, :status) + declared_params.slice(:order_by, :sort, :package_type, :package_name, :package_version, :include_versionless, :status) ).execute present paginate(packages), with: ::API::Entities::Package, user: current_user, namespace: user_project.namespace diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb index 5777b8754e7..b79348c87bf 100644 --- a/lib/api/project_repository_storage_moves.rb +++ b/lib/api/project_repository_storage_moves.rb @@ -8,6 +8,16 @@ module API feature_category :gitaly + helpers do + extend ::Gitlab::Utils::Override + + # Allow to move projects in hidden/pending_delete state + override :find_project_scopes + def find_project_scopes + Project + end + end + resource :project_repository_storage_moves do desc 'Get a list of all project repository storage moves' do detail 'This feature was introduced in GitLab 13.0.' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ac28effea43..3b80fd125ca 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -159,6 +159,7 @@ module API optional :topic_id, type: Integer, desc: 'Limit results to projects with the assigned topic given by the topic ID' optional :updated_before, type: DateTime, desc: 'Return projects updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :updated_after, type: DateTime, desc: 'Return projects updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' + optional :include_pending_delete, type: Boolean, desc: 'Include projects in pending delete state. Can only be set by admins' use :optional_filter_params_ee end @@ -470,6 +471,7 @@ module API optional :description, type: String, desc: 'The description that will be assigned to the fork', documentation: { example: 'Description' } optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork' optional :mr_default_target_self, type: Boolean, desc: 'Merge requests of this forked project targets itself by default' + optional :branches, type: String, desc: 'Branches to fork' end post ':id/fork', feature_category: :source_code_management do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20759') @@ -489,6 +491,7 @@ module API service = ::Projects::ForkService.new(user_project, current_user, fork_params) + not_found!('Source Branch') if fork_params[:branches].present? && !service.valid_fork_branch?(fork_params[:branches]) not_found!('Target Namespace') unless service.valid_fork_target? forked_project = service.execute @@ -792,7 +795,12 @@ module API not_found!('Group Link') unless link destroy_conditionally!(link) do - ::Projects::GroupLinks::DestroyService.new(user_project, current_user).execute(link) + result = ::Projects::GroupLinks::DestroyService.new(user_project, current_user).execute(link) + + if result.error? + status = :not_found if result.reason == :not_found + render_api_error!(result.message, status) + end end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 027a11738d3..3313b3a87cd 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -280,6 +280,13 @@ module API optional :requires_python, type: String, documentation: { example: '>=3.7' } optional :md5_digest, type: String, documentation: { example: '900150983cd24fb0d6963f7d28e17f72' } optional :sha256_digest, type: String, regexp: Gitlab::Regex.sha256_regex, documentation: { example: 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad' } + optional :metadata_version, type: String, documentation: { example: '2.3' } + optional :author_email, type: String, documentation: { example: 'cschultz@example.com, snoopy@peanuts.com' } + optional :description, type: String + optional :description_content_type, type: String, + documentation: { example: 'text/markdown; charset=UTF-8; variant=GFM' } + optional :summary, type: String, documentation: { example: 'A module for collecting votes from beagles.' } + optional :keywords, type: String, documentation: { example: 'dog,puppy,voting,election' } end route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 5d056ade3da..83085b5b7e3 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -270,8 +270,6 @@ module API .execute if result[:status] == :success - log_release_created_audit_event(result[:release]) - present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) @@ -317,9 +315,6 @@ module API .execute if result[:status] == :success - log_release_updated_audit_event - log_release_milestones_updated_audit_event if result[:milestones_updated] - present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) @@ -350,8 +345,6 @@ module API .execute if result[:status] == :success - log_release_deleted_audit_event - present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) @@ -406,22 +399,6 @@ module API Rack::Utils.parse_nested_query(@request.query_string) end - def log_release_created_audit_event(release) - # extended in EE - end - - def log_release_updated_audit_event - # extended in EE - end - - def log_release_deleted_audit_event - # extended in EE - end - - def log_release_milestones_updated_audit_event - # extended in EE - end - def release_cli? request.env['HTTP_USER_AGENT']&.include?(RELEASE_CLI_USER_AGENT) == true end diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb index 1ad5bc8d421..752feb1455f 100644 --- a/lib/api/resource_access_tokens.rb +++ b/lib/api/resource_access_tokens.rb @@ -141,6 +141,10 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" requires :token_id, type: String, desc: "The ID of the token" + optional :expires_at, + type: Date, + desc: "The expiration date of the token", + documentation: { example: '2021-01-31' } end post ':id/access_tokens/:token_id/rotate' do resource = find_source(source_type, params[:id]) @@ -149,7 +153,7 @@ module API token = find_token(resource, params[:token_id]) if resource_accessible if token - response = ::PersonalAccessTokens::RotateService.new(current_user, token).execute + response = ::PersonalAccessTokens::RotateService.new(current_user, token).execute(declared_params) if response.success? status :ok diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 9120421fadf..7ad4ecd88b1 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -204,6 +204,7 @@ module API optional :floc_enabled, type: Grape::API::Boolean, desc: 'Enable FloC (Federated Learning of Cohorts)' optional :user_deactivation_emails_enabled, type: Boolean, desc: 'Send emails to users upon account deactivation' optional :suggest_pipeline_enabled, type: Boolean, desc: 'Enable pipeline suggestion banner' + optional :enable_artifact_external_redirect_warning_page, type: Boolean, desc: 'Show the external redirect page that warns you about user-generated content in GitLab Pages' optional :users_get_by_id_limit, type: Integer, desc: "Maximum number of calls to the /users/:id API per 10 minutes per user. Set to 0 for unlimited requests." optional :runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for shared runners, in seconds' optional :group_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for group runners, in seconds' diff --git a/lib/api/users.rb b/lib/api/users.rb index dd9cb2ee019..5fa6d50581b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -34,10 +34,14 @@ module API helpers do # rubocop: disable CodeReuse/ActiveRecord def reorder_users(users) - if params[:order_by] && params[:sort] - users.reorder(order_options_with_tie_breaker) - else + # Users#search orders by exact matches and handles pagination, + # so we should prioritize that. + if params[:search] users + else + # Note that params[:order_by] and params[:sort] will always be present and + # default to "id" and "desc" as defined in `sort_params`. + users.reorder(order_options_with_tie_breaker) end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb deleted file mode 100644 index 0ce5cdd06de..00000000000 --- a/lib/api/v3/github.rb +++ /dev/null @@ -1,289 +0,0 @@ -# frozen_string_literal: true - -# The endpoints by default return `404` in preparation for their removal -# (also see comment above `#reversible_end_of_life!`). -# https://gitlab.com/gitlab-org/gitlab/-/issues/362168 -# -# These endpoints partially mimic Github API behavior in order to successfully -# integrate with Jira Development Panel. -module API - module V3 - class Github < ::API::Base - NO_SLASH_URL_PART_REGEX = %r{[^/]+} - ENDPOINT_REQUIREMENTS = { - namespace: NO_SLASH_URL_PART_REGEX, - project: NO_SLASH_URL_PART_REGEX, - username: NO_SLASH_URL_PART_REGEX - }.freeze - - # Used to differentiate Jira Cloud requests from Jira Server requests - # Jira Cloud user agent format: Jira DVCS Connector Vertigo/version - # Jira Server user agent format: Jira DVCS Connector/version - JIRA_DVCS_CLOUD_USER_AGENT = 'Jira DVCS Connector Vertigo' - - GITALY_TIMEOUT_CACHE_KEY = 'api:v3:Gitaly-timeout-cache-key' - GITALY_TIMEOUT_CACHE_EXPIRY = 1.day - - include PaginationParams - - feature_category :integrations - - before do - authorize_jira_user_agent!(request) - authenticate! - reversible_end_of_life! - end - - helpers do - params :project_full_path do - requires :namespace, type: String - requires :project, type: String - end - - # The endpoints in this class have been deprecated since 15.1. - # - # Due to uncertainty about the impact of a full removal in 16.0, all endpoints return `404` - # by default but we allow customers to toggle a flag to reverse this breaking change. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/362168#note_1347692683. - # - # TODO Make the breaking change irreversible https://gitlab.com/gitlab-org/gitlab/-/issues/408148. - def reversible_end_of_life! - not_found! unless Feature.enabled?(:jira_dvcs_end_of_life_amnesty) - - Gitlab::IntegrationsLogger.info( - user_id: current_user&.id, - namespace: params[:namespace], - project: params[:project], - message: 'Deprecated Jira DVCS endpoint request' - ) - end - - def authorize_jira_user_agent!(request) - not_found! unless Gitlab::Jira::Middleware.jira_dvcs_connector?(request.env) - end - - def update_project_feature_usage_for(project) - # Prevent errors on GitLab Geo not allowing - # UPDATE statements to happen in GET requests. - return if Gitlab::Database.read_only? - - project.log_jira_dvcs_integration_usage(cloud: jira_cloud?) - end - - def jira_cloud? - request.env['HTTP_USER_AGENT'].include?(JIRA_DVCS_CLOUD_USER_AGENT) - end - - def find_project_with_access(params) - project = find_project!( - ::Gitlab::Jira::Dvcs.restore_full_path(**params.slice(:namespace, :project).symbolize_keys) - ) - not_found! unless can?(current_user, :read_code, project) - project - end - - # rubocop: disable CodeReuse/ActiveRecord - def find_merge_requests - merge_requests = authorized_merge_requests.reorder(updated_at: :desc) - paginate(merge_requests) - end - # rubocop: enable CodeReuse/ActiveRecord - - # rubocop: disable CodeReuse/ActiveRecord - def find_merge_request_with_access(id, access_level = :read_merge_request) - merge_request = authorized_merge_requests.find_by(id: id) - not_found! unless can?(current_user, access_level, merge_request) - merge_request - end - # rubocop: enable CodeReuse/ActiveRecord - - def authorized_merge_requests - MergeRequestsFinder.new(current_user, authorized_only: !current_user.can_read_all_resources?) - .execute.with_jira_integration_associations - end - - def authorized_merge_requests_for_project(project) - MergeRequestsFinder - .new(current_user, authorized_only: !current_user.can_read_all_resources?, project_id: project.id) - .execute.with_jira_integration_associations - end - - # rubocop: disable CodeReuse/ActiveRecord - def find_notes(noteable) - # They're not presented on Jira Dev Panel ATM. A comments count with a - # redirect link is presented. - notes = paginate(noteable.notes.user.reorder(nil)) - notes.select { |n| n.readable_by?(current_user) } - end - # rubocop: enable CodeReuse/ActiveRecord - - # Returns an empty Array instead of the Commit diff files for a period - # of time after a Gitaly timeout, to mitigate frequent Gitaly timeouts - # for some Commit diffs. - def diff_files(commit) - cache_key = [ - GITALY_TIMEOUT_CACHE_KEY, - commit.project.id, - commit.cache_key - ].join(':') - - return [] if Rails.cache.read(cache_key).present? - - begin - commit.diffs.diff_files - rescue GRPC::DeadlineExceeded => error - # Gitaly fails to load diffs consistently for some commits. The other information - # is still valuable for Jira. So we skip the loading and respond with a 200 excluding diffs - # Remove this when https://gitlab.com/gitlab-org/gitaly/-/issues/3741 is fixed. - Rails.cache.write(cache_key, 1, expires_in: GITALY_TIMEOUT_CACHE_EXPIRY) - Gitlab::ErrorTracking.track_exception(error) - [] - end - end - end - - resource :orgs do - get ':namespace/repos' do - present [] - end - end - - resource :user do - get :repos do - present [] - end - end - - resource :users do - params do - use :pagination - end - - get ':namespace/repos' do - namespace = Namespace.find_by_full_path(params[:namespace]) - not_found!('Namespace') unless namespace - - projects = current_user.can_read_all_resources? ? Project.all : current_user.authorized_projects - projects = projects.in_namespace(namespace.self_and_descendants) - - projects_cte = Project.wrap_with_cte(projects) - .eager_load_namespace_and_owner - .with_route - - present paginate(projects_cte), - with: ::API::Github::Entities::Repository, - root_namespace: namespace.root_ancestor - end - - get ':username' do - forbidden! unless can?(current_user, :read_users_list) - user = UsersFinder.new(current_user, { username: params[:username] }).execute.first - not_found! unless user - present user, with: ::API::Github::Entities::User - end - end - - # Jira dev panel integration weirdly requests for "/-/jira/pulls" instead - # "/api/v3/repos/<namespace>/<project>/pulls". This forces us into - # returning _all_ Merge Requests from authorized projects (user is a member), - # instead just the authorized MRs from a project. - # Jira handles the filtering, presenting just MRs mentioning the Jira - # issue ID on the MR title / description. - resource :repos do - # Keeping for backwards compatibility with old Jira integration instructions - # so that users that do not change it will not suddenly have a broken integration - get '/-/jira/pulls' do - present find_merge_requests, with: ::API::Github::Entities::PullRequest - end - - get '/-/jira/events' do - present [] - end - - params do - use :project_full_path - end - # TODO Remove the custom Apdex SLO target `urgency: :low` when this endpoint has been optimised. - # https://gitlab.com/gitlab-org/gitlab/-/issues/337269 - get ':namespace/:project/pulls', urgency: :low do - user_project = find_project_with_access(params) - - merge_requests = authorized_merge_requests_for_project(user_project) - - present paginate(merge_requests), with: ::API::Github::Entities::PullRequest - end - - params do - use :project_full_path - end - get ':namespace/:project/pulls/:id' do - merge_request = find_merge_request_with_access(params[:id]) - - present merge_request, with: ::API::Github::Entities::PullRequest - end - - # In Github, each Merge Request is automatically also an issue. - # Therefore we return its comments here. - # It'll present _just_ the comments counting with a link to GitLab on - # Jira dev panel, not the actual note content. - get ':namespace/:project/issues/:id/comments' do - merge_request = find_merge_request_with_access(params[:id]) - - present find_notes(merge_request), with: ::API::Github::Entities::NoteableComment - end - - # This refer to "review" comments but Jira dev panel doesn't seem to - # present it accordingly. - get ':namespace/:project/pulls/:id/comments' do - present [] - end - - # Commits are not presented within "Pull Requests" modal on Jira dev - # panel. - get ':namespace/:project/pulls/:id/commits' do - present [] - end - - # Self-hosted Jira (tested on 7.11.1) requests this endpoint right - # after fetching branches. - get ':namespace/:project/events' do - user_project = find_project_with_access(params) - - merge_requests = authorized_merge_requests_for_project(user_project) - - present paginate(merge_requests), with: ::API::Github::Entities::PullRequestEvent - end - - params do - use :project_full_path - use :pagination - end - # TODO Remove the custom Apdex SLO target `urgency: :low` when this endpoint has been optimised. - # https://gitlab.com/gitlab-org/gitlab/-/issues/337268 - get ':namespace/:project/branches', urgency: :low do - user_project = find_project_with_access(params) - - update_project_feature_usage_for(user_project) - - next [] unless user_project.repo_exists? - - branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name)) - - present paginate(branches), with: ::API::Github::Entities::Branch, project: user_project - end - - params do - use :project_full_path - end - get ':namespace/:project/commits/:sha' do - user_project = find_project_with_access(params) - - commit = user_project.commit(params[:sha]) - not_found! 'Commit' unless commit - - present commit, with: ::API::Github::Entities::RepoCommit, diff_files: diff_files(commit) - end - end - end - end -end diff --git a/lib/api/vs_code/settings/entities/vs_code_setting_reference.rb b/lib/api/vs_code/settings/entities/vs_code_setting_reference.rb new file mode 100644 index 00000000000..38af85dc0c7 --- /dev/null +++ b/lib/api/vs_code/settings/entities/vs_code_setting_reference.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module API + module VsCode + module Settings + module Entities + class VsCodeSettingReference < Grape::Entity + include ::API::Helpers::RelatedResourcesHelpers + + expose :url do |setting| + expose_path(api_v4_vscode_settings_sync_v1_resource_path( + resource_name: setting[:setting_type], + id: setting[:uuid] + )) + end + expose :created do |setting| + setting[:updated_at]&.to_i + end + end + end + end + end +end diff --git a/lib/api/vs_code/settings/vs_code_settings_sync.rb b/lib/api/vs_code/settings/vs_code_settings_sync.rb index dc22496e380..1e53125a3aa 100644 --- a/lib/api/vs_code/settings/vs_code_settings_sync.rb +++ b/lib/api/vs_code/settings/vs_code_settings_sync.rb @@ -8,6 +8,14 @@ module API feature_category :web_ide + helpers do + def find_settings + return [DEFAULT_MACHINE] if params[:resource_name] == DEFAULT_MACHINE[:setting_type] + + SettingsFinder.new(current_user, [params[:resource_name]]).execute + end + end + before do authenticate! @@ -21,6 +29,9 @@ module API desc 'Get the settings manifest for Settings Sync' do success [Entities::VsCodeManifest] + failure [ + { code: 401, message: '401 Unauthorized' } + ] tags %w[vscode] end get '/v1/manifest' do @@ -31,44 +42,71 @@ module API end desc 'Get a specific setting resource' do - success [Entities::VsCodeSetting] + success [ + Entities::VsCodeSetting, + { code: 204, message: 'No content' } + ] + failure [ + { code: 400, message: '400 bad request' }, + { code: 401, message: '401 Unauthorized' } + ] tags %w[vscode] end params do - requires :resource_name, type: String, desc: 'Name of the resource such as settings' + requires :resource_name, type: String, desc: 'Name of the resource such as settings', + values: SETTINGS_TYPES requires :id, type: String, desc: 'ID of the resource to retrieve' end get '/v1/resource/:resource_name/:id' do - authenticate! - - setting_name = params[:resource_name] - setting = nil + settings = find_settings - if params[:resource_name] == 'machines' - setting = DEFAULT_MACHINE - else - settings = SettingsFinder.new(current_user, [setting_name]).execute - setting = settings.first if settings.present? - end - - if setting.nil? + if settings.blank? status :no_content header :etag, NO_CONTENT_ETAG body false else + # This endpoint does not use the :id parameter + # because the first iteration of this API only + # supports storing a single record of a given setting_type. + # We can rely on obtaining the first record of the setting + # result. + setting = settings.first header :etag, setting[:uuid] presenter = VsCodeSettingPresenter.new setting present presenter, with: Entities::VsCodeSetting end end - desc 'Update a specific setting' + desc 'Get a list of references to one or more vscode setting resources' do + success [Entities::VsCodeSettingReference] + failure [ + { code: 400, message: '400 bad request' }, + { code: 401, message: '401 Unauthorized' } + ] + tags %w[vscode] + end params do - requires :resource_name, type: String, desc: 'Name of the resource such as settings' + requires :resource_name, type: String, desc: 'Name of the resource such as settings', + values: SETTINGS_TYPES end - post '/v1/resource/:resource_name' do - authenticate! + get '/v1/resource/:resource_name' do + settings = find_settings + present settings, with: Entities::VsCodeSettingReference + end + + desc 'Creates or updates a specific setting' do + success [{ code: 200, message: 'OK' }] + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: '401 Unauthorized' } + ] + end + params do + requires :resource_name, type: String, desc: 'Name of the resource such as settings', + values: SETTINGS_TYPES + end + post '/v1/resource/:resource_name' do response = CreateOrUpdateService.new(current_user: current_user, params: { content: params[:content], version: params[:version], @@ -83,6 +121,19 @@ module API error!(response.message, 400) end end + + desc 'Deletes all user vscode setting resources' do + success [{ code: 200, message: 'OK' }] + failure [ + { code: 401, message: '401 Unauthorized' } + ] + tags %w[vscode] + end + delete '/v1/collection' do + DeleteService.new(current_user: current_user).execute + + present "OK" + end end end end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 2058f5de706..a7408512102 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -85,6 +85,9 @@ module API end params do requires :title, type: String, desc: 'Title of a wiki page' + optional :front_matter, type: Hash do + optional :title, type: String, desc: 'Front matter title of a wiki page' + end requires :content, type: String, desc: 'Content of a wiki page' use :common_wiki_page_params end @@ -112,6 +115,9 @@ module API end params do optional :title, type: String, desc: 'Title of a wiki page' + optional :front_matter, type: Hash do + optional :title, type: String, desc: 'Front matter title of a wiki page' + end optional :content, type: String, desc: 'Content of a wiki page' use :common_wiki_page_params at_least_one_of :content, :title, :format diff --git a/lib/atlassian/jira_connect/jira_user.rb b/lib/atlassian/jira_connect/jira_user.rb index 57ceb8fdf13..051165474af 100644 --- a/lib/atlassian/jira_connect/jira_user.rb +++ b/lib/atlassian/jira_connect/jira_user.rb @@ -3,15 +3,17 @@ module Atlassian module JiraConnect class JiraUser + ADMIN_GROUPS = %w[site-admins org-admins].freeze + def initialize(data) @data = data end - def site_admin? + def jira_admin? groups = @data.dig('groups', 'items') return false unless groups - groups.any? { |g| g['name'] == 'site-admins' } + groups.any? { |group| ADMIN_GROUPS.include?(group['name']) } end end end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index 5b55c2cbdf7..366151a63b4 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -92,7 +92,7 @@ module Backup args += ['-id', backup_id] if backup_id when :restore args += ['-remove-all-repositories', remove_all_repositories.join(',')] if remove_all_repositories - args += ['-id', backup_id] if backup_id && server_side? + args += ['-id', backup_id] if backup_id end args diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb index 512c55381ec..eae69700465 100644 --- a/lib/banzai/filter/asset_proxy_filter.rb +++ b/lib/banzai/filter/asset_proxy_filter.rb @@ -23,7 +23,8 @@ module Banzai begin uri = URI.parse(original_src) - next if uri.host.nil? && !original_src.start_with?('///') + # Skip URLs like `/path.ext` or `path.ext` which are relative to the current host + next if uri.relative? && uri.host.nil? && original_src.match(%r{\A/*})[0].length < 2 next if asset_host_allowed?(uri.host) rescue StandardError # Ignored diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 3161e030194..511da4b6ba5 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -93,10 +93,20 @@ module Banzai end def render_nodes_limit_reached?(count) + return false if wiki? + return false if blob? return false unless settings.math_rendering_limits_enabled? count >= RENDER_NODES_LIMIT end + + def wiki? + context[:wiki].present? + end + + def blob? + context[:text_source] == :blob + end end end end diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb index a3784004087..3fcb36c4714 100644 --- a/lib/banzai/filter/references/user_reference_filter.rb +++ b/lib/banzai/filter/references/user_reference_filter.rb @@ -65,13 +65,10 @@ module Banzai # The keys of this Hash are the namespace paths, the values the # corresponding Namespace objects. def namespaces - cross_join_issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/417466" - Gitlab::Database.allow_cross_joins_across_databases(url: cross_join_issue) do - @namespaces ||= Namespace.eager_load(:owner, :route) - .where_full_path_in(usernames) - .index_by(&:full_path) - .transform_keys(&:downcase) - end + @namespaces ||= Namespace.preload(:owner, :route) + .where_full_path_in(usernames) + .index_by(&:full_path) + .transform_keys(&:downcase) end # Returns all usernames referenced in the current document. diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index ec96181e7f1..bba5a7dfd09 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -6,9 +6,9 @@ module Banzai self.reference_type = :user def referenced_by(nodes, options = {}) - group_ids = [] - user_ids = [] - project_ids = [] + group_ids = Set.new + user_ids = Set.new + project_ids = Set.new nodes.each do |node| if node.has_attribute?('data-group') @@ -20,8 +20,10 @@ module Banzai end end - find_users_for_groups(group_ids) | find_users(user_ids) | - find_users_for_projects(project_ids) + user_ids += find_user_ids_for_groups(group_ids) + user_ids += find_user_ids_for_projects(project_ids) + + find_users(user_ids) end def nodes_visible_to_user(user, nodes) @@ -49,20 +51,6 @@ module Banzai visible + super(current_user, remaining) end - # Check if project belongs to a group which - # user can read. - def can_read_group_reference?(node, user, groups) - node_group = groups[node] - - node_group && can?(user, :read_group, node_group) - end - - def can_read_project_reference?(node) - node_id = node.attr('data-project').to_i - - project_for_node(node)&.id == node_id - end - def nodes_user_can_reference(current_user, nodes) project_attr = 'data-project' author_attr = 'data-author' @@ -88,28 +76,44 @@ module Banzai end end + private + + # Check if project belongs to a group which + # user can read. + def can_read_group_reference?(node, user, groups) + node_group = groups[node] + + node_group && can?(user, :read_group, node_group) + end + + def can_read_project_reference?(node) + node_id = node.attr('data-project').to_i + + project_for_node(node)&.id == node_id + end + def find_users(ids) return [] if ids.empty? collection_objects_for_ids(User, ids) end - def find_users_for_groups(ids) - return [] if ids.empty? + def find_user_ids_for_groups(group_ids) + return [] if group_ids.empty? - cross_join_issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/417466" - ::Gitlab::Database.allow_cross_joins_across_databases(url: cross_join_issue) do - User.joins(:group_members).where(members: { - source_id: Namespace.where(id: ids).where('mentions_disabled IS NOT TRUE').select(:id) - }).to_a - end + GroupMember + .of_groups(Group.id_in(group_ids).where('mentions_disabled IS NOT TRUE')) + .non_request + .non_invite + .non_minimal_access + .distinct + .pluck(:user_id) end - def find_users_for_projects(ids) - return [] if ids.empty? + def find_user_ids_for_projects(project_ids) + return [] if project_ids.empty? - collection_objects_for_ids(Project, ids) - .flat_map { |p| p.team.members.to_a } + ProjectAuthorization.for_project(project_ids).pluck(:user_id) end def can_read_reference?(user, ref_project, node) diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index 34dbf9ad22d..11ce0c26677 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -31,6 +31,10 @@ module Bitbucket raw.key?('parent') end + def deleted? + raw.fetch('deleted', false) + end + private def inline diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb index 8d5b15e299a..3764d116a36 100644 --- a/lib/bitbucket/representation/repo.rb +++ b/lib/bitbucket/representation/repo.rb @@ -59,6 +59,10 @@ module Bitbucket end end + def default_branch + raw.dig('mainbranch', 'name') + end + def to_s full_name end diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb index 94bbdfaa681..3055c8d24ce 100644 --- a/lib/bulk_imports/clients/graphql.rb +++ b/lib/bulk_imports/clients/graphql.rb @@ -4,11 +4,14 @@ module BulkImports module Clients class Graphql class HTTP < Graphlient::Adapters::HTTP::Adapter + REQUEST_TIMEOUT = 60 + def execute(document:, operation_name: nil, variables: {}, context: {}) response = ::Gitlab::HTTP.post( url, headers: headers, follow_redirects: false, + timeout: REQUEST_TIMEOUT, body: { query: document.to_query_string, operationName: operation_name, diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index fa09f36fdd6..723359aa438 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -30,8 +30,7 @@ module BulkImports source_full_path: entity.source_full_path, pipeline_class: self.class.name, message: "Entity #{entity.status_name}", - source_version: entity.bulk_import.source_version_info.to_s, - importer: 'gitlab_migration' + source_version: entity.bulk_import.source_version_info.to_s ) ::BulkImports::FinishProjectImportWorker.perform_async(entity.project_id) if entity.project? @@ -42,7 +41,7 @@ module BulkImports attr_reader :context, :entity, :trackers def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end def all_other_trackers_failed? diff --git a/lib/bulk_imports/logger.rb b/lib/bulk_imports/logger.rb new file mode 100644 index 00000000000..be15c050770 --- /dev/null +++ b/lib/bulk_imports/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module BulkImports + class Logger < ::Gitlab::Import::Logger + IMPORTER_NAME = 'gitlab_migration' + + def default_attributes + super.merge(importer: IMPORTER_NAME) + end + end +end diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb index 89ae66938af..07118c3b55c 100644 --- a/lib/bulk_imports/ndjson_pipeline.rb +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -135,7 +135,7 @@ module BulkImports bulk_import_entity_id: tracker.entity.id, pipeline_class: tracker.pipeline_name, exception_class: 'RecordInvalid', - exception_message: record.errors.full_messages.to_sentence.truncate(255), + exception_message: record.errors.full_messages.to_sentence, correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id ) end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 666916f8758..e2a14c35e79 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -17,7 +17,7 @@ module BulkImports if extracted_data extracted_data.each_with_index do |entry, index| raw_entry = entry.dup - next if Feature.enabled?(:bulk_import_idempotent_workers) && already_processed?(raw_entry, index) + next if already_processed?(raw_entry, index) transformers.each do |transformer| entry = run_pipeline_step(:transformer, transformer.class.name) do @@ -25,11 +25,11 @@ module BulkImports end end - run_pipeline_step(:loader, loader.class.name) do + run_pipeline_step(:loader, loader.class.name, entry) do loader.load(context, entry) end - save_processed_entry(raw_entry, index) if Feature.enabled?(:bulk_import_idempotent_workers) + save_processed_entry(raw_entry, index) end tracker.update!( @@ -40,6 +40,14 @@ module BulkImports run_pipeline_step(:after_run) do after_run(extracted_data) end + + # For batches, `#on_finish` is called once within `FinishBatchedPipelineWorker` + # after all batches have completed. + unless tracker.batched? + run_pipeline_step(:on_finish) do + on_finish + end + end end info(message: 'Pipeline finished') @@ -47,9 +55,11 @@ module BulkImports skip!('Skipping pipeline due to failed entity') end + def on_finish; end + private # rubocop:disable Lint/UselessAccessModifier - def run_pipeline_step(step, class_name = nil) + def run_pipeline_step(step, class_name = nil, entry = nil) raise MarkedAsFailedError if context.entity.failed? info(pipeline_step: step, step_class: class_name) @@ -65,11 +75,11 @@ module BulkImports rescue BulkImports::NetworkError => e raise BulkImports::RetryPipelineError.new(e.message, e.retry_delay) if e.retriable?(context.tracker) - log_and_fail(e, step) + log_and_fail(e, step, entry) rescue BulkImports::RetryPipelineError raise rescue StandardError => e - log_and_fail(e, step) + log_and_fail(e, step, entry) end def extracted_data_from @@ -95,8 +105,8 @@ module BulkImports run if extracted_data.has_next_page? end - def log_and_fail(exception, step) - log_import_failure(exception, step) + def log_and_fail(exception, step, entry = nil) + log_import_failure(exception, step, entry) if abort_on_failure? tracker.fail_op! @@ -114,16 +124,21 @@ module BulkImports tracker.skip! end - def log_import_failure(exception, step) + def log_import_failure(exception, step, entry) failure_attributes = { bulk_import_entity_id: context.entity.id, pipeline_class: pipeline, pipeline_step: step, exception_class: exception.class.to_s, - exception_message: exception.message.truncate(255), + exception_message: exception.message, correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id } + if entry + failure_attributes[:source_url] = BulkImports::SourceUrlBuilder.new(context, entry).url + failure_attributes[:source_title] = entry.try(:title) || entry.try(:name) + end + log_exception( exception, log_params( @@ -154,8 +169,7 @@ module BulkImports source_full_path: context.entity.source_full_path, pipeline_class: pipeline, context_extra: context.extra, - source_version: context.entity.bulk_import.source_version_info.to_s, - importer: 'gitlab_migration' + source_version: context.entity.bulk_import.source_version_info.to_s } defaults @@ -164,7 +178,7 @@ module BulkImports end def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end def log_exception(exception, payload) diff --git a/lib/bulk_imports/pipeline_schema_info.rb b/lib/bulk_imports/pipeline_schema_info.rb new file mode 100644 index 00000000000..df35a3569d6 --- /dev/null +++ b/lib/bulk_imports/pipeline_schema_info.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module BulkImports + class PipelineSchemaInfo + def initialize(pipeline_class, portable_class) + @pipeline_class = pipeline_class + @portable_class = portable_class + end + + def db_schema + return unless relation + return unless association + + Gitlab::Database::GitlabSchema.tables_to_schema[association.table_name] + end + + def db_table + return unless relation + return unless association + + association.table_name + end + + private + + attr_reader :pipeline_class, :portable_class + + def relation + @relation ||= pipeline_class.try(:relation) + end + + def association + @association ||= portable_class.reflect_on_association(relation) + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb b/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb index 264bda6e654..fe5c61e81a3 100644 --- a/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb @@ -10,8 +10,8 @@ module BulkImports extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation - def after_run(_) - context.portable.merge_requests.set_latest_merge_request_diff_ids! + def on_finish + ::Projects::ImportExport::AfterImportMergeRequestsWorker.perform_async(context.portable.id) end end end diff --git a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb index c77e53b9aec..433419f4c5c 100644 --- a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb @@ -10,9 +10,7 @@ module BulkImports extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation - def after_run(_context) - super - + def on_finish portable.releases.find_each do |release| create_release_evidence(release) end diff --git a/lib/bulk_imports/source_url_builder.rb b/lib/bulk_imports/source_url_builder.rb new file mode 100644 index 00000000000..875b2eae9f7 --- /dev/null +++ b/lib/bulk_imports/source_url_builder.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module BulkImports + class SourceUrlBuilder + ALLOWED_RELATIONS = %w[ + issues + merge_requests + epics + milestones + ].freeze + + attr_reader :context, :entity, :entry + + # @param [BulkImports::Pipeline::Context] context + # @param [ApplicationRecord] entry + def initialize(context, entry) + @context = context + @entity = context.entity + @entry = entry + end + + # Builds a source URL for the given entry if iid is present + def url + return unless entry.is_a?(ApplicationRecord) + return unless iid + return unless ALLOWED_RELATIONS.include?(relation) + + File.join(source_instance_url, group_prefix, source_full_path, '-', relation, iid.to_s) + end + + private + + def iid + @iid ||= entry.try(:iid) + end + + def relation + @relation ||= context.tracker.pipeline_class.relation + end + + def source_instance_url + @source_instance_url ||= context.bulk_import.configuration.url + end + + def source_full_path + @source_full_path ||= entity.source_full_path + end + + # Group milestone (or epic) url is /groups/:group_path/-/milestones/:iid + # Project milestone url is /:project_path/-/milestones/:iid + def group_prefix + return '' if entity.project? + + entity.pluralized_name + end + end +end diff --git a/lib/click_house/migration.rb b/lib/click_house/migration.rb new file mode 100644 index 00000000000..410a7ec86bc --- /dev/null +++ b/lib/click_house/migration.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module ClickHouse + class Migration + cattr_accessor :verbose, :client_configuration + attr_accessor :name, :version + + class << self + attr_accessor :delegate + end + + def initialize(name = self.class.name, version = nil) + @name = name + @version = version + end + + self.client_configuration = ClickHouse::Client.configuration + self.verbose = true + # instantiate the delegate object after initialize is defined + self.delegate = new + + MIGRATION_FILENAME_REGEXP = /\A([0-9]+)_([_a-z0-9]*)\.?([_a-z0-9]*)?\.rb\z/ + + def database + self.class.constants.include?(:SCHEMA) ? self.class.const_get(:SCHEMA, false) : :main + end + + def execute(query) + ClickHouse::Client.execute(query, database, self.class.client_configuration) + end + + def up + self.class.delegate = self + + return unless self.class.respond_to?(:up) + + self.class.up + end + + def down + self.class.delegate = self + + return unless self.class.respond_to?(:down) + + self.class.down + end + + # Execute this migration in the named direction + def migrate(direction) + return unless respond_to?(direction) + + case direction + when :up then announce 'migrating' + when :down then announce 'reverting' + end + + time = Benchmark.measure do + exec_migration(direction) + end + + case direction + when :up then announce format("migrated (%.4fs)", time.real) + write + when :down then announce format("reverted (%.4fs)", time.real) + write + end + end + + private + + def exec_migration(direction) + # noinspection RubyCaseWithoutElseBlockInspection + case direction + when :up then up + when :down then down + end + end + + def write(text = '') + $stdout.puts(text) if verbose + end + + def announce(message) + text = "#{version} #{name}: #{message}" + length = [0, 75 - text.length].max + write format('== %s %s', text, '=' * length) + end + end +end diff --git a/lib/click_house/migration_support/migration_context.rb b/lib/click_house/migration_support/migration_context.rb new file mode 100644 index 00000000000..6e4dd2a97c2 --- /dev/null +++ b/lib/click_house/migration_support/migration_context.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module ClickHouse + module MigrationSupport + # MigrationContext sets the context in which a migration is run. + # + # A migration context requires the path to the migrations is set + # in the +migrations_paths+ parameter. Optionally a +schema_migration+ + # class can be provided. For most applications, +SchemaMigration+ is + # sufficient. Multiple database applications need a +SchemaMigration+ + # per primary database. + class MigrationContext + attr_reader :migrations_paths, :schema_migration + + def initialize(migrations_paths, schema_migration) + @migrations_paths = migrations_paths + @schema_migration = schema_migration + end + + def up(target_version = nil, &block) + selected_migrations = block ? migrations.select(&block) : migrations + + migrate(:up, selected_migrations, target_version) + end + + def down(target_version = nil, &block) + selected_migrations = block ? migrations.select(&block) : migrations + + migrate(:down, selected_migrations, target_version) + end + + private + + def migrate(direction, selected_migrations, target_version = nil) + ClickHouse::MigrationSupport::Migrator.new( + direction, + selected_migrations, + schema_migration, + target_version + ).migrate + end + + def migrations + migrations = migration_files.map do |file| + version, name, scope = parse_migration_filename(file) + + raise ClickHouse::MigrationSupport::IllegalMigrationNameError, file unless version + + version = version.to_i + name = name.camelize + + MigrationProxy.new(name, version, file, scope) + end + + migrations.sort_by(&:version) + end + + def migration_files + paths = Array(migrations_paths) + Dir[*paths.flat_map { |path| "#{path}/**/[0-9]*_*.rb" }] + end + + def parse_migration_filename(filename) + File.basename(filename).scan(ClickHouse::Migration::MIGRATION_FILENAME_REGEXP).first + end + end + + # MigrationProxy is used to defer loading of the actual migration classes + # until they are needed + MigrationProxy = Struct.new(:name, :version, :filename, :scope) do + def initialize(name, version, filename, scope) + super + @migration = nil + end + + def basename + File.basename(filename) + end + + delegate :migrate, :announce, :write, :database, to: :migration + + private + + def migration + @migration ||= load_migration + end + + def load_migration + require(File.expand_path(filename)) + name.constantize.new(name, version) + end + end + end +end diff --git a/lib/click_house/migration_support/migration_error.rb b/lib/click_house/migration_support/migration_error.rb new file mode 100644 index 00000000000..0638d487e37 --- /dev/null +++ b/lib/click_house/migration_support/migration_error.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module ClickHouse + module MigrationSupport + class MigrationError < StandardError + def initialize(message = nil) + message = "\n\n#{message}\n\n" if message + super + end + end + + class IllegalMigrationNameError < MigrationError + def initialize(name = nil) + if name + super("Illegal name for migration file: #{name}\n\t(only lower case letters, numbers, and '_' allowed).") + else + super('Illegal name for migration.') + end + end + end + + IrreversibleMigration = Class.new(MigrationError) + + class DuplicateMigrationVersionError < MigrationError + def initialize(version = nil) + if version + super("Multiple migrations have the version number #{version}.") + else + super('Duplicate migration version error.') + end + end + end + + class DuplicateMigrationNameError < MigrationError + def initialize(name = nil) + if name + super("Multiple migrations have the name #{name}.") + else + super('Duplicate migration name.') + end + end + end + + class UnknownMigrationVersionError < MigrationError + def initialize(version = nil) + if version + super("No migration with version number #{version}.") + else + super('Unknown migration version.') + end + end + end + end +end diff --git a/lib/click_house/migration_support/migrator.rb b/lib/click_house/migration_support/migrator.rb new file mode 100644 index 00000000000..5c67b3a5ff1 --- /dev/null +++ b/lib/click_house/migration_support/migrator.rb @@ -0,0 +1,160 @@ +# frozen_string_literal: true + +module ClickHouse + module MigrationSupport + class Migrator + class << self + attr_accessor :migrations_paths + end + + attr_accessor :logger + + self.migrations_paths = ["db/click_house/migrate"] + + def initialize(direction, migrations, schema_migration, target_version = nil, logger = Gitlab::AppLogger) + @direction = direction + @target_version = target_version + @migrated_versions = {} + @migrations = migrations + @schema_migration = schema_migration + @logger = logger + + validate(@migrations) + + migrations.map(&:database).uniq.each do |database| + @schema_migration.create_table(database) + end + end + + def current_version + @migrated_versions.values.flatten.max || 0 + end + + def current_migration + migrations.detect { |m| m.version == current_version } + end + alias_method :current, :current_migration + + def run + run_without_lock + end + + def migrate + migrate_without_lock + end + + def runnable + runnable = migrations[start..finish] + + if up? + runnable.reject { |m| ran?(m) } + else + # skip the last migration if we're headed down, but not ALL the way down + runnable.pop if target + runnable.find_all { |m| ran?(m) } + end + end + + def migrations + down? ? @migrations.reverse : @migrations.sort_by(&:version) + end + + def pending_migrations(database) + already_migrated = migrated(database) + + migrations.reject { |m| already_migrated.include?(m.version) } + end + + def migrated(database) + @migrated_versions[database] || load_migrated(database) + end + + def load_migrated(database) + @migrated_versions[database] = Set.new(@schema_migration.all_versions(database).map(&:to_i)) + end + + private + + # Used for running a specific migration. + def run_without_lock + migration = migrations.detect { |m| m.version == @target_version } + + raise ClickHouse::MigrationSupport::UnknownMigrationVersionError, @target_version if migration.nil? + + execute_migration(migration) + end + + # Used for running multiple migrations up to or down to a certain value. + def migrate_without_lock + raise ClickHouse::MigrationSupport::UnknownMigrationVersionError, @target_version if invalid_target? + + runnable.each(&method(:execute_migration)) # rubocop: disable Performance/MethodObjectAsBlock -- Execute through proxy + end + + def ran?(migration) + migrated(migration.database).include?(migration.version.to_i) + end + + # Return true if a valid version is not provided. + def invalid_target? + return unless @target_version + return if @target_version == 0 + + !target + end + + def execute_migration(migration) + database = migration.database + + return if down? && migrated(database).exclude?(migration.version.to_i) + return if up? && migrated(database).include?(migration.version.to_i) + + logger.info "Migrating to #{migration.name} (#{migration.version})" if logger + + migration.migrate(@direction) + record_version_state_after_migrating(database, migration.version) + rescue StandardError => e + msg = "An error has occurred, all later migrations canceled:\n\n#{e}" + raise StandardError, msg, e.backtrace + end + + def target + migrations.detect { |m| m.version == @target_version } + end + + def finish + migrations.index(target) || (migrations.size - 1) + end + + def start + up? ? 0 : (migrations.index(current) || 0) + end + + def validate(migrations) + name, = migrations.group_by(&:name).find { |_, v| v.length > 1 } + raise ClickHouse::MigrationSupport::DuplicateMigrationNameError, name if name + + version, = migrations.group_by(&:version).find { |_, v| v.length > 1 } + raise ClickHouse::MigrationSupport::DuplicateMigrationVersionError, version if version + end + + def record_version_state_after_migrating(database, version) + if down? + migrated(database).delete(version) + @schema_migration.create!(database, version: version.to_s, active: 0) + else + migrated(database) << version + @schema_migration.create!(database, version: version.to_s) + end + end + + def up? + @direction == :up + end + + def down? + @direction == :down + end + end + end +end diff --git a/lib/click_house/migration_support/schema_migration.rb b/lib/click_house/migration_support/schema_migration.rb new file mode 100644 index 00000000000..e82debbad0d --- /dev/null +++ b/lib/click_house/migration_support/schema_migration.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module ClickHouse + module MigrationSupport + class SchemaMigration + class_attribute :table_name_prefix, instance_writer: false, default: '' + class_attribute :table_name_suffix, instance_writer: false, default: '' + class_attribute :schema_migrations_table_name, instance_accessor: false, default: 'schema_migrations' + + class << self + TABLE_EXISTS_QUERY = <<~SQL.squish + SELECT 1 FROM system.tables + WHERE name = {table_name: String} AND database = {database_name: String} + SQL + + def primary_key + 'version' + end + + def table_name + "#{table_name_prefix}#{schema_migrations_table_name}#{table_name_suffix}" + end + + def table_exists?(database, configuration = ClickHouse::Migration.client_configuration) + database_name = configuration.databases[database]&.database + return false unless database_name + + placeholders = { table_name: table_name, database_name: database_name } + query = ClickHouse::Client::Query.new(raw_query: TABLE_EXISTS_QUERY, placeholders: placeholders) + + ClickHouse::Client.select(query, database, configuration).any? + end + + def create_table(database, configuration = ClickHouse::Migration.client_configuration) + return if table_exists?(database, configuration) + + query = <<~SQL + CREATE TABLE #{table_name} ( + version LowCardinality(String), + active UInt8 NOT NULL DEFAULT 1, + applied_at DateTime64(6, 'UTC') NOT NULL DEFAULT now64() + ) + ENGINE = ReplacingMergeTree(applied_at) + PRIMARY KEY(version) + ORDER BY (version) + SQL + + ClickHouse::Client.execute(query, database, configuration) + end + + def all_versions(database) + query = <<~SQL + SELECT version FROM #{table_name} FINAL + WHERE active = 1 + ORDER BY (version) + SQL + + ClickHouse::Client.select(query, database, ClickHouse::Migration.client_configuration).pluck('version') + end + + def create!(database, **args) + insert_sql = <<~SQL + INSERT INTO #{table_name} (#{args.keys.join(',')}) VALUES (#{args.values.join(',')}) + SQL + + ClickHouse::Client.execute(insert_sql, database, ClickHouse::Migration.client_configuration) + end + end + end + end +end diff --git a/lib/click_house/models/audit_event.rb b/lib/click_house/models/audit_event.rb new file mode 100644 index 00000000000..a31b4a45298 --- /dev/null +++ b/lib/click_house/models/audit_event.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module ClickHouse + module Models + class AuditEvent < ClickHouse::Models::BaseModel + def self.table_name + 'audit_events' + end + + def by_entity_type(entity_type) + where(entity_type: entity_type) + end + + def by_entity_id(entity_id) + where(entity_id: entity_id) + end + + def by_author_id(author_id) + where(author_id: author_id) + end + + def by_entity_username(username) + where(entity_id: self.class.find_user_id(username)) + end + + def by_author_username(username) + where(author_id: self.class.find_user_id(username)) + end + + def self.by_entity_type(entity_type) + new.by_entity_type(entity_type) + end + + def self.by_entity_id(entity_id) + new.by_entity_id(entity_id) + end + + def self.by_author_id(author_id) + new.by_author_id(author_id) + end + + def self.by_entity_username(username) + new.by_entity_username(username) + end + + def self.by_author_username(username) + new.by_author_username(username) + end + + def self.find_user_id(username) + ::User.find_by_username(username)&.id + end + end + end +end diff --git a/lib/click_house/models/base_model.rb b/lib/click_house/models/base_model.rb new file mode 100644 index 00000000000..89624076f15 --- /dev/null +++ b/lib/click_house/models/base_model.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# rubocop: disable CodeReuse/ActiveRecord +module ClickHouse + module Models + class BaseModel + extend Forwardable + + def_delegators :@query_builder, :to_sql + + def initialize(query_builder = ClickHouse::QueryBuilder.new(self.class.table_name)) + @query_builder = query_builder + end + + def self.table_name + raise NotImplementedError, "Subclasses must define a `table_name` class method" + end + + def where(conditions) + self.class.new(@query_builder.where(conditions)) + end + + def order(field, direction = :asc) + self.class.new(@query_builder.order(field, direction)) + end + + def limit(count) + self.class.new(@query_builder.limit(count)) + end + + def offset(count) + self.class.new(@query_builder.offset(count)) + end + + def select(...) + self.class.new(@query_builder.select(...)) + end + end + end +end +# rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index e2a1b8296f6..580ba2bdc0d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -78,13 +78,9 @@ module ContainerRegistry delete_if_exists("/v2/#{name}/manifests/#{reference}") end - def delete_repository_tag_by_name(name, reference) - delete_if_exists("/v2/#{name}/tags/reference/#{reference}") - end - # Check if the registry supports tag deletion. This is only supported by the # GitLab registry fork. The fastest and safest way to check this is to send - # an OPTIONS request to /v2/<name>/tags/reference/<tag>, using a random + # an OPTIONS request to /v2/<name>/manifests/<tag>, using a random # repository name and tag (the registry won't check if they exist). # Registries that support tag deletion will reply with a 200 OK and include # the DELETE method in the Allow header. Others reply with an 404 Not Found. @@ -93,7 +89,7 @@ module ContainerRegistry registry_features = Gitlab::CurrentSettings.container_registry_features || [] next true if ::Gitlab.com_except_jh? && registry_features.include?(REGISTRY_TAG_DELETE_FEATURE) - response = faraday.run_request(:options, '/v2/name/tags/reference/tag', '', {}) + response = faraday.run_request(:options, '/v2/name/manifests/tag', '', {}) response.success? && response.headers['allow']&.include?('DELETE') end end diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index bd833ec00af..9b6c37da847 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -103,7 +103,7 @@ module ContainerRegistry end end - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#compliance-check + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/gitlab/api.md#compliance-check def supports_gitlab_api? strong_memoize(:supports_gitlab_api) do registry_features = Gitlab::CurrentSettings.container_registry_features || [] @@ -116,19 +116,19 @@ module ContainerRegistry end end - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#import-repository + # Deprecated. Will be removed as part of https://gitlab.com/gitlab-org/gitlab/-/issues/409873. def pre_import_repository(path) response = start_import_for(path, pre: true) IMPORT_RESPONSES.fetch(response.status, :error) end - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#import-repository + # Deprecated. Will be removed as part of https://gitlab.com/gitlab-org/gitlab/-/issues/409873. def import_repository(path) response = start_import_for(path, pre: false) IMPORT_RESPONSES.fetch(response.status, :error) end - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#cancel-repository-import + # Deprecated. Will be removed as part of https://gitlab.com/gitlab-org/gitlab/-/issues/409873. def cancel_repository_import(path, force: false) response = with_import_token_faraday do |faraday_client| faraday_client.delete(import_url_for(path)) do |req| @@ -142,7 +142,7 @@ module ContainerRegistry { status: status, migration_state: actual_state } end - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#get-repository-import-status + # Deprecated. Will be removed as part of https://gitlab.com/gitlab-org/gitlab/-/issues/409873. def import_status(path) with_import_token_faraday do |faraday_client| response = faraday_client.get(import_url_for(path)) @@ -156,7 +156,7 @@ module ContainerRegistry end end - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#get-repository-details + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/gitlab/api.md#get-repository-details def repository_details(path, sizing: nil) with_token_faraday do |faraday_client| req = faraday_client.get("#{GITLAB_REPOSITORIES_PATH}/#{path}/") do |req| @@ -169,7 +169,7 @@ module ContainerRegistry end end - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#list-repository-tags + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/gitlab/api.md#list-repository-tags def tags(path, page_size: 100, last: nil, before: nil, name: nil, sort: nil) limited_page_size = [page_size, MAX_TAGS_PAGE_SIZE].min with_token_faraday do |faraday_client| @@ -178,7 +178,7 @@ module ContainerRegistry req.params['n'] = limited_page_size req.params['last'] = last if last req.params['before'] = before if before - req.params['name'] = name if name + req.params['name'] = name if name.present? req.params['sort'] = sort if sort end @@ -202,7 +202,7 @@ module ContainerRegistry end end - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#list-sub-repositories + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/gitlab/api.md#list-sub-repositories def sub_repositories_with_tag(path, page_size: 100, last: nil) limited_page_size = [page_size, MAX_REPOSITORIES_PAGE_SIZE].min @@ -235,7 +235,7 @@ module ContainerRegistry # Given a path 'group/subgroup/project' and name 'newname', # with a successful rename, it will be 'group/subgroup/newname' - # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#rename-base-repository + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/gitlab/api.md#rename-base-repository def rename_base_repository_path(path, name:, dry_run: false) with_token_faraday do |faraday_client| url = "#{GITLAB_REPOSITORIES_PATH}/#{path}/" diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index bf44b74cf7b..70742e8bd38 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -5,16 +5,25 @@ module ContainerRegistry include Gitlab::Utils::StrongMemoize attr_reader :repository, :name, :updated_at - attr_writer :created_at + attr_writer :created_at, :manifest_digest, :revision, :total_size delegate :registry, :client, to: :repository - delegate :revision, :short_revision, to: :config_blob, allow_nil: true def initialize(repository, name) @repository = repository @name = name end + def revision + @revision || config_blob&.revision + end + + def short_revision + return unless revision + + revision[0..8] + end + def valid? manifest.present? end @@ -53,7 +62,7 @@ module ContainerRegistry def digest strong_memoize(:digest) do - client.repository_tag_digest(repository.path, name) + @manifest_digest || client.repository_tag_digest(repository.path, name) end end @@ -126,6 +135,8 @@ module ContainerRegistry # rubocop: disable CodeReuse/ActiveRecord def total_size + return @total_size if @total_size + return unless layers layers.sum(&:size) if v2? diff --git a/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template b/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template index 886a3bd3116..df4c5382749 100644 --- a/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template +++ b/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template @@ -6,6 +6,8 @@ # Update below commented lines with appropriate values. class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + milestone '<%= Gitlab.current_milestone %>' + MIGRATION = "<%= class_name %>" # DELAY_INTERVAL = 2.minutes # BATCH_SIZE = <%= Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers::BATCH_SIZE %> diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template index f2f9acea923..a9cf5d085d1 100644 --- a/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template +++ b/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template @@ -1,6 +1,8 @@ # frozen_string_literal: true class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + milestone '<%= Gitlab.current_milestone %>' + disable_ddl_transaction! SOURCE_TABLE_NAME = :<%= source_table_name %> diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template index 4896d931333..4ca4dd3c842 100644 --- a/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template +++ b/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template @@ -1,6 +1,8 @@ # frozen_string_literal: true class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + milestone '<%= Gitlab.current_milestone %>' + disable_ddl_transaction! INDEX_NAME = :index_<%= source_table_name -%>_on_<%= partitioning_column -%>_<%= foreign_key_column %> diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template index b4e881074ad..16bd2548f18 100644 --- a/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template +++ b/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template @@ -1,6 +1,8 @@ # frozen_string_literal: true class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + milestone '<%= Gitlab.current_milestone %>' + disable_ddl_transaction! SOURCE_TABLE_NAME = :<%= source_table_name %> diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template index bad7d17a51b..b065f390863 100644 --- a/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template +++ b/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template @@ -1,6 +1,8 @@ # frozen_string_literal: true class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + milestone '<%= Gitlab.current_milestone %>' + disable_ddl_transaction! TABLE_NAME = :<%= source_table_name %> diff --git a/lib/generators/gitlab/snowplow_event_definition_generator.rb b/lib/generators/gitlab/snowplow_event_definition_generator.rb deleted file mode 100644 index b1a31541350..00000000000 --- a/lib/generators/gitlab/snowplow_event_definition_generator.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -require 'rails/generators' - -module Gitlab - class SnowplowEventDefinitionGenerator < Rails::Generators::Base - CE_DIR = 'config/events' - EE_DIR = 'ee/config/events' - - source_root File.expand_path('../../../generator_templates/snowplow_event_definition', __dir__) - - desc 'Generates an event definition yml file' - - class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if event is for ee' - class_option :category, type: :string, optional: false, desc: 'Category of the event' - class_option :action, type: :string, optional: false, desc: 'Action of the event' - - def create_event_file - raise "Event definition already exists at #{file_path}" if definition_exists? - - template "event_definition.yml", file_path, force: false - end - - def distributions - (ee? ? ['- ee'] : ['- ce', '- ee']).join("\n") - end - - def event_category - options[:category] - end - - def event_action - options[:action] - end - - def milestone - Gitlab::VERSION.match('(\d+\.\d+)').captures.first - end - - def ee? - options[:ee] - end - - private - - def definition_exists? - File.exist?(ce_file_path) || File.exist?(ee_file_path) - end - - def file_path - ee? ? ee_file_path : ce_file_path - end - - def ce_file_path - File.join(CE_DIR, file_name) - end - - def ee_file_path - File.join(EE_DIR, file_name) - end - - # Example of file name - # 20230227000018_project_management_issue_title_changed.yml - def file_name - name = remove_special_chars("#{Time.now.utc.strftime('%Y%m%d%H%M%S')}_#{event_category}_#{event_action}") - "#{name[0..95]}.yml" # max 100 chars, see https://gitlab.com/gitlab-com/gl-infra/delivery/-/issues/2030#note_679501200 - end - - def remove_special_chars(input) - input.gsub("::", "__").gsub(/[^A-Za-z0-9_]/, '') - end - end -end diff --git a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb index 8cd03978f27..f8a05d3132f 100644 --- a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb @@ -1,11 +1,14 @@ # frozen_string_literal: true +# DEPRECATED. Consider using using Internal Events tracking framework +# https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/quick_start.html + require 'rails/generators' module Gitlab module UsageMetricDefinition class RedisHllGenerator < Rails::Generators::Base - desc 'Generates a metric definition .yml file with defaults for Redis HLL.' + desc '[DEPRECATED] Generates a metric definition .yml file with defaults for Redis HLL.' argument :category, type: :string, desc: "Category name" argument :events, type: :array, desc: "Unique event names", banner: 'event_one event_two event_three' diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb index d57a6b0b724..c231697e22e 100644 --- a/lib/generators/gitlab/usage_metric_definition_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition_generator.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# DEPRECATED. Consider using using Internal Events tracking framework +# https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/quick_start.html + require 'rails/generators' module Gitlab @@ -30,7 +33,7 @@ module Gitlab source_root File.expand_path('../../../generator_templates/usage_metric_definition', __dir__) - desc 'Generates metric definitions yml files' + desc '[DEPRECATED] Generates metric definitions yml files' class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if metric is for ee' class_option :dir, @@ -40,6 +43,13 @@ module Gitlab argument :key_paths, type: :array, desc: 'Unique JSON key paths for the metrics' def create_metric_file + say("This generator is DEPRECATED. Use Internal Events tracking framework instead.") + # rubocop: disable Gitlab/DocUrl -- link for developers, not users + say("https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/quick_start.html") + # rubocop: enable Gitlab/DocUrl + desc = ask("Would you like to continue anyway? y/N") || 'n' + return unless desc.casecmp('y') == 0 + validate! key_paths.each do |key_path| diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 0875b14f7d0..b98a0207567 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -16,6 +16,11 @@ module Gitlab Gitlab::VersionInfo.parse(Gitlab::VERSION) end + def self.current_milestone + v = version_info + "#{v.major}.#{v.minor}" + end + def self.pre_release? VERSION.include?('pre') end diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index cea25ba2db4..0c4a0afa1d5 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -203,7 +203,8 @@ module Gitlab def validate_date_range return if created_after.nil? || created_before.nil? - if (created_before - created_after) > MAX_RANGE_DAYS + time_period = created_before.at_beginning_of_day - created_after.at_beginning_of_day + if time_period > MAX_RANGE_DAYS errors.add(:created_after, s_('CycleAnalytics|The given date range is larger than 180 days')) end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index bf3f5b61825..469927b8a53 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -55,6 +55,7 @@ module Gitlab phone_verification_send_code: { threshold: 10, interval: 1.hour }, phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, namespace_exists: { threshold: 20, interval: 1.minute }, + update_namespace_name: { threshold: -> { application_settings.update_namespace_name_rate_limit }, interval: 1.hour }, fetch_google_ip_list: { threshold: 10, interval: 1.minute }, project_fork_sync: { threshold: 10, interval: 30.minutes }, ai_action: { threshold: 160, interval: 8.hours }, diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index fc1f7a1583c..578cfb52714 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -60,10 +60,10 @@ module Gitlab Gitlab.config.omniauth.enabled end - def find_for_git_client(login, password, project:, ip:) - raise "Must provide an IP for rate limiting" if ip.nil? + def find_for_git_client(login, password, project:, request:) + raise "Must provide an IP for rate limiting" if request.ip.nil? - rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) + rate_limiter = Gitlab::Auth::IpRateLimiter.new(request.ip) raise IpBlocked if !skip_rate_limit?(login: login) && rate_limiter.banned? @@ -80,7 +80,7 @@ module Gitlab user_with_password_for_git(login, password) || Gitlab::Auth::Result::EMPTY - rate_limit!(rate_limiter, success: result.success?, login: login) + rate_limit!(rate_limiter, success: result.success?, login: login, request: request) look_to_limit_user(result.actor) return result if result.success? || authenticate_using_internal_or_ldap_password? @@ -142,7 +142,7 @@ module Gitlab private - def rate_limit!(rate_limiter, success:, login:) + def rate_limit!(rate_limiter, success:, login:, request:) return if skip_rate_limit?(login: login) if success @@ -155,8 +155,18 @@ module Gitlab # request from this IP if needed. # This returns true when the failures are over the threshold and the IP # is banned. - Gitlab::AppLogger.info "IP #{rate_limiter.ip} failed to login " \ - "as #{login} but has been temporarily banned from Git auth" + + message = "Rack_Attack: Git auth failures has exceeded the threshold. " \ + "IP has been temporarily banned from Git auth." + + Gitlab::AuthLogger.error( + message: message, + env: :blocklist, + remote_ip: request.ip, + request_method: request.request_method, + path: request.fullpath, + login: login + ) end end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 7524d8b9f85..e6c9f04eff5 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -8,6 +8,21 @@ module Gitlab def enabled? ::AuthHelper.saml_providers.any? end + + def default_attribute_statements + defaults = OmniAuth::Strategies::SAML.default_options[:attribute_statements].to_hash.deep_symbolize_keys + defaults[:nickname] = %w[username nickname] + defaults[:name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name' + defaults[:name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/name' + defaults[:email] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress' + defaults[:email] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress' + defaults[:first_name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname' + defaults[:first_name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname' + defaults[:last_name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname' + defaults[:last_name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/surname' + + defaults + end end DEFAULT_PROVIDER_NAME = 'saml' diff --git a/lib/gitlab/auth/two_factor_auth_verifier.rb b/lib/gitlab/auth/two_factor_auth_verifier.rb index fbdfd105ee3..4b66aaf0e6a 100644 --- a/lib/gitlab/auth/two_factor_auth_verifier.rb +++ b/lib/gitlab/auth/two_factor_auth_verifier.rb @@ -36,7 +36,7 @@ module Gitlab return false unless time - two_factor_grace_period.hours.since(time) < Time.current + two_factor_grace_period.hours.since(time).past? end def allow_2fa_bypass_for_provider diff --git a/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb b/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb new file mode 100644 index 00000000000..04fd09f81f0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration populates the new `packages_tags.project_id` column from joining with `packages_packages` table + class BackfillPackagesTagsProjectId < BatchedMigrationJob + operation_name :update_all # This is used as the key on collecting metrics + scope_to ->(relation) { relation.where(project_id: nil) } + feature_category :package_registry + + def perform + each_sub_batch do |sub_batch| + joined = sub_batch + .joins('INNER JOIN packages_packages ON packages_tags.package_id = packages_packages.id') + .select('packages_tags.id, packages_packages.project_id') + + ApplicationRecord.connection.execute <<~SQL + WITH joined_cte(packages_tag_id, project_id) AS MATERIALIZED ( + #{joined.to_sql} + ) + UPDATE packages_tags + SET project_id = joined_cte.project_id + FROM joined_cte + WHERE id = joined_cte.packages_tag_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 952e6d01f1a..9e9fc9b98b7 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -130,7 +130,7 @@ module Gitlab end def base_relation - define_batchable_model(batch_table, connection: connection) + define_batchable_model(batch_table, connection: connection, primary_key: batch_column) .where(batch_column => start_id..end_id) end diff --git a/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb new file mode 100644 index 00000000000..99bc638532a --- /dev/null +++ b/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to remove protected_branch_merge_access_levels for groups that do not have project_group_links + # to the project for the associated protected branch + class DeleteInvalidProtectedBranchMergeAccessLevels < BatchedMigrationJob + operation_name :delete_invalid_protected_branch_merge_access_levels + scope_to ->(relation) { relation.where.not(group_id: nil) } + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch + .joins('INNER JOIN protected_branches ON protected_branches.id = protected_branch_id') + .joins(%( + LEFT OUTER JOIN project_group_links pgl + ON pgl.group_id = protected_branch_merge_access_levels.group_id + AND pgl.project_id = protected_branches.project_id + )) + .where(%( + pgl.id IS NULL + )).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb new file mode 100644 index 00000000000..a6934cf5adc --- /dev/null +++ b/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to remove protected_branch_push_access_levels for groups that do not have project_group_links + # to the project for the associated protected branch + class DeleteInvalidProtectedBranchPushAccessLevels < BatchedMigrationJob + operation_name :delete_invalid_protected_branch_push_access_levels + scope_to ->(relation) { relation.where.not(group_id: nil) } + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch + .joins('INNER JOIN protected_branches ON protected_branches.id = protected_branch_id') + .joins(%( + LEFT OUTER JOIN project_group_links pgl + ON pgl.group_id = protected_branch_push_access_levels.group_id + AND pgl.project_id = protected_branches.project_id + )) + .where(%( + pgl.id IS NULL + )).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb new file mode 100644 index 00000000000..8c59e42a9f6 --- /dev/null +++ b/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to remove protected_tag_create_access_levels for groups that do not have project_group_links + # to the project for the associated protected branch + class DeleteInvalidProtectedTagCreateAccessLevels < BatchedMigrationJob + operation_name :delete_invalid_protected_tag_create_access_levels + scope_to ->(relation) { relation.where.not(group_id: nil) } + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch + .joins('INNER JOIN protected_tags ON protected_tags.id = protected_tag_id') + .joins(%( + LEFT OUTER JOIN project_group_links pgl + ON pgl.group_id = protected_tag_create_access_levels.group_id + AND pgl.project_id = protected_tags.project_id + )) + .where(%( + pgl.id IS NULL + )).delete_all + end + end + end + end +end diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index 91994c2fa95..c8520993b8e 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -3,7 +3,8 @@ # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab - class BaseDoorkeeperController < BaseActionController + # rubocop:disable Rails/ApplicationController + class BaseDoorkeeperController < ActionController::Base include Gitlab::Allowable include EnforcesTwoFactorAuthentication include SessionsHelper @@ -12,4 +13,5 @@ module Gitlab helper_method :can? end + # rubocop:enable Rails/ApplicationController end diff --git a/lib/gitlab/bitbucket_import/importers/issue_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_importer.rb index 2c3be67eabc..d194a311278 100644 --- a/lib/gitlab/bitbucket_import/importers/issue_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/issue_importer.rb @@ -40,6 +40,8 @@ module Gitlab project.issues.create!(attributes) + metrics.issues_counter.increment + log_info(import_stage: 'import_issue', message: 'finished', iid: object[:iid]) rescue StandardError => e track_import_failure!(project, exception: e) diff --git a/lib/gitlab/bitbucket_import/importers/issues_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_importer.rb index 6162433e701..8ab82ddb0be 100644 --- a/lib/gitlab/bitbucket_import/importers/issues_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/issues_importer.rb @@ -7,17 +7,21 @@ module Gitlab include ParallelScheduling def execute + return job_waiter unless repo.issues_enabled? + log_info(import_stage: 'import_issues', message: 'importing issues') issues = client.issues(project.import_source) labels = build_labels_hash - issues.each do |issue| + issues.each_with_index do |issue, index| job_waiter.jobs_remaining += 1 next if already_enqueued?(issue) + allocate_issues_internal_id! if index == 0 + job_delay = calculate_job_delay(job_waiter.jobs_remaining) issue_hash = issue.to_hash.merge({ issue_type_id: default_issue_type_id, label_id: labels[issue.kind] }) @@ -49,11 +53,23 @@ module Gitlab ::WorkItems::Type.default_issue_type.id end + def allocate_issues_internal_id! + last_bitbucket_issue = client.last_issue(repo) + + return unless last_bitbucket_issue + + Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid) + end + def build_labels_hash labels = {} project.labels.each { |l| labels[l.title.to_s] = l.id } labels end + + def repo + @repo ||= client.repo(project.import_source) + end end end end diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb index a18d50e8fce..f7b1753a9f9 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb @@ -45,6 +45,8 @@ module Gitlab merge_request.assignee_ids = [author_id] merge_request.reviewer_ids = reviewers merge_request.save! + + metrics.merge_requests_counter.increment end log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid]) diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb index 8ea8b1562f2..934e4ee1720 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb @@ -4,21 +4,22 @@ module Gitlab module BitbucketImport module Importers class PullRequestNotesImporter - include Loggable - include ErrorTracking + include ParallelScheduling def initialize(project, hash) @project = project - @importer = Gitlab::BitbucketImport::Importer.new(project) + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project) @object = hash.with_indifferent_access + @position_map = {} + @discussion_map = {} end def execute log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid]) - merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord - - importer.import_pull_request_comments(merge_request, merge_request) if merge_request + import_pull_request_comments if merge_request log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid]) rescue StandardError => e @@ -27,7 +28,116 @@ module Gitlab private - attr_reader :object, :project, :importer + attr_reader :object, :project, :formatter, :user_finder, :ref_converter, :discussion_map, :position_map + + def import_pull_request_comments + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments) + import_standalone_pr_comments(pr_comments) + end + + def import_inline_comments(inline_comments) + children, parents = inline_comments.partition(&:has_parent?) + + parents.each do |comment| + position_map[comment.iid] = build_position(comment) + + import_comment(comment) + end + + children.each do |comment| + position_map[comment.iid] = position_map.fetch(comment.parent_id, nil) + + import_comment(comment) + end + end + + def import_comment(comment) + position = position_map[comment.iid] + discussion_id = discussion_map[comment.parent_id] + + note = create_diff_note(comment, position, discussion_id) + + discussion_map[comment.iid] = note&.discussion_id + end + + def create_diff_note(comment, position, discussion_id) + attributes = pull_request_comment_attributes(comment) + attributes.merge!(position: position, type: 'DiffNote', discussion_id: discussion_id) + + note = merge_request.notes.build(attributes) + + return note if note.save + + # Bitbucket supports the ability to comment on any line, not just the + # line in the diff. If we can't add the note as a DiffNote, fallback to creating + # a regular note. + + log_info(import_stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) + create_fallback_diff_note(comment, position) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception( + e, + import_stage: 'create_diff_note', comment_id: comment.iid, error: e.message + ) + + nil + end + + def create_fallback_diff_note(comment, position) + attributes = pull_request_comment_attributes(comment) + note = "*Comment on" + + note += " #{position.old_path}:#{position.old_line} -->" if position&.old_line + note += " #{position.new_path}:#{position.new_line}" if position&.new_line + note += "*\n\n#{comment.note}" + + attributes[:note] = note + merge_request.notes.create!(attributes) + end + + def build_position(pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments) + pr_comments.each do |comment| + attributes = pull_request_comment_attributes(comment) + merge_request.notes.create!(attributes) + end + end + + def pull_request_comment_attributes(comment) + { + project: project, + author_id: user_finder.gitlab_user_id(project, comment.author), + note: comment_note(comment), + created_at: comment.created_at, + updated_at: comment.updated_at + } + end + + def comment_note(comment) + author = formatter.author_line(comment.author) unless user_finder.find_user_id(comment.author) + author.to_s + ref_converter.convert_note(comment.note.to_s) + end + + def merge_request + @merge_request ||= project.merge_requests.iid_in(object[:iid]).first + end + + def comments + client.pull_request_comments(project.import_source, merge_request.iid).reject(&:deleted?) + end end end end diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb index b8c0ba69d37..9be7ed99436 100644 --- a/lib/gitlab/bitbucket_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb @@ -19,6 +19,7 @@ module Gitlab validate_repository_size! + set_default_branch update_clone_time end @@ -76,6 +77,16 @@ module Gitlab def validate_repository_size! # Defined in EE end + + def set_default_branch + default_branch = client.repo(project.import_source).default_branch + + project.change_head(default_branch) if default_branch + end + + def client + Bitbucket::Client.new(project.import_data.credentials) + end end end end diff --git a/lib/gitlab/bitbucket_import/loggable.rb b/lib/gitlab/bitbucket_import/loggable.rb index eda3cc96d4d..aeae993b9eb 100644 --- a/lib/gitlab/bitbucket_import/loggable.rb +++ b/lib/gitlab/bitbucket_import/loggable.rb @@ -19,6 +19,10 @@ module Gitlab logger.error(log_data(messages)) end + def metrics + Gitlab::Import::Metrics.new(:bitbucket_importer, project) + end + private def logger diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 15b38188f13..a359236e150 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -31,8 +31,7 @@ module Gitlab def treeish_objects objects = commits - return objects unless project.repository.empty? && - Feature.enabled?(:verify_push_rules_for_first_commit, project) + return objects unless project.repository.empty? # It's a special case for the push to the empty repository # diff --git a/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb index 35f969dbb46..b8c6bdee1bb 100644 --- a/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb +++ b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb @@ -6,7 +6,7 @@ module Gitlab class AnyOversizedBlobs def initialize(project:, changes:, file_size_limit_megabytes:) @project = project - @newrevs = changes.pluck(:newrev).compact # rubocop:disable CodeReuse/ActiveRecord just plucking from an array + @newrevs = changes.pluck(:newrev).compact # rubocop:disable CodeReuse/ActiveRecord -- Array#pluck @file_size_limit_megabytes = file_size_limit_megabytes end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 21fc2980cdc..791b8a963e9 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -35,13 +35,15 @@ module Gitlab end attr_reader :offset, :sections, :segments, :current_segment, - :section_header, :section_duration, :section_options + :section_header, :section_footer, :section_duration, + :section_options def initialize(offset:, style:, sections: []) @offset = offset @segments = [] @sections = sections @section_header = false + @section_footer = false @duration = nil @current_segment = Segment.new(style: style) end @@ -79,6 +81,10 @@ module Gitlab @section_header = true end + def set_as_section_footer + @section_footer = true + end + def set_section_duration(duration_in_seconds) normalized_duration_in_seconds = duration_in_seconds.to_i.clamp(0, 1.year) duration = ActiveSupport::Duration.build(normalized_duration_in_seconds) @@ -103,6 +109,7 @@ module Gitlab { offset: offset, content: @segments }.tap do |result| result[:section] = sections.last if sections.any? result[:section_header] = true if @section_header + result[:section_footer] = true if @section_footer result[:section_duration] = @section_duration if @section_duration result[:section_options] = @section_options if @section_options end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 3aec1cde1bc..6cf76fbbb51 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -49,6 +49,7 @@ module Gitlab duration = timestamp.to_i - @open_sections[section].to_i @current_line.set_section_duration(duration) + @current_line.set_as_section_footer @open_sections.delete(section) end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index 48b138b0258..bbcdcd7d389 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -33,13 +33,9 @@ module Gitlab # Assigning tags and needs is slow and they are not needed for rules # evaluation since we don't use them to compute the variables at this point. def build_attributes - if pipeline.reduced_build_attributes_list_for_rules? - attributes - .except(:tag_list, :needs_attributes) - .merge!(pipeline_attributes, ci_stage_attributes) - else - attributes.merge(pipeline_attributes, ci_stage_attributes) - end + attributes + .except(:tag_list, :needs_attributes) + .merge!(pipeline_attributes, ci_stage_attributes) end def ci_stage_attributes diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index df2b2a14fc6..50731d54fc0 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -18,7 +18,6 @@ module Gitlab def initialize(address:) @full_path, @version = address.to_s.split('@', 2) @host = Settings.gitlab_ci['component_fqdn'] - @component_project = ::Ci::Catalog::ComponentsProject.new(project, sha) end def fetch_content!(current_user:) @@ -27,7 +26,8 @@ module Gitlab raise Gitlab::Access::AccessDeniedError unless Ability.allowed?(current_user, :download_code, project) - @component_project.fetch_component(component_name) + component_project = ::Ci::Catalog::ComponentsProject.new(project, sha) + component_project.fetch_component(component_name) end def project diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index bf8a99ef45e..5fcafcba829 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks coverage retry parallel interruptible timeout - release id_tokens publish].freeze + release id_tokens publish pages].freeze validations do validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS @@ -40,13 +40,19 @@ module Gitlab if needs_value[:job].nil? && needs_value[:cross_dependency].present? errors.add(:needs, "corresponding to dependencies must be from the same pipeline") else - missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) + missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord -- Array#pluck errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any? end end - validates :publish, absence: { message: "can only be used within a `pages` job" }, unless: -> { pages_job? } + validates :publish, + absence: { message: "can only be used within a `pages` job" }, + unless: -> { pages_job? } + + validates :pages, + absence: { message: "can only be used within a `pages` job" }, + unless: -> { pages_job? } end entry :before_script, Entry::Commands, @@ -127,10 +133,14 @@ module Gitlab description: 'Path to be published with Pages', inherit: false + entry :pages, ::Gitlab::Ci::Config::Entry::Pages, + inherit: false, + description: 'Pages configuration.' + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, - :release, :allow_failure, :publish + :interruptible, :timeout, :release, + :allow_failure, :publish, :pages def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -170,7 +180,8 @@ module Gitlab needs: needs_defined? ? needs_value : nil, scheduling_type: needs_defined? ? :dag : :stage, id_tokens: id_tokens_value, - publish: publish + publish: publish, + pages: pages ).compact end diff --git a/lib/gitlab/ci/config/entry/pages.rb b/lib/gitlab/ci/config/entry/pages.rb new file mode 100644 index 00000000000..57d9e944f51 --- /dev/null +++ b/lib/gitlab/ci/config/entry/pages.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the pages path prefix + # Entry that represents the pages attributes + # + class Pages < ::Gitlab::Config::Entry::Node + ALLOWED_KEYS = %i[path_prefix].freeze + + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Validatable + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :path_prefix, type: String + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 88734ac1186..d0e9a9afc51 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -25,6 +25,8 @@ module Gitlab validates :name, type: Symbol validates :name, length: { maximum: 255 } + validates :config, mutually_exclusive_keys: %i[script trigger] + validates :config, disallowed_keys: { in: %i[only except start_in], message: 'key may not be used with `rules`', diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb index dcb96006459..08ee70b6290 100644 --- a/lib/gitlab/ci/config/header/input.rb +++ b/lib/gitlab/ci/config/header/input.rb @@ -11,17 +11,24 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[default description regex type].freeze + ALLOWED_KEYS = %i[default description options regex type].freeze + ALLOWED_OPTIONS_LIMIT = 50 attributes ALLOWED_KEYS, prefix: :input validations do validates :config, type: Hash, allowed_keys: ALLOWED_KEYS validates :key, alphanumeric: true - validates :input_default, alphanumeric: true, allow_nil: true validates :input_description, alphanumeric: true, allow_nil: true validates :input_regex, type: String, allow_nil: true validates :input_type, allow_nil: true, allowed_values: Interpolation::Inputs.input_types + validates :input_options, type: Array, allow_nil: true + + validate do + if input_options&.size.to_i > ALLOWED_OPTIONS_LIMIT + errors.add(:config, "cannot define more than #{ALLOWED_OPTIONS_LIMIT} options") + end + end end end end diff --git a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb index ba519776635..987268b0525 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb @@ -20,11 +20,6 @@ module Gitlab raise NotImplementedError end - # Checks whether the provided value is of the given type - def valid_value?(value) - raise NotImplementedError - end - attr_reader :errors, :name, :spec, :value def initialize(name:, spec:, value:) @@ -54,20 +49,39 @@ module Gitlab private def validate! - return error('required value has not been provided') if required_input? && value.nil? + validate_required + + return if errors.present? - # validate default value - if !required_input? && !valid_value?(default) - return error("default value is not a #{self.class.type_name}") - end + run_validations(default, default: true) unless required_input? - # validate provided value - return error("provided value is not a #{self.class.type_name}") unless valid_value?(actual_value) + run_validations(value) unless value.nil? + end + + def validate_required + error('required value has not been provided') if required_input? && value.nil? + end - validate_regex! + def run_validations(value, default: false) + validate_type(value, default) + validate_options(value) + validate_regex(value, default) end - def validate_regex! + # Type validations are done separately for different input types. + def validate_type(_value, _default) + raise NotImplementedError + end + + # Options can be either StringInput or NumberInput and are validated accordingly. + def validate_options(_value) + return unless options + + error('Options can only be used with string and number inputs') + end + + # Regex can be only be a StringInput and is validated accordingly. + def validate_regex(_value, _default) return unless spec.key?(:regex) error('RegEx validation can only be used with string inputs') @@ -96,6 +110,10 @@ module Gitlab def default spec[:default] end + + def options + spec[:options] + end end end end diff --git a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb index 0293c01a5a8..4c34f7e7fdd 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb @@ -6,6 +6,8 @@ module Gitlab module Interpolation class Inputs class BooleanInput < BaseInput + extend ::Gitlab::Utils::Override + def self.matches?(spec) spec.is_a?(Hash) && spec[:type] == type_name end @@ -14,8 +16,11 @@ module Gitlab 'boolean' end - def valid_value?(value) - [true, false].include?(value) + override :validate_type + def validate_type(value, default) + return if [true, false].include?(value) + + error("#{default ? 'default' : 'provided'} value is not a boolean") end end end diff --git a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb index 314315d2b6d..59bc057749a 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb @@ -6,6 +6,8 @@ module Gitlab module Interpolation class Inputs class NumberInput < BaseInput + extend ::Gitlab::Utils::Override + def self.matches?(spec) spec.is_a?(Hash) && spec[:type] == type_name end @@ -14,8 +16,19 @@ module Gitlab 'number' end - def valid_value?(value) - value.is_a?(Numeric) + override :validate_type + def validate_type(value, default) + return if value.is_a?(Numeric) + + error("#{default ? 'default' : 'provided'} value is not a number") + end + + override :validate_options + def validate_options(value) + return unless options && value + return if options.include?(value) + + error("`#{value}` cannot be used because it is not in the list of the allowed options") end end end diff --git a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb index 3f40e851f11..01b9d34a883 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb @@ -6,6 +6,8 @@ module Gitlab module Interpolation class Inputs class StringInput < BaseInput + extend ::Gitlab::Utils::Override + def self.matches?(spec) # The input spec can be `nil` when using a minimal specification # and also when `type` is not specified. @@ -22,24 +24,32 @@ module Gitlab 'string' end - def valid_value?(value) - value.nil? || value.is_a?(String) + override :validate_type + def validate_type(value, default) + return if value.is_a?(String) + + error("#{default ? 'default' : 'provided'} value is not a string") + end + + override :validate_options + def validate_options(value) + return unless options && value + return if options.include?(value) + + error("`#{value}` cannot be used because it is not in the list of allowed options") end private - def validate_regex! + override :validate_regex + def validate_regex(value, default) return unless spec.key?(:regex) safe_regex = ::Gitlab::UntrustedRegexp.new(spec[:regex]) - return if safe_regex.match?(actual_value) + return if safe_regex.match?(value) - if value.nil? - error('default value does not match required RegEx pattern') - else - error('provided value does not match required RegEx pattern') - end + error("#{default ? 'default' : 'provided'} value does not match required RegEx pattern") rescue RegexpError error('invalid regular expression') end diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 29beba4774a..90db9d13d85 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -25,13 +25,26 @@ module Gitlab def reserved_claims super.merge({ - iss: Settings.gitlab.base_url, + iss: Feature.enabled?(:oidc_issuer_url) ? Gitlab.config.gitlab.url : Settings.gitlab.base_url, sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", - aud: aud, - user_identities: user_identities + aud: aud }.compact) end + def custom_claims + additional_custom_claims = { + runner_id: runner&.id, + runner_environment: runner_environment, + sha: pipeline.sha, + project_visibility: Gitlab::VisibilityLevel.string_level(project.visibility_level), + user_identities: user_identities + }.compact + + mapper = ClaimMapper.new(project_config, pipeline) + + super.merge(additional_custom_claims).merge(mapper.to_h) + end + def user_identities return unless user&.pass_user_identities_to_ci_jwt @@ -43,17 +56,6 @@ module Gitlab end end - def custom_claims - mapper = ClaimMapper.new(project_config, pipeline) - - super.merge({ - runner_id: runner&.id, - runner_environment: runner_environment, - sha: pipeline.sha, - project_visibility: Gitlab::VisibilityLevel.string_level(project.visibility_level) - }).merge(mapper.to_h) - end - def project_config Gitlab::Ci::ProjectConfig.new( project: project, diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb index 3dc73544208..35548358c57 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb @@ -15,7 +15,8 @@ module Gitlab SUPPORTED_SCHEMA_VERSION = '1' GITLAB_PREFIX = 'gitlab:' SOURCE_PARSERS = { - 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning + 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning, + 'container_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::ContainerScanning }.freeze SUPPORTED_PROPERTIES = %w[ meta:schema_version @@ -24,6 +25,10 @@ module Gitlab dependency_scanning:source_file:path dependency_scanning:package_manager:name dependency_scanning:language:name + container_scanning:image:name + container_scanning:image:tag + container_scanning:operating_system:name + container_scanning:operating_system:version ].freeze def self.parse_source(...) diff --git a/lib/gitlab/ci/parsers/sbom/source/base_source.rb b/lib/gitlab/ci/parsers/sbom/source/base_source.rb new file mode 100644 index 00000000000..744555aa25a --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/source/base_source.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Source + class BaseSource + REQUIRED_ATTRIBUTES = [].freeze + + def self.source(...) + new(...).source + end + + def initialize(data) + @data = data + end + + def source + return unless required_attributes_present? + + ::Gitlab::Ci::Reports::Sbom::Source.new( + type: type, + data: data + ) + end + + private + + attr_reader :data + + # Implement in child class + # returns a symbol of the source type + def type; end + + def required_attributes_present? + self.class::REQUIRED_ATTRIBUTES.all? do |keys| + data.dig(*keys).present? + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb new file mode 100644 index 00000000000..33f9631c424 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Source + class ContainerScanning < BaseSource + REQUIRED_ATTRIBUTES = [ + %w[image name], + %w[image tag] + ].freeze + + OPERATING_SYSTEM_ATTRIBUTES = [ + %w[operating_system name], + %w[operating_system version] + ].freeze + + private + + def type + :container_scanning + end + + def required_attributes_present? + operating_system_attributes_valid? && super + end + + def operating_system_attributes_valid? + return true if data['operating_system'].blank? + + OPERATING_SYSTEM_ATTRIBUTES.all? do |keys| + data.dig(*keys).present? + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb index c76a4309779..fc5a7606e39 100644 --- a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -5,36 +5,15 @@ module Gitlab module Parsers module Sbom module Source - class DependencyScanning + class DependencyScanning < BaseSource REQUIRED_ATTRIBUTES = [ %w[input_file path] ].freeze - def self.source(...) - new(...).source - end - - def initialize(data) - @data = data - end - - def source - return unless required_attributes_present? - - ::Gitlab::Ci::Reports::Sbom::Source.new( - type: :dependency_scanning, - data: data - ) - end - private - attr_reader :data - - def required_attributes_present? - REQUIRED_ATTRIBUTES.all? do |keys| - data.dig(*keys).present? - end + def type + :dependency_scanning end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 9032faa66d4..be6c6c2558b 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -141,7 +141,7 @@ module Gitlab project_id: @project.id, found_by_pipeline: report.pipeline, vulnerability_finding_signatures_enabled: @signatures_enabled, - cvss: data['cvss'] || [] + cvss: data['cvss_vectors'] || [] ) ) end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index e39482481c7..e2a8044b708 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6] + cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7] }.freeze VERSIONS_TO_REMOVE_IN_17_0 = %w[].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..e27096d071f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json @@ -0,0 +1,1085 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json new file mode 100644 index 00000000000..94c3b3fc919 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json @@ -0,0 +1,1017 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..e15fbc3ed56 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json @@ -0,0 +1,975 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json new file mode 100644 index 00000000000..8a9519f442f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json @@ -0,0 +1,1380 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json new file mode 100644 index 00000000000..83b3537b5f1 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json @@ -0,0 +1,1083 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json new file mode 100644 index 00000000000..3597ed169d5 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json @@ -0,0 +1,970 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json new file mode 100644 index 00000000000..afd80ca916b --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json @@ -0,0 +1,994 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 43fb5cdbbe6..b8c8cfa802c 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -6,7 +6,7 @@ module Gitlab module Build class Cancelable < Status::Extended def has_action? - can?(user, :update_build, subject) + can?(user, :cancel_build, subject) end def action_icon diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 1ba78b357e5..fe4f6db9549 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -61,6 +61,8 @@ module Gitlab 'running' elsif any_of?(:waiting_for_resource) 'waiting_for_resource' + elsif any_of?(:waiting_for_callback) + 'waiting_for_callback' elsif any_of?(:manual) 'manual' elsif any_of?(:scheduled) diff --git a/lib/gitlab/ci/status/waiting_for_callback.rb b/lib/gitlab/ci/status/waiting_for_callback.rb new file mode 100644 index 00000000000..0184a910ede --- /dev/null +++ b/lib/gitlab/ci/status/waiting_for_callback.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + class WaitingForCallback < Status::Core + def text + s_('CiStatusText|Waiting') + end + + def label + s_('CiStatusLabel|waiting for callback') + end + + def icon + 'status_pending' + end + + def favicon + 'favicon_status_pending' + end + + def group + 'waiting-for-callback' + end + + def details_path + nil + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml index 356062c734e..324128678de 100644 --- a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml @@ -12,9 +12,9 @@ include: docker-build: variables: - COSIGN_YES: "true" # Used by Cosign to skip confirmation prompts for non-destructive operations + COSIGN_YES: "true" # Used by Cosign to skip confirmation prompts for non-destructive operations id_tokens: - SIGSTORE_ID_TOKEN: # Used by Cosign to get certificate from Fulcio + SIGSTORE_ID_TOKEN: # Used by Cosign to get certificate from Fulcio aud: sigstore after_script: - apk add --update cosign diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 2d04c97b32e..6898923bc53 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.44.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.49.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 2d04c97b32e..6898923bc53 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.44.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.49.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 4d53b92763a..7d923245d79 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 390824e8e49..0f8d5bf6d8f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index a9681c0f927..e29d18ea45a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml index d7a6104082d..4c89497fa97 100644 --- a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml @@ -46,13 +46,30 @@ kaniko-build: # Write credentials to access Gitlab Container Registry within the runner/ci - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json # Build and push the container. To disable push add --no-push - - DOCKERFILE_PATH=${DOCKERFILE_PATH:-"$KANIKO_BUILD_CONTEXT/Dockerfile"} + # Both Dockerfile and Containerfile are supported. For retrocompatibility, if both files are present, Dockerfile will be used. + - | + if [ -z "$DOCKERFILE_PATH" ]; then + if [ -f "$KANIKO_BUILD_CONTEXT/Dockerfile" ]; then + DOCKERFILE_PATH="$KANIKO_BUILD_CONTEXT/Dockerfile" + elif [ -n "$CONTAINERFILE_PATH" ]; then + DOCKERFILE_PATH="$CONTAINERFILE_PATH" + elif [ -f "$KANIKO_BUILD_CONTEXT/Containerfile" ]; then + DOCKERFILE_PATH="$KANIKO_BUILD_CONTEXT/Containerfile" + else \ + echo "No suitable configuration for the build context have been found. Please check your configuration." + exit 1 + fi + fi + - echo $DOCKERFILE_PATH - /kaniko/executor --context $KANIKO_BUILD_CONTEXT --dockerfile $DOCKERFILE_PATH --destination $IMAGE_TAG $KANIKO_ARGS - # Run this job in a branch/tag where a Dockerfile exists + # Run this job in a branch/tag where a Containerfile/Dockerfile exists rules: - exists: + - Containerfile - Dockerfile - # custom Dockerfile path + # custom Containerfile/Dockerfile path + # If both variables are set, DOCKERFILE_PATH will be used - if: $DOCKERFILE_PATH + - if: $CONTAINERFILE_PATH # custom build context without an explicit Dockerfile path - if: $KANIKO_BUILD_CONTEXT != $CI_PROJECT_DIR diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index d2b929cf995..0ba4f9715c5 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -50,7 +50,11 @@ variables: - gitlab-terraform plan-json resource_group: ${TF_STATE_NAME} artifacts: - # The next line, which disables public access to pipeline artifacts, may not be available everywhere. + # Terraform's cache files can include secrets which can be accidentally exposed. + # Please exercise caution when utilizing secrets in your Terraform infrastructure and + # consider limiting access to artifacts or take other security measures to protect sensitive information. + # + # The next line, which disables public access to pipeline artifacts, is not available on GitLab.com. # See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic public: false paths: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index c1a90955f7f..8c9e0a329dd 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -19,6 +19,7 @@ browser_performance: SITESPEED_IMAGE: sitespeedio/sitespeed.io SITESPEED_VERSION: 26.1.0 SITESPEED_OPTIONS: '' + SITESPEED_DOCKER_OPTIONS: '' services: - docker:dind script: @@ -48,7 +49,7 @@ browser_performance: HTTP_PROXY \ NO_PROXY \ ) \ - --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + $SITESPEED_DOCKER_OPTIONS --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index adc92fde5ae..3f4c0c53850 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -19,6 +19,7 @@ browser_performance: SITESPEED_IMAGE: sitespeedio/sitespeed.io SITESPEED_VERSION: latest SITESPEED_OPTIONS: '' + SITESPEED_DOCKER_OPTIONS: '' services: - docker:dind script: @@ -48,7 +49,7 @@ browser_performance: HTTP_PROXY \ NO_PROXY \ ) \ - --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + $SITESPEED_DOCKER_OPTIONS --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb index 4a122c73e80..d3047385c99 100644 --- a/lib/gitlab/ci/yaml_processor/dag.rb +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -17,13 +17,15 @@ module Gitlab def self.check_circular_dependencies!(jobs) new(jobs).tsort - rescue TSort::Cyclic - raise ValidationError, 'The pipeline has circular dependencies' + rescue TSort::Cyclic => e + raise ValidationError, "The pipeline has circular dependencies: #{e.message}" end def tsort_each_child(node, &block) return unless @nodes[node] + raise TSort::Cyclic, "self-dependency: #{node}" if @nodes[node].include?(node) + @nodes[node].each(&block) end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 6207b595fc6..2435d128bf2 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -8,8 +8,7 @@ module Gitlab class Result attr_reader :errors, :warnings, :root_variables, :root_variables_with_prefill_data, - :stages, :jobs, - :workflow_rules, :workflow_name + :stages, :jobs, :workflow_rules, :workflow_name def initialize(ci_config: nil, errors: [], warnings: []) @ci_config = ci_config @@ -124,7 +123,8 @@ module Gitlab trigger: job[:trigger], bridge_needs: job.dig(:needs, :bridge)&.first, release: job[:release], - publish: job[:publish] + publish: job[:publish], + pages: job[:pages] }.compact }.compact end diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb index bfa3112b795..0834fda9cf9 100644 --- a/lib/gitlab/composer/version_index.rb +++ b/lib/gitlab/composer/version_index.rb @@ -48,8 +48,7 @@ module Gitlab end def package_source(package) - use_http_url = package.project.public? || Feature.disabled?(:composer_use_ssh_source_urls, package.project) - git_url = use_http_url ? package.project.http_url_to_repo : package.project.ssh_url_to_repo + git_url = package.project.public? ? package.project.http_url_to_repo : package.project.ssh_url_to_repo { 'type' => 'git', diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 87b7cab3f6d..c7dd11b0432 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -56,8 +56,7 @@ module Gitlab mutually_exclusive_keys = value.try(:keys).to_a & options[:in] if mutually_exclusive_keys.length > 1 - record.errors.add(attribute, "please use only one of the following keys: " + - mutually_exclusive_keys.join(', ')) + record.errors.add(attribute, "these keys cannot be used together: #{mutually_exclusive_keys.join(', ')}") end end end diff --git a/lib/gitlab/config_checker/puma_rugged_checker.rb b/lib/gitlab/config_checker/puma_rugged_checker.rb deleted file mode 100644 index 82c59f3328b..00000000000 --- a/lib/gitlab/config_checker/puma_rugged_checker.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ConfigChecker - module PumaRuggedChecker - extend self - extend Gitlab::Git::RuggedImpl::UseRugged - - def check - notices = [] - - if running_puma_with_multiple_threads? && rugged_enabled_through_feature_flag? - link_start = '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html#performance-caveat-when-using-puma-with-rugged">' - link_end = '</a>' - notices << { - type: 'warning', - message: _('Puma is running with a thread count above 1 and the Rugged '\ - 'service is enabled. This may decrease performance in some environments. '\ - 'See our %{link_start}documentation%{link_end} '\ - 'for details of this issue.') % { link_start: link_start, link_end: link_end } - } - end - - notices - end - end - end -end diff --git a/lib/gitlab/database/dictionary.rb b/lib/gitlab/database/dictionary.rb new file mode 100644 index 00000000000..7b0c8560a26 --- /dev/null +++ b/lib/gitlab/database/dictionary.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class Dictionary + def initialize(file_path) + @file_path = file_path + @data = YAML.load_file(file_path) + end + + def name_and_schema + [key_name, gitlab_schema.to_sym] + end + + def table_name + data['table_name'] + end + + def view_name + data['view_name'] + end + + def milestone + data['milestone'] + end + + def gitlab_schema + data['gitlab_schema'] + end + + def schema?(schema_name) + gitlab_schema == schema_name.to_s + end + + def key_name + table_name || view_name + end + + def validate! + return true unless gitlab_schema.nil? + + raise( + GitlabSchema::UnknownSchemaError, + "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ + "See #{help_page_url}" + ) + end + + private + + attr_reader :file_path, :data + + def help_page_url + # rubocop:disable Gitlab/DocUrl -- link directly to docs.gitlab.com, always + 'https://docs.gitlab.com/ee/development/database/database_dictionary.html' + # rubocop:enable Gitlab/DocUrl + end + end + end +end diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb index 83edf77f37e..18854530278 100644 --- a/lib/gitlab/database/dynamic_model_helpers.rb +++ b/lib/gitlab/database/dynamic_model_helpers.rb @@ -5,7 +5,7 @@ module Gitlab module DynamicModelHelpers BATCH_SIZE = 1_000 - def define_batchable_model(table_name, connection:) + def define_batchable_model(table_name, connection:, primary_key: nil) klass = Class.new(ActiveRecord::Base) do include EachBatch @@ -13,6 +13,7 @@ module Gitlab self.inheritance_column = :_type_disabled end + klass.primary_key = primary_key if connection.primary_keys(table_name).length > 1 klass.connection = connection klass end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 31ceb898eee..ecb45622061 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -31,6 +31,7 @@ module Gitlab '_test_gitlab_main_cell_' => :gitlab_main_cell, '_test_gitlab_main_' => :gitlab_main, '_test_gitlab_ci_' => :gitlab_ci, + '_test_gitlab_jh_' => :gitlab_jh, '_test_gitlab_embedding_' => :gitlab_embedding, '_test_gitlab_geo_' => :gitlab_geo, '_test_gitlab_pm_' => :gitlab_pm, @@ -138,19 +139,19 @@ module Gitlab end def self.deleted_tables_to_schema - @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').to_h + @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').map(&:name_and_schema).to_h end def self.deleted_views_to_schema - @deleted_views_to_schema ||= self.build_dictionary('deleted_views').to_h + @deleted_views_to_schema ||= self.build_dictionary('deleted_views').map(&:name_and_schema).to_h end def self.tables_to_schema - @tables_to_schema ||= self.build_dictionary('').to_h + @tables_to_schema ||= self.build_dictionary('').map(&:name_and_schema).to_h end def self.views_to_schema - @views_to_schema ||= self.build_dictionary('views').to_h + @views_to_schema ||= self.build_dictionary('views').map(&:name_and_schema).to_h end def self.schema_names @@ -159,21 +160,9 @@ module Gitlab def self.build_dictionary(scope) Dir.glob(dictionary_path_globs(scope)).map do |file_path| - data = YAML.load_file(file_path) - - key_name = data['table_name'] || data['view_name'] - - # rubocop:disable Gitlab/DocUrl - if data['gitlab_schema'].nil? - raise( - UnknownSchemaError, - "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ - "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" - ) - end - # rubocop:enable Gitlab/DocUrl - - [key_name, data['gitlab_schema'].to_sym] + dictionary = Dictionary.new(file_path) + dictionary.validate! + dictionary end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index 41044816de9..1d17c2ca608 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -56,6 +56,10 @@ module Gitlab include Gitlab::Database::Migrations::RunnerBackoff::MigrationHelpers end + class V2_2 < V2_1 + include Gitlab::Database::Migrations::MilestoneMixin + end + def self.[](version) version = version.to_s name = "V#{version.tr('.', '_')}" @@ -66,7 +70,7 @@ module Gitlab # The current version to be used in new migrations def self.current_version - 2.1 + 2.2 end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index efcceafda90..a57bce789c7 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -18,8 +18,8 @@ module Gitlab include AsyncConstraints::MigrationHelpers include WraparoundVacuumHelpers - def define_batchable_model(table_name, connection: self.connection) - super(table_name, connection: connection) + def define_batchable_model(table_name, connection: self.connection, primary_key: nil) + super(table_name, connection: connection, primary_key: primary_key) end def each_batch(table_name, connection: self.connection, **kwargs) @@ -821,6 +821,7 @@ module Gitlab primary_key: :id, batch_size: 20_000, sub_batch_size: 1000, + pause_ms: 100, interval: 2.minutes ) @@ -848,6 +849,7 @@ module Gitlab conversions.keys, conversions.values, job_interval: interval, + pause_ms: pause_ms, batch_size: batch_size, sub_batch_size: sub_batch_size) end diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb index 11f1e62e8b9..d1edb739b85 100644 --- a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb +++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb @@ -4,12 +4,16 @@ module Gitlab module Database module MigrationHelpers module ConvertToBigint - # This helper is extracted for the purpose of - # https://gitlab.com/gitlab-org/gitlab/-/issues/392815 - # so that we can test all combinations just once, - # and simplify migration tests. - # - # Once we are done with the PK conversions we can remove this. + INDEX_OPTIONS_MAP = { + unique: :unique, + order: :orders, + opclass: :opclasses, + where: :where, + type: :type, + using: :using, + comment: :comment + }.freeze + def com_or_dev_or_test_but_not_jh? return true if Gitlab.dev_or_test_env? @@ -29,6 +33,78 @@ module Gitlab column.sql_type == 'bigint' && temp_column.sql_type == 'integer' end + + def add_bigint_column_indexes(table_name, int_column_name) + bigint_column_name = convert_to_bigint_column(int_column_name) + + unless column_exists?(table_name.to_s, bigint_column_name) + raise "Bigint column '#{bigint_column_name}' does not exist on #{table_name}" + end + + indexes(table_name).each do |i| + next unless Array(i.columns).join(' ').match?(/\b#{int_column_name}\b/) + + create_bigint_index(table_name, i, int_column_name, bigint_column_name) + end + end + + # default 'index_name' method is not used because this method can be reused while swapping/dropping the indexes + def bigint_index_name(int_column_index_name) + # First 20 digits of the hash is chosen to make sure it fits the 63 chars limit + digest = Digest::SHA256.hexdigest(int_column_index_name).first(20) + "bigint_idx_#{digest}" + end + + private + + def create_bigint_index(table_name, index_definition, int_column_name, bigint_column_name) + index_attributes = index_definition.as_json + index_options = INDEX_OPTIONS_MAP + .transform_values { |key| index_attributes[key.to_s] } + .select { |_, v| v.present? } + + bigint_index_options = create_bigint_options( + index_options, + index_definition.name, + int_column_name, + bigint_column_name + ) + + add_concurrent_index( + table_name, + bigint_index_columns(int_column_name, bigint_column_name, index_definition.columns), + name: bigint_index_options.delete(:name), + ** bigint_index_options + ) + end + + def bigint_index_columns(int_column_name, bigint_column_name, int_index_columns) + if int_index_columns.is_a?(String) + int_index_columns.gsub(/\b#{int_column_name}\b/, bigint_column_name) + else + int_index_columns.map do |column| + column == int_column_name.to_s ? bigint_column_name : column + end + end + end + + def create_bigint_options(index_options, int_index_name, int_column_name, bigint_column_name) + index_options[:name] = bigint_index_name(int_index_name) + index_options[:where]&.gsub!(/\b#{int_column_name}\b/, bigint_column_name) + + # ordering on multiple columns will return a Hash instead of string + index_options[:order] = + if index_options[:order].is_a?(Hash) + index_options[:order].to_h do |column, order| + column = bigint_column_name if column == int_column_name + [column, order] + end + else + index_options[:order]&.gsub(/\b#{int_column_name}\b/, bigint_column_name) + end + + index_options.select { |_, v| v.present? } + end end end end diff --git a/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb index 555efb58606..7f215bc0db7 100644 --- a/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb +++ b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb @@ -13,7 +13,7 @@ module Gitlab # 3. Introduce the migration again for self-managed. # def can_execute_on?(*tables) - return false unless Gitlab.com? || Gitlab.dev_or_test_env? + return false unless Gitlab.com_except_jh? || Gitlab.dev_or_test_env? if wraparound_prevention_on_tables?(tables) Gitlab::AppLogger.info(message: "Wraparound prevention vacuum detected", class: self.class) diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 64cde273a59..3d4ac113bf6 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -72,6 +72,7 @@ module Gitlab batch_max_value: nil, batch_class_name: BATCH_CLASS_NAME, batch_size: BATCH_SIZE, + pause_ms: 100, max_batch_size: nil, sub_batch_size: SUB_BATCH_SIZE, gitlab_schema: nil @@ -105,6 +106,7 @@ module Gitlab column_name: batch_column_name, job_arguments: job_arguments, interval: job_interval, + pause_ms: pause_ms, min_value: batch_min_value, max_value: batch_max_value, batch_class_name: batch_class_name, diff --git a/lib/gitlab/database/migrations/milestone_mixin.rb b/lib/gitlab/database/migrations/milestone_mixin.rb index 10bc0c192e7..7d78f74d237 100644 --- a/lib/gitlab/database/migrations/milestone_mixin.rb +++ b/lib/gitlab/database/migrations/milestone_mixin.rb @@ -19,11 +19,10 @@ module Gitlab end end - def initialize(name = class_name, version = nil, type = nil) - raise MilestoneNotSetError, "Milestone is not set for #{self.class.name}" if milestone.nil? + def initialize(name = self.class.name, version = nil, _type = nil) + raise MilestoneNotSetError, "Milestone is not set for #{name}" if milestone.nil? super(name, version) - @version = Gitlab::Database::Migrations::Version.new(version, milestone, type) end def milestone # rubocop:disable Lint/DuplicateMethods diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index bb70d052e3e..83cd446534c 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -89,6 +89,8 @@ module Gitlab Gitlab::AppLogger.info(message: "Created partition", partition_name: partition.partition_name, table_name: partition.table) + + lock_partitions_for_writes(partition) if should_lock_for_writes? end model.partitioning_strategy.after_adding_partitions @@ -205,6 +207,23 @@ module Gitlab end end end + + def should_lock_for_writes? + Feature.enabled?(:automatic_lock_writes_on_partition_tables, type: :ops) && + Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES && + connection != model.connection + end + strong_memoize_attr :should_lock_for_writes? + + def lock_partitions_for_writes(partition) + table_name = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partition.partition_name}" + Gitlab::Database::LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: @connection_name, + with_retries: !connection.transaction_open? + ).lock_writes + end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 1ce0a44e37f..b486ddb8e76 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -8,7 +8,8 @@ module Gitlab include ::Gitlab::Database::MigrationHelpers include ::Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers - ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze + ALLOWED_TABLES = %w[audit_events web_hook_logs merge_request_diff_files merge_request_diff_commits].freeze + ERROR_SCOPE = 'table partitioning' MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable" @@ -16,6 +17,60 @@ module Gitlab BATCH_INTERVAL = 2.minutes.freeze BATCH_SIZE = 50_000 SUB_BATCH_SIZE = 2_500 + PARTITION_BUFFER = 6 + MIN_ID = 1 + + # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a int/bigint column. + # One partition is created per partition_size between 1 and MAX(column_name). Also installs a trigger on + # the original table to copy writes into the partitioned table. To copy over historic data from before creation + # of the partitioned table, use the `enqueue_partitioning_data_migration` helper in a post-deploy migration. + # Note: If the original table is empty the system creates 6 partitions in the new table. + # + # A copy of the original table is required as PG currently does not support partitioning existing tables. + # + # Example: + # + # partition_table_by_int_range :merge_request_diff_commits, :merge_request_diff_id, partition_size: 500, primary_key: ['merge_request_diff_id', 'relative_order'] + # + # Options are: + # :partition_size - a int specifying the partition size + # :primary_key - a array specifying the primary query of the new table + # + # Note: The system always adds a buffer of 6 partitions. + def partition_table_by_int_range(table_name, column_name, partition_size:, primary_key:) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + assert_table_is_allowed(table_name) + + assert_not_in_transaction_block(scope: ERROR_SCOPE) + + current_primary_key = Array.wrap(connection.primary_key(table_name)) + raise "primary key not defined for #{table_name}" if current_primary_key.blank? + + partition_column = find_column_definition(table_name, column_name) + raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil? + + primary_key = Array.wrap(primary_key).map(&:to_s) + raise "the partition column must be part of the primary key" unless primary_key.include?(column_name.to_s) + + primary_key_objects = connection.columns(table_name).select { |column| primary_key.include?(column.name) } + + raise 'partition_size must be greater than 1' unless partition_size > 1 + + max_id = Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection.with_suppressed do + define_batchable_model(table_name, connection: connection).maximum(column_name) || partition_size * PARTITION_BUFFER + end + end + + partitioned_table_name = make_partitioned_table_name(table_name) + + with_lock_retries do + create_range_id_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key_objects) + create_int_range_partitions(partitioned_table_name, partition_size, MIN_ID, max_id) + create_trigger_to_sync_tables(table_name, partitioned_table_name, current_primary_key) + end + end # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column. # One partition is created per month between the given `min_date` and `max_date`. Also installs a trigger on @@ -332,6 +387,34 @@ module Gitlab connection.columns(table).find { |c| c.name == column.to_s } end + def create_range_id_partitioned_copy(source_table_name, partitioned_table_name, partition_column, primary_keys) + if table_exists?(partitioned_table_name) + Gitlab::AppLogger.warn "Partitioned table not created because it already exists" \ + " (this may be due to an aborted migration or similar): table_name: #{partitioned_table_name} " + return + end + + tmp_partitioning_column_name = "#{partition_column.name}_tmp" + + temporary_columns = primary_keys.map { |key| "#{key.name}_tmp" }.join(", ") + temporary_columns_statement = build_temporary_columns_statement(primary_keys) + + transaction do + execute(<<~SQL) + CREATE TABLE #{partitioned_table_name} ( + LIKE #{source_table_name} INCLUDING ALL EXCLUDING INDEXES, + #{temporary_columns_statement}, + PRIMARY KEY (#{temporary_columns}) + ) PARTITION BY RANGE (#{tmp_partitioning_column_name}) + SQL + + primary_keys.each do |key| + remove_column(partitioned_table_name, key.name) + rename_column(partitioned_table_name, "#{key.name}_tmp", key.name) + end + end + end + def create_range_partitioned_copy(source_table_name, partitioned_table_name, partition_column, primary_key) if table_exists?(partitioned_table_name) Gitlab::AppLogger.warn "Partitioned table not created because it already exists" \ @@ -382,6 +465,20 @@ module Gitlab end end + def create_int_range_partitions(table_name, partition_size, min_id, max_id) + lower_bound = min_id + upper_bound = min_id + partition_size + + end_id = max_id + PARTITION_BUFFER * partition_size # Adds a buffer of 6 partitions + + while lower_bound < end_id + create_range_partition_safely("#{table_name}_#{lower_bound}", table_name, lower_bound, upper_bound) + + lower_bound += partition_size + upper_bound += partition_size + end + end + def to_sql_date_literal(date) connection.quote(date.strftime('%Y-%m-%d')) end @@ -411,19 +508,23 @@ module Gitlab return end + unique_key = Array.wrap(unique_key) + delimiter = ",\n " column_names = connection.columns(partitioned_table_name).map(&:name) set_statements = build_set_statements(column_names, unique_key) insert_values = column_names.map { |name| "NEW.#{name}" } + delete_where_statement = unique_key.map { |unique_key| "#{unique_key} = OLD.#{unique_key}" }.join(' AND ') + update_where_statement = unique_key.map { |unique_key| "#{partitioned_table_name}.#{unique_key} = NEW.#{unique_key}" }.join(' AND ') create_trigger_function(name, replace: false) do <<~SQL IF (TG_OP = 'DELETE') THEN - DELETE FROM #{partitioned_table_name} where #{unique_key} = OLD.#{unique_key}; + DELETE FROM #{partitioned_table_name} where #{delete_where_statement}; ELSIF (TG_OP = 'UPDATE') THEN UPDATE #{partitioned_table_name} SET #{set_statements.join(delimiter)} - WHERE #{partitioned_table_name}.#{unique_key} = NEW.#{unique_key}; + WHERE #{update_where_statement}; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO #{partitioned_table_name} (#{column_names.join(delimiter)}) VALUES (#{insert_values.join(delimiter)}); @@ -433,8 +534,16 @@ module Gitlab end end + def build_temporary_columns_statement(columns) + columns.map do |column| + type = column.name == 'id' || column.name.end_with?('_id') ? 'bigint' : column.sql_type + + "#{column.name}_tmp #{type} NOT NULL" + end.join(", ") + end + def build_set_statements(column_names, unique_key) - column_names.reject { |name| name == unique_key }.map { |name| "#{name} = NEW.#{name}" } + column_names.reject { |name| unique_key.include?(name) }.map { |name| "#{name} = NEW.#{name}" } end def create_sync_trigger(table_name, trigger_name, function_name) diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb index eb55ebc7619..c2f94b7b0e6 100644 --- a/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb @@ -8,7 +8,7 @@ module Gitlab class PartitioningRoutingAnalyzer < Database::QueryAnalyzers::Base RoutingTableNotUsedError = Class.new(QueryAnalyzerError) - ENABLED_TABLES = %w[ci_builds_metadata].freeze + ENABLED_TABLES = %w[ci_builds ci_builds_metadata].freeze class << self def enabled? diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb new file mode 100644 index 00000000000..583aceba098 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch < Base + SetOperatorStarError = Class.new(QueryAnalyzerError) + + DETECT_REGEX = /.*SELECT.+(UNION|EXCEPT|INTERSECT)/i + + class << self + def enabled? + ::Feature::FlipperFeature.table_exists? && + Feature.enabled?(:query_analyzer_gitlab_schema_metrics, type: :ops) + end + + def analyze(parsed) + return unless requires_detection?(parsed.sql) + + # Only handle SELECT queries. + parsed.pg.tree.stmts.each do |stmt| + select_stmt = next_select_stmt(stmt) + next unless select_stmt + + types = SelectStmt.new(select_stmt).types + + raise SetOperatorStarError if types.any?(Type::INVALID) + end + end + + private + + def next_select_stmt(node) + return unless node.stmt.respond_to?(:select_stmt) + + node.stmt.select_stmt + end + + # This not entirely correct and will run true on `SELECT union_station, ...` + def requires_detection?(sql) + sql.match DETECT_REGEX + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb new file mode 100644 index 00000000000..87120b8ffce --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + # Columns refer to table columns produced by queries and parts of queries. + # If we have `SELECT namespaces.id` then `id` is a column. But, we can also have + # `WHERE namespaces.id > 10` and `id` is also a column. + # + # In static analysis of a SQL query a column source can be ambiguous. + # Such as in `SELECT id FROM users, namespaces. In such cases we assume `id` could come from either `users` or + # `namespaces`. + class Columns + class << self + # Determine the type of each column in the select statement. + # Returns a Set object containing a Types enum. + # When an error is found parsing will return immediately. + def types(select_stmt) + # Forward through any errors when the column refers to a part of the SQL query that is known to include + # errors. For example, the column may refer to a column from a CTE that was invalid. + return Set.new([Type::INVALID]) if References.errors?(select_stmt.all_references) + + types = Set.new + + # Resolve the type of reference for each target in the select statement. + target_list = select_stmt.node.target_list + targets = target_list.map(&:res_target) + targets.each do |target| + target_type = get_target_type(target, select_stmt) + + # A NULL target is of the form: + # SELECT NULL::namespaces FROM namespaces + types += if Targets.null?(target) + # Maintain any errors but otherwise ignore this target. + target_type & [Type::INVALID] + else + target_type + end + end + + types + end + + private + + def get_target_type(target, select_stmt) + target_ref_names = Targets.reference_names(target, select_stmt) + + resolved_refs = References.resolved(select_stmt.all_references) + + # Cross reference column references with resolved references. + # A resolved reference is part of a SQL query that we were able to analyze already. + # A CTE or sub-query would be such a case. The only non-resolvable reference is a table. + all_resolved = (target_ref_names - resolved_refs.keys).empty? + + # Is this target `*` such as `SELECT *`. + a_star = Targets.a_star?(target) + + if all_resolved + # Defer to the reference source types. + col_refs = resolved_refs.slice(*target_ref_names) + .values + .reduce(:union) || Set.new + + if a_star + # When * the target forwards through the types of the references. + col_refs + else + # When not * the column is static, but we also forward through any nested errors. + (col_refs.to_a & [Type::INVALID]) << Type::STATIC + end + elsif a_star + # This is a * on a table. The * lookup occurs dynamically during query runtime and will + # change when the table schema changes. + [Type::DYNAMIC] + else + # This references a column on a table or intermediate result set such as: + # SELECT namespaces.id FROM namespaces + # + # or: + # WITH some_cte AS ( ... ) SELECT some_cte.id FROM some_cte + [Type::STATIC] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb new file mode 100644 index 00000000000..0ab58ff7c6f --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# The CTE in a SELECT can reference CTEs defined by the current scope, but also CTEs defined by earlier scopes. +# With the following query as an example: +# +# WITH some_cte AS (select 1) +# SELECT * +# FROM (SELECT * FROM some_cte) subquery +# +# The CTE some_cte is visible from within the subquery scope. +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class CommonTableExpressions + class << self + # Convert CTEs available within this SELECT statement into a set of References. + # + # @param [PgQuery::Node] node The PgQuery SELECT statement node containing the CTEs. + # @param [References] cte_refs Inherited CTEs from scopes that wrap this SELECT statement. + def references(node, cte_refs) + return cte_refs if node&.with_clause.nil? + + refs = cte_refs.dup + + node.with_clause.ctes.each do |cte| + cte_name = name(cte) + cte_select_stmt = select_stmt(cte) + + # Resolve the CTE type to dynamic/static/error. + refs[cte_name] = if node.with_clause.recursive + # Recursive CTEs need special handling to avoid infinite loops. + recursive_refs(cte_refs, cte_name, cte_select_stmt) + else + SelectStmt.new(cte_select_stmt, cte_refs).types + end + end + + refs + end + + private + + def name(cte) + cte.common_table_expr.ctename + end + + def select_stmt(cte) + cte.common_table_expr.ctequery.select_stmt + end + + # Return whether the recursive CTE is dynamic/static/error. + def recursive_refs(cte_refs, cte_name, select_stmt) + # Resolve the non-recursive term before the recursive term. + larg_select_stmt = SelectStmt.new(select_stmt.larg, cte_refs) + larg_type = larg_select_stmt.types + new_cte_refs = cte_refs.merge({ cte_name => larg_type }) + + # Now we can resolve the recursive side. + rarg_type = SelectStmt.new(select_stmt.rarg, new_cte_refs).types + + final_type = larg_type | rarg_type + if final_type.count > 1 + final_type | [Type::INVALID] + else + final_type + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb new file mode 100644 index 00000000000..c205243694a --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class Froms + class << self + # Parse the FROM part of the SELECT. Construct a mapping of FROM names to their PgQuery node. Recurse any + # sub-queries and resolve to a Set of dynamic/static/error. + # + # Whenever a node is aliased, use the alias name as it's reference and ignore it's original name. + # + # For example, given: + # + # SELECT id + # FROM namespaces ns + # + # Return a Hash of { 'ns' => NodeObject } + # + # @param [PgQuery::Node] node The PgQuery SELECT statement node containing the CTEs. + # @param [References] cte_refs Inherited CTEs from scopes that wrap this SELECT statement. + # + # @return [Hash] name of from references mapped to the node that defines their value, or Set if already + # resolved. + def references(node, cte_refs) + refs = {} + + return refs unless node + + node.from_clause.each do |from| + range_var = Node.dig(from, :range_var) + range_sq = Node.dig(from, :range_subselect) + + if range_var + # FROM some_table + # FROM some_table some_alias + refs.merge!(range_var_reference(range_var, cte_refs)) + elsif Node.dig(from, :join_expr) + # FROM some_table INNER JOIN other_table + range_vars = Node.locate_descendants(from, :range_var) + range_vars.each do |range_var| + refs.merge!(range_var_reference(range_var, cte_refs)) + end + elsif range_sq + # FROM (SELECT ...) some_alias + select_stmt = Node.dig(range_sq, :subquery, :select_stmt) + refs[range_sq.alias.aliasname] = SelectStmt.new(select_stmt, cte_refs).types + end + end + + refs + end + + private + + def range_var_reference(range_var, cte_refs) + relname = Node.dig(range_var, :alias, :aliasname) || range_var.relname + reference = cte_refs[range_var.relname] || range_var + + { relname => reference } + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb new file mode 100644 index 00000000000..ee41eaa9d3a --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + # The Node class allows us to traverse PgQuery nodes with tree like semantics. + # + # This class balances convenience and performance. The PgQuery nodes are Google::Protobuf::MessageExts which + # contain a dynamic set of attributes known as fields. Accessing these fields can cause performance problems + # due to the large volume of iterable fields. + # + # When possible use #dig over the *descendant* methods. + # + # The filter available to each method reduces the traversed attributes. The default filter only traverses nodes + # required to parse for set operator mismatches. + class Node + class << self + include Gitlab::Utils::StrongMemoize + + # The default nodes help speed up traversal. Traversal of other nodes can greatly affect performance. + DEFAULT_NODES = %i[ + a_star + alias + args + column_ref + fields + func_call + join_expr + larg + range_subselect + range_var + rarg + res_target + subquery + val + ].freeze + DEFAULT_FIELD_FILTER = ->(field) { field.is_a?(Integer) || DEFAULT_NODES.include?(field) }.freeze + + # Recurse through children. + # The block will yield the child node and the name of that node. + # Calling without a block will return an Enumerator. + def descendants(node, filter: DEFAULT_FIELD_FILTER, &blk) + if blk + children(node, filter: filter) do |child_node, child_field| + yield(child_node, child_field) + + descendants(child_node, filter: filter, &blk) + end + nil + else + enum_for(:descendants, node, filter: filter, &blk) + end + end + + # Return the first node that matches the field. + def locate_descendant(node, field, filter: DEFAULT_FIELD_FILTER) + descendants(node, filter: filter).find { |_, child_field| child_field == field }&.first + end + + # Return all nodes that match the field. + def locate_descendants(node, field, filter: DEFAULT_FIELD_FILTER) + descendants(node, filter: filter).select { |_, child_field| child_field == field }.map(&:first) + end + + # Like Hash#dig, traverse attributes in sequential order and return the final value. + # Return nil if any of the fields are not available. + def dig(node, *attrs) + obj = node + attrs.each do |attr| + if obj.respond_to?(attr) + obj = obj.public_send(attr) # rubocop:disable GitlabSecurity/PublicSend + else + obj = nil + break + end + end + obj + end + + private + + # Interface with a PgQuery result as though it was a tree node. + # All elements in a PgQuery result are ancestors of Google::Protobuf::AbstractMessage + # + # Based off PgQuery's treewalker https://github.com/pganalyze/pg_query/blob/main/lib/pg_query/treewalker.rb + def children(node, filter: DEFAULT_FIELD_FILTER, &_blk) + attributes = case node + when Google::Protobuf::MessageExts + descriptor_fields(node.class.descriptor) + when Google::Protobuf::RepeatedField + node.count.times.to_a + end + + attributes.select(&filter).each do |attr| + attr_key = attr.is_a?(Symbol) ? attr.to_s : attr + child = node[attr_key] + next if child.nil? + + yield(child, attr) + end + end + + def descriptor_fields(descriptor) + strong_memoize_with(:descriptor_fields, descriptor) do + keys = [] + descriptor.each do |field| + keys << field.name.to_sym + end + keys + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb new file mode 100644 index 00000000000..ba6e9752905 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +# References form the base data structure of the PreventSetOperatorMismatch query analyzer. +# +# A reference refers to a table, CTE, or other named entity in a SQL query. References are a set of mappings between the +# name of the reference and the PgQuery node that represents that reference in the parsed tree. +# +# Given the SQL: +# +# WITH some_cte AS (SELECT 1) +# SELECT * +# FROM some_cte, users, namespace ns +# +# The reference names would be `some_cte`, `users`, `ns`. The reference values are the nodes in the parse tree that +# represent that reference: +# - some_cte: the common table expression node +# - users: nil, being a table +# - ns: nil, being a table, but importantly we use the alias name +# +# A reference can be "resolved". A resolved reference value is a Set of Types. The reference value was a select +# statement that has since been parsed. +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class References + class << self + # All references that have already been parsed to determine static/dynamic/error state. + # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types. + def resolved(refs) + refs.select { |_name, ref| ref.is_a?(Set) } + end + + # All references that have not been parsed to determine static/dynamic/error state. + # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types. + def unresolved(refs) + refs.select { |_name, ref| unresolved?(ref) } + end + + # Whether any currently resolved references have resulted in an error state. + # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types. + def errors?(refs) + resolved(refs).any? { |_, values| values.include?(Type::INVALID) } + end + + private + + def resolved?(ref) + ref.is_a?(Set) + end + + def unresolved?(ref) + !resolved?(ref) && table?(ref) + end + + def table?(ref) + !ref.is_a?(PgQuery::RangeVar) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb new file mode 100644 index 00000000000..bdbcc49f63f --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class SelectStmt + include Gitlab::Utils::StrongMemoize + + attr_reader :node, :cte_references, :all_references + + # @param [PgQuery::SelectStmt] node The PgQuery node of the select statement. + # @param [Hash] inherited_cte_references CTE References available to the select statement. + def initialize(node, inherited_cte_references = {}) + @node = node + @cte_references = CommonTableExpressions.references(node, inherited_cte_references) + from_references = Froms.references(node, cte_references) + @all_references = from_references.merge(cte_references) + end + + # returns Set of Types. + # + # STATIC - queries that don't require a database schema lookup. E.g. `SELECT users.id FROM users` + # DYNAMIC - queries that require a database schema lookup. E.g. `SELECT users.* FROM users` + # INVALID - set operator queries that mix static and dynamic queries. + def types + if set_operator? + resolve_set_operator_select_types + else + resolve_normal_select_types + end + end + + private + + # Standard SELECT, not a set operator (UNION/INTERSECT/EXCEPT) + def resolve_normal_select_types + # Cross reference resolved sources with what is requested by the SELECT. + types = Columns.types(self) + + # Mixed dynamic and static queries can be normalized to simply dynamic queries for the purposes of + # detecting mismatched set operator parts. + types.delete(Type::STATIC) if types.include?(Type::DYNAMIC) + + types + end + + # Set operator (UNION/INTERSECT/EXCEPT) + def resolve_set_operator_select_types + types = Set.new + + # Recurse each set operator part as a SELECT statement. + # select statement part => type + set_operator_parts do |part| + types += SelectStmt.new(part, cte_references).types + end + + types << Type::INVALID if types.count > 1 + + types + end + + def set_operator? + !(node.respond_to?(:op) && node.op == :SETOP_NONE) + end + + SET_OPERATOR_PART_LOCATIONS = %i[larg rarg].freeze + private_constant :SET_OPERATOR_PART_LOCATIONS + + def set_operator_parts(&_blk) + return unless node + + yield node if node.op == :SETOP_NONE + yield node.larg if node.larg && node.larg.op == :SETOP_NONE + yield node.rarg if node.rarg && node.rarg.op == :SETOP_NONE + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb new file mode 100644 index 00000000000..99db368efcb --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +# Targets refer to SELECT columns but also JOIN fields, etc. +# A target can have a qualifying reference to some other entity like a table or CTE. +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class Targets + class << self + # Return the reference names used by the given target. + # + # For example: + # `SELECT users.id` would return ['users'] + # `SELECT * FROM users, namespaces` would return ['users', 'namespaces'] + def reference_names(target, select_stmt) + # Parse all targets to determine what is referenced. + fields = fields(target) + case fields.count + when 0 + literal_ref_names(target, select_stmt) + when 1 + unqualified_ref_names(fields, select_stmt) + else + # The target is qualified such as SELECT reference.id + field_ref = fields[fields.count - 2] + [field_ref.string.sval] + end + end + + # True when `SELECT *` + def a_star?(target) + Node.locate_descendant(target, :a_star) + end + + # Null targets are used to produce "polymorphic" query result sets that can be aggregated through a UNION + # without having to worry about mismatched columns. + # + # A null target would be something like: + # SELECT NULL::namespaces FROM namespaces + def null?(target) + target&.val&.type_cast&.arg&.a_const&.isnull + end + + private + + def literal_ref_names(target, select_stmt) + # The target is unqualified and is not part of a column_ref, such as in `SELECT 1`. + # These include targets like literals, functions, and subselects. + sub_select_stmt = subselect_select_stmt(target) + if sub_select_stmt + name = (target.name.presence || "loc_#{target.location}") + # The select is anonymous, so we provide a name. + k = "#{name}_subselect" + # Force parsing of the select. + # We don't care about the static/dynamic nature in this case, but we do need to parse for + # any nested error states. + sub_select = SelectStmt.new(sub_select_stmt, select_stmt.cte_references) + select_stmt.all_references[k] = sub_select.types + [k] + else + # TODO we need to parse function references. Assuming no sources for now. + # https://gitlab.com/gitlab-org/gitlab/-/issues/428102 + [] + end + end + + def unqualified_ref_names(fields, select_stmt) + # The target is unqualified, but is part of a column_ref. + # E.g. `SELECT id FROM namespaces` or `SELECT namespaces FROM namespaces` + + # Otherwise, check all FROM/JOIN/CTE entries. + field = fields[0] + field_sval = field&.string&.sval + if field_sval && select_stmt.all_references.key?(field_sval) + # SELECT some_table_name + [field.string.sval] + else + # SELECT * + # SELECT some_column + select_stmt.all_references.keys + end + end + + def fields(target) + Node.locate_descendants(target, :fields).flatten + end + + def subselect_select_stmt(target) + Node.dig(target, :val, :sub_link, :subselect, :select_stmt) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb new file mode 100644 index 00000000000..5988f963827 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + # An enumerated set of constants that represent the state of the parse. + module Type + STATIC = :static + DYNAMIC = :dynamic + INVALID = :invalid + end + end + end + end +end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb index 6da76803f7c..e110fc44b7b 100644 --- a/lib/gitlab/database/schema_cache_with_renamed_table.rb +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -11,26 +11,26 @@ module Gitlab clear_renamed_tables_cache! end - def clear_data_source_cache!(name) - super(name) + def clear_data_source_cache!(connection, table_name) + super(connection, table_name) clear_renamed_tables_cache! end - def primary_keys(table_name) - super(underlying_table(table_name)) + def primary_keys(connection, table_name) + super(connection, underlying_table(table_name)) end - def columns(table_name) - super(underlying_table(table_name)) + def columns(connection, table_name) + super(connection, underlying_table(table_name)) end - def columns_hash(table_name) - super(underlying_table(table_name)) + def columns_hash(connection, table_name) + super(connection, underlying_table(table_name)) end - def indexes(table_name) - super(underlying_table(table_name)) + def indexes(connection, table_name) + super(connection, underlying_table(table_name)) end private @@ -40,7 +40,7 @@ module Gitlab end def renamed_tables_cache - @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| + @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, _new_name| connection.view_exists?(old_name) end end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb b/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb new file mode 100644 index 00000000000..acc9bbd0aff --- /dev/null +++ b/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This is a legacy extension targeted at Rails versions prior to 7.1 + # In Rails 7.1, the method parameters have been changed to (connection, table_name) + module SchemaCacheWithRenamedTableLegacy + # Override methods in ActiveRecord::ConnectionAdapters::SchemaCache + + def clear! + super + + clear_renamed_tables_cache! + end + + def clear_data_source_cache!(name) + super(name) + + clear_renamed_tables_cache! + end + + def primary_keys(table_name) + super(underlying_table(table_name)) + end + + def columns(table_name) + super(underlying_table(table_name)) + end + + def columns_hash(table_name) + super(underlying_table(table_name)) + end + + def indexes(table_name) + super(underlying_table(table_name)) + end + + private + + def underlying_table(table_name) + renamed_tables_cache.fetch(table_name, table_name) + end + + def renamed_tables_cache + @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, _new_name| + connection.view_exists?(old_name) + end + end + + def clear_renamed_tables_cache! + @renamed_tables = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index aa880b709fe..608dea9e3c5 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -3,7 +3,7 @@ module Gitlab module Database class TablesLocker - GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo gitlab_jh].freeze def initialize(logger: nil, dry_run: false, include_partitions: true) @logger = logger diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index f91146fff3d..5394dee6fec 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -3,7 +3,7 @@ module Gitlab module Database class TablesTruncate - GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding].freeze + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding gitlab_jh].freeze def initialize(database_name:, min_batch_size: 5, logger: nil, until_table: nil, dry_run: false) @database_name = database_name diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb index 60b3a1738f1..3d1f7ab86b3 100644 --- a/lib/gitlab/discussions_diff/file_collection.rb +++ b/lib/gitlab/discussions_diff/file_collection.rb @@ -25,8 +25,9 @@ module Gitlab # # - Highlight cache is written just for uncached diff files # - The cache content is not updated (there's no need to do so) - def load_highlight - ids = highlightable_collection_ids + # - Load only the related diff note ids + def load_highlight(diff_note_ids: nil) + ids = highlightable_collection_ids(diff_note_ids) return if ids.empty? cached_content = read_cache(ids) @@ -47,8 +48,13 @@ module Gitlab private - def highlightable_collection_ids - each.with_object([]) { |file, memo| memo << file.id unless file.resolved_at } + def highlightable_collection_ids(diff_note_ids) + each.with_object([]) do |file, memo| + # We ignore if file is resolved, or not part of the highlight requested notes + next if file.resolved_at || (diff_note_ids.present? && diff_note_ids.exclude?(file.diff_note_id)) + + memo << file.id + end end def read_cache(ids) diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index ebc4e9c2c8c..e3249b143c8 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -38,7 +38,7 @@ module Gitlab create_issue_or_note if from_address - add_email_participant + add_email_participants send_thank_you_email unless reply_email? end end @@ -215,6 +215,10 @@ module Gitlab end strong_memoize_attr :to_address + def cc_addresses + mail.cc || [] + end + def can_handle_legacy_format? project_path && project_path.include?('/') && !mail_key.include?('+') end @@ -223,11 +227,33 @@ module Gitlab Users::Internal.support_bot end - def add_email_participant + def add_email_participants return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) @issue.issue_email_participants.create(email: from_address) + + add_external_participants_from_cc + end + + def add_external_participants_from_cc + return if project.service_desk_setting.nil? + return unless project.service_desk_setting.add_external_participants_from_cc? + + cc_addresses.each do |email| + next if service_desk_addresses.include?(email) + + @issue.issue_email_participants.create!(email: email) + end + end + + def service_desk_addresses + [ + project.service_desk_incoming_address, + project.service_desk_alias_address, + project.service_desk_custom_address + ].compact end + strong_memoize_attr :service_desk_addresses end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 7d47bfe88fe..1a7a2fba2f3 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -6,7 +6,7 @@ module Gitlab # When updating emoji assets increase the version below # and update the version number in `app/assets/javascripts/emoji/index.js` - EMOJI_VERSION = 2 + EMOJI_VERSION = 3 # Return a Pathname to emoji's current versioned folder # diff --git a/lib/gitlab/encrypted_command_base.rb b/lib/gitlab/encrypted_command_base.rb index b35c28b85cd..679d9d8e31a 100644 --- a/lib/gitlab/encrypted_command_base.rb +++ b/lib/gitlab/encrypted_command_base.rb @@ -7,12 +7,12 @@ module Gitlab EDIT_COMMAND_NAME = "base" class << self - def encrypted_secrets + def encrypted_secrets(**args) raise NotImplementedError end - def write(contents) - encrypted = encrypted_secrets + def write(contents, args: {}) + encrypted = encrypted_secrets(**args) return unless validate_config(encrypted) validate_contents(contents) @@ -25,8 +25,8 @@ module Gitlab warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?" end - def edit - encrypted = encrypted_secrets + def edit(args: {}) + encrypted = encrypted_secrets(**args) return unless validate_config(encrypted) if ENV["EDITOR"].blank? @@ -58,8 +58,8 @@ module Gitlab temp_file&.unlink end - def show - encrypted = encrypted_secrets + def show(args: {}) + encrypted = encrypted_secrets(**args) return unless validate_config(encrypted) puts encrypted.read.presence || "File '#{encrypted.content_path}' does not exist. Use `gitlab-rake #{self::EDIT_COMMAND_NAME}` to change that." diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb index 6b64281e631..5ead57e17fd 100644 --- a/lib/gitlab/encrypted_configuration.rb +++ b/lib/gitlab/encrypted_configuration.rb @@ -30,7 +30,7 @@ module Gitlab end def initialize(content_path: nil, base_key: nil, previous_keys: []) - @content_path = Pathname.new(content_path).yield_self { |path| path.symlink? ? path.realpath : path } if content_path + @content_path = Pathname.new(content_path).then { |path| path.symlink? ? path.realpath : path } if content_path @key = self.class.generate_key(base_key) if base_key @previous_keys = previous_keys end diff --git a/lib/gitlab/encrypted_ldap_command.rb b/lib/gitlab/encrypted_ldap_command.rb index 5e1eabe7ec6..442c675f19e 100644 --- a/lib/gitlab/encrypted_ldap_command.rb +++ b/lib/gitlab/encrypted_ldap_command.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -# rubocop:disable Rails/Output module Gitlab class EncryptedLdapCommand < EncryptedCommandBase DISPLAY_NAME = "LDAP" @@ -21,4 +20,3 @@ module Gitlab end end end -# rubocop:enable Rails/Output diff --git a/lib/gitlab/encrypted_redis_command.rb b/lib/gitlab/encrypted_redis_command.rb new file mode 100644 index 00000000000..608edcdb950 --- /dev/null +++ b/lib/gitlab/encrypted_redis_command.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Output +module Gitlab + class EncryptedRedisCommand < EncryptedCommandBase + DISPLAY_NAME = "Redis" + EDIT_COMMAND_NAME = "gitlab:redis:secret:edit" + + class << self + def all_redis_instance_class_names + Gitlab::Redis::ALL_CLASSES.map do |c| + normalized_instance_name(c) + end + end + + def normalized_instance_name(instance) + if instance.is_a?(Class) + # Gitlab::Redis::SharedState => sharedstate + instance.name.demodulize.to_s.downcase + else + # Drop all hyphens, underscores, and spaces from the name + # eg.: shared_state => sharedstate + instance.gsub(/[-_ ]/, '').downcase + end + end + + def encrypted_secrets(**args) + if args[:instance_name] + instance_class = Gitlab::Redis::ALL_CLASSES.find do |instance| + normalized_instance_name(instance) == normalized_instance_name(args[:instance_name]) + end + + unless instance_class + error_message = <<~MSG + Specified instance name #{args[:instance_name]} does not exist. + The available instances are #{all_redis_instance_class_names.join(', ')}." + MSG + + raise error_message + end + else + instance_class = Gitlab::Redis::Cache + end + + instance_class.encrypted_secrets + end + + def encrypted_file_template + <<~YAML + # password: '123' + YAML + end + end + end +end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 13959f6aa68..ef8f2d4d61b 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -21,7 +21,7 @@ module Gitlab # Configuration files gitignore: '.gitignore', - gitlab_ci: '.gitlab-ci.yml', + gitlab_ci: ::Ci::Pipeline::DEFAULT_CONFIG_PATH, route_map: '.gitlab/route-map.yml', # Dependency files diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 3d2bde6f0a7..e134fb31879 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -4,7 +4,6 @@ module Gitlab module Git class Blame include Gitlab::EncodingHelper - include Gitlab::Git::WrapsGitalyErrors attr_reader :lines, :blames, :range @@ -35,11 +34,9 @@ module Gitlab end def fetch_raw_blame - wrapped_gitaly_errors do - @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec) - end - # Return empty result when blame range is out-of-range + @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec) rescue ArgumentError + # Return an empty result when the blame range is out-of-range or path is not found "" end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index ae90291c0a3..3744c81f51d 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -230,5 +230,3 @@ module Gitlab end end end - -Gitlab::Git::Blob.singleton_class.prepend Gitlab::Git::RuggedImpl::Blob::ClassMethods diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 571dde6fcfc..1086ea45a7a 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -5,7 +5,6 @@ module Gitlab module Git class Commit include Gitlab::EncodingHelper - prepend Gitlab::Git::RuggedImpl::Commit extend Gitlab::Git::WrapsGitalyErrors include Gitlab::Utils::StrongMemoize @@ -502,5 +501,3 @@ module Gitlab end end end - -Gitlab::Git::Commit.singleton_class.prepend Gitlab::Git::RuggedImpl::Commit::ClassMethods diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 4a09f866db4..205dd5be35a 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -4,7 +4,6 @@ module Gitlab module Git class Ref include Gitlab::EncodingHelper - include Gitlab::Git::RuggedImpl::Ref # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index a98cf95edf4..db6e6b4d00b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -11,7 +11,6 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize - prepend Gitlab::Git::RuggedImpl::Repository SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb deleted file mode 100644 index dc869ff5279..00000000000 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -module Gitlab - module Git - module RuggedImpl - module Blob - module ClassMethods - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - override :tree_entry - def tree_entry(repository, sha, path, limit) - if use_rugged?(repository, :rugged_tree_entry) - execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit) - else - super - end - end - - private - - def rugged_tree_entry(repository, sha, path, limit) - return unless path - - # Strip any leading / characters from the path - path = path.sub(%r{\A/*}, '') - - rugged_commit = repository.lookup(sha) - root_tree = rugged_commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) - - return unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - # Rugged::Blob#content is expensive; don't call it if we don't have to. - data: limit == 0 ? '' : blob.content(limit), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) - end - end - rescue Rugged::ReferenceError - nil - end - - # Recursive search of blob id by path - # - # Ex. - # blog/ # oid: 1a - # app/ # oid: 2a - # models/ # oid: 3a - # file.rb # oid: 4a - # - # - # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' - # - def find_entry_by_path(repository, root_id, *path_parts) - root_tree = repository.lookup(root_id) - - entry = root_tree.find do |entry| - entry[:name] == path_parts[0] - end - - return unless entry - - if path_parts.size > 1 - return unless entry[:type] == :tree - - path_parts.shift - find_entry_by_path(repository, entry[:oid], *path_parts) - else - [:blob, :commit].include?(entry[:type]) ? entry : nil - end - end - - def submodule_blob(blob_entry, path, sha) - new( - id: blob_entry[:oid], - name: blob_entry[:name], - size: 0, - data: '', - path: path, - commit_id: sha - ) - end - end - end - end - end -end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb deleted file mode 100644 index cf547414b0d..00000000000 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ /dev/null @@ -1,115 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -# rubocop:disable Gitlab/ModuleWithInstanceVariables -module Gitlab - module Git - module RuggedImpl - module Commit - module ClassMethods - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - def rugged_find(repo, commit_id) - obj = repo.rev_parse_target(commit_id) - - obj.is_a?(::Rugged::Commit) ? obj : nil - rescue ::Rugged::Error - nil - end - - # This needs to return an array of Gitlab::Git:Commit objects - # instead of Rugged::Commit objects to ensure upstream models - # operate on a consistent interface. Unlike - # Gitlab::Git::Commit.find, Gitlab::Git::Commit.batch_by_oid - # doesn't attempt to decorate the result. - def rugged_batch_by_oid(repo, oids) - oids.map { |oid| rugged_find(repo, oid) } - .compact - .map { |commit| decorate(repo, commit) } - # Match Gitaly's list_commits_by_oid behavior - rescue ::Gitlab::Git::Repository::NoRepository - [] - end - - override :find_commit - def find_commit(repo, commit_id) - if use_rugged?(repo, :rugged_find_commit) - execute_rugged_call(:rugged_find, repo, commit_id) - else - super - end - end - - override :batch_by_oid - def batch_by_oid(repo, oids) - if use_rugged?(repo, :rugged_list_commits_by_oid) - execute_rugged_call(:rugged_batch_by_oid, repo, oids) - else - super - end - end - end - - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - override :init_commit - def init_commit(raw_commit) - case raw_commit - when ::Rugged::Commit - init_from_rugged(raw_commit) - else - super - end - end - - override :commit_tree_entry - def commit_tree_entry(path) - if use_rugged?(@repository, :rugged_commit_tree_entry) - execute_rugged_call(:rugged_tree_entry, path) - else - super - end - end - - # Is this the same as Blob.find_entry_by_path ? - def rugged_tree_entry(path) - rugged_commit.tree.path(path) - rescue Rugged::TreeError - nil - end - - def rugged_commit - @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit) - raw_commit - else - @repository.rev_parse_target(id) - end - end - - def init_from_rugged(commit) - author = commit.author - committer = commit.committer - - @raw_commit = commit - @id = commit.oid - @message = commit.message - @authored_date = author[:time] - @committed_date = committer[:time] - @author_name = author[:name] - @author_email = author[:email] - @committer_name = committer[:name] - @committer_email = committer[:email] - @parent_ids = commit.parents.map(&:oid) - @trailers = Hash[commit.trailers] - end - end - end - end -end -# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/ref.rb b/lib/gitlab/git/rugged_impl/ref.rb deleted file mode 100644 index b553e82dc47..00000000000 --- a/lib/gitlab/git/rugged_impl/ref.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -module Gitlab - module Git - module RuggedImpl - module Ref - def self.dereference_object(object) - object = object.target while object.is_a?(::Rugged::Tag::Annotation) - - object - end - end - end - end -end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb deleted file mode 100644 index cd4eefa158e..00000000000 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -# rubocop:disable Gitlab/ModuleWithInstanceVariables -module Gitlab - module Git - module RuggedImpl - module Repository - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - FEATURE_FLAGS = %i[rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid].freeze - - def alternate_object_directories - relative_object_directories.map { |d| File.join(path, d) } - end - - ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY_RELATIVE - GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE - ].freeze - - def relative_object_directories - Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact - end - - def rugged - @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories) - rescue ::Rugged::RepositoryError, ::Rugged::OSError - raise ::Gitlab::Git::Repository::NoRepository, 'no repository for such path' - end - - def cleanup - @rugged&.close - end - - # Return the object that +revspec+ points to. If +revspec+ is an - # annotated tag, then return the tag's target instead. - def rev_parse_target(revspec) - obj = rugged.rev_parse(revspec) - Ref.dereference_object(obj) - end - - override :ancestor? - def ancestor?(from, to) - if use_rugged?(self, :rugged_commit_is_ancestor) - execute_rugged_call(:rugged_is_ancestor?, from, to) - else - super - end - end - - def rugged_is_ancestor?(ancestor_id, descendant_id) - return false if ancestor_id.nil? || descendant_id.nil? - - rugged_merge_base(ancestor_id, descendant_id) == ancestor_id - rescue Rugged::OdbError - false - end - - def rugged_merge_base(from, to) - rugged.merge_base(from, to) - rescue Rugged::ReferenceError - nil - end - - # Lookup for rugged object by oid or ref name - def lookup(oid_or_ref_name) - rev_parse_target(oid_or_ref_name) - end - end - end - end -end -# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb deleted file mode 100644 index bc3ff01e1e2..00000000000 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -module Gitlab - module Git - module RuggedImpl - module Tree - module ClassMethods - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - TREE_SORT_ORDER = { tree: 0, blob: 1, commit: 2 }.freeze - - override :tree_entries - def tree_entries(repository, sha, path, recursive, skip_flat_paths, rescue_not_found, pagination_params = nil) - if use_rugged?(repository, :rugged_tree_entries) - entries = execute_rugged_call( - :tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive, skip_flat_paths) - - if pagination_params - paginated_response(entries, pagination_params[:limit], pagination_params[:page_token].to_s) - else - [entries, nil] - end - else - super - end - end - - # Rugged version of TreePagination in Go: https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3611 - def paginated_response(entries, limit, token) - total_entries = entries.count - - return [[], nil] if limit == 0 || limit.blank? - - entries = Gitlab::Utils.stable_sort_by(entries) { |x| TREE_SORT_ORDER[x.type] } - - if token.blank? - index = 0 - else - index = entries.index { |entry| entry.id == token } - - raise Gitlab::Git::CommandError, "could not find starting OID: #{token}" if index.nil? - - index += 1 - end - - return [entries[index..], nil] if limit < 0 - - last_index = index + limit - result = entries[index...last_index] - - if last_index < total_entries - cursor = Gitaly::PaginationCursor.new(next_cursor: result.last.id) - end - - [result, cursor] - end - - def tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive, skip_flat_paths) - tree_entries_from_rugged(repository, sha, path, recursive).tap do |entries| - # This was an optimization to reduce N+1 queries for Gitaly - # (https://gitlab.com/gitlab-org/gitaly/issues/530). - rugged_populate_flat_path(repository, sha, path, entries) unless skip_flat_paths - end - end - - def tree_entries_from_rugged(repository, sha, path, recursive) - current_path_entries = get_tree_entries_from_rugged(repository, sha, path) - ordered_entries = [] - - current_path_entries.each do |entry| - ordered_entries << entry - - if recursive && entry.dir? - ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) - end - end - - ordered_entries - end - - def rugged_populate_flat_path(repository, sha, path, entries) - entries.each do |entry| - entry.flat_path = entry.path - - next unless entry.dir? - - entry.flat_path = - if path - File.join(path, rugged_flatten_tree(repository, sha, entry, path)) - else - rugged_flatten_tree(repository, sha, entry, path) - end - end - end - - # Returns the relative path of the first subdir that doesn't have only one directory descendant - def rugged_flatten_tree(repository, sha, tree, root_path) - subtree = tree_entries_from_rugged(repository, sha, tree.path, false) - - if subtree.count == 1 && subtree.first.dir? - File.join(tree.name, rugged_flatten_tree(repository, sha, subtree.first, root_path)) - else - tree.name - end - end - - def get_tree_entries_from_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - current_path = path ? File.join(path, entry[:name]) : entry[:name] - - new( - id: entry[:oid], - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: current_path, - commit_id: sha - ) - end - rescue Rugged::ReferenceError - [] - end - end - end - end - end -end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb deleted file mode 100644 index 57cced97d02..00000000000 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Git - module RuggedImpl - module UseRugged - def use_rugged?(_, _) - false - end - - def execute_rugged_call(method_name, *args) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - start = Gitlab::Metrics::System.monotonic_time - - result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend - - duration = Gitlab::Metrics::System.monotonic_time - start - - if Gitlab::RuggedInstrumentation.active? - Gitlab::RuggedInstrumentation.increment_query_count - Gitlab::RuggedInstrumentation.add_query_time(duration) - - Gitlab::RuggedInstrumentation.add_call_details( - feature: method_name, - args: args, - duration: duration, - backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)) - end - - result - end - end - - def running_puma_with_multiple_threads? - return false unless Gitlab::Runtime.puma? - - ::Puma.respond_to?(:cli_config) && ::Puma.cli_config.options[:max_threads] > 1 - end - - def rugged_feature_keys - Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS - end - - def rugged_enabled_through_feature_flag? - false - end - end - end - end -end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 6e97e412b91..4747ab55c63 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -12,9 +12,6 @@ module Gitlab class << self # Get list of tree objects # for repository based on commit sha and path - # Uses rugged for raw objects - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 def where( repository, sha, path = nil, recursive = false, skip_flat_paths = true, rescue_not_found = true, pagination_params = nil) @@ -110,5 +107,3 @@ module Gitlab end end end - -Gitlab::Git::Tree.singleton_class.prepend Gitlab::Git::RuggedImpl::Tree::ClassMethods diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 45283d51b1b..72016aa1183 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -101,7 +101,7 @@ module Gitlab end def guest_can_download? - Guest.can?(download_ability, container) + ::Users::Anonymous.can?(download_ability, container) end def deploy_key_can_download_code? @@ -395,7 +395,7 @@ module Gitlab user.can?(:read_project, project) elsif ci? false - end || Guest.can?(:read_project, project) + end || ::Users::Anonymous.can?(:read_project, project) end def http? diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb index 732e0e14257..b007a957348 100644 --- a/lib/gitlab/git_access_project.rb +++ b/lib/gitlab/git_access_project.rb @@ -47,7 +47,7 @@ module Gitlab end def repository_path_match - strong_memoize(:repository_path_match) { repository_path.match(Gitlab::PathRegex.full_project_git_path_regex) || {} } + strong_memoize(:repository_path_match) { repository_path&.match(Gitlab::PathRegex.full_project_git_path_regex) || {} } end def ensure_project_on_push! diff --git a/lib/gitlab/git_audit_event.rb b/lib/gitlab/git_audit_event.rb deleted file mode 100644 index b8365bdf41f..00000000000 --- a/lib/gitlab/git_audit_event.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class GitAuditEvent # rubocop:disable Gitlab/NamespacedClass - attr_reader :project, :user, :author - - def initialize(player, project) - @project = project - @author = player.is_a?(::API::Support::GitAccessActor) ? player.deploy_key_or_user : player - @user = player.is_a?(::API::Support::GitAccessActor) ? player.user : player - end - - def send_audit_event(msg) - return if user.blank? || project.blank? - - audit_context = { - name: 'repository_git_operation', - stream_only: true, - author: author, - scope: project, - target: project, - message: msg - } - - ::Gitlab::Audit::Auditor.audit(audit_context) - end - end -end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 5ec58fc4f44..da38c11ebca 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -328,6 +328,8 @@ module Gitlab 'client_name' => CLIENT_NAME } + relative_path = fetch_relative_path + context_data = Gitlab::ApplicationContext.current feature_stack = Thread.current[:gitaly_feature_stack] @@ -339,6 +341,7 @@ module Gitlab metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil) metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) + metadata['relative-path-bin'] = relative_path if relative_path metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors)) metadata.merge!(route_to_primary) @@ -348,6 +351,17 @@ module Gitlab { metadata: metadata, deadline: deadline_info[:deadline] } end + # The GitLab `internal/allowed/` API sets the :gitlab_git_relative_path + # variable. This provides the repository relative path which can be used to + # locate snapshot repositories in Gitaly which act as a quarantine repository + # until a transaction is committed. + def self.fetch_relative_path + return unless Gitlab::SafeRequestStore.active? + return if Gitlab::SafeRequestStore[:gitlab_git_relative_path].blank? + + Gitlab::SafeRequestStore.fetch(:gitlab_git_relative_path) + end + # Gitlab::Git::HookEnv will set the :gitlab_git_env variable in case we're # running in the context of a Gitaly hook call, which may make use of # quarantined object directories. We thus need to pass along the path of diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 1ef5b0f96c2..3949e8e6416 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -418,6 +418,15 @@ module Gitlab response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce([]) { |memo, msg| memo << msg.data }.join + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error.try(:error) + when :out_of_range, :path_not_found + raise ArgumentError, e.details + else + raise e + end end def find_commit(revision) diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb index b1278e3bfac..a6912547ce9 100644 --- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -43,11 +43,15 @@ module Gitlab def conflict_from_gitaly_file_header(header) { - ancestor: { path: header.ancestor_path }, - ours: { path: header.our_path, mode: header.our_mode }, - theirs: { path: header.their_path } + ancestor: { path: encode_path(header.ancestor_path) }, + ours: { path: encode_path(header.our_path), mode: header.our_mode }, + theirs: { path: encode_path(header.their_path) } } end + + def encode_path(path) + Gitlab::EncodingHelper.encode_utf8(path) + end end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index d92bf5263f1..457380615f7 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -136,10 +136,13 @@ module Gitlab response.base.presence end - def fork_repository(source_repository) + def fork_repository(source_repository, branch = nil) + revision = branch.present? ? "refs/heads/#{branch}" : "" + request = Gitaly::CreateForkRequest.new( repository: @gitaly_repo, - source_repository: source_repository.gitaly_repository + source_repository: source_repository.gitaly_repository, + revision: revision ) gitaly_client_call( diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 4cc0269673f..adf0c811274 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -31,19 +31,11 @@ module Gitlab end def self.disk_access_denied? - return false if rugged_enabled? - !temporarily_allowed?(ALLOW_KEY) rescue StandardError false # Err on the side of caution, don't break gitlab for people end - def self.rugged_enabled? - Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.any? do |flag| - Feature.enabled?(flag) - end - end - def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') diff --git a/lib/gitlab/github_import/attachments_downloader.rb b/lib/gitlab/github_import/attachments_downloader.rb index 4db55a6aabb..df9c6c8342d 100644 --- a/lib/gitlab/github_import/attachments_downloader.rb +++ b/lib/gitlab/github_import/attachments_downloader.rb @@ -29,8 +29,8 @@ module Gitlab validate_content_length validate_filepath - redirection_url = get_assets_download_redirection_url - file = download_from(redirection_url) + download_url = get_assets_download_redirection_url + file = download_from(download_url) validate_symlink file @@ -60,16 +60,16 @@ module Gitlab options[:follow_redirects] = false response = Gitlab::HTTP.perform_request(Net::HTTP::Get, file_url, options) - raise_error("expected a redirect response, got #{response.code}") unless response.redirection? - redirection_url = response.headers[:location] - filename = URI.parse(redirection_url).path + download_url = if response.redirection? + response.headers[:location] + else + file_url + end - unless Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.any? { |type| filename.ends_with?(type) } - raise UnsupportedAttachmentError - end + file_type_valid?(URI.parse(download_url).path) - redirection_url + download_url end def github_assets_url_regex @@ -89,6 +89,12 @@ module Gitlab File.join(dir, filename) end end + + def file_type_valid?(file_url) + return if Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.any? { |type| file_url.ends_with?(type) } + + raise UnsupportedAttachmentError + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 5a0ae680ab8..33e74c90115 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -182,12 +182,12 @@ module Gitlab request_count_counter.increment - raise_or_wait_for_rate_limit unless requests_remaining? + raise_or_wait_for_rate_limit('Internal threshold reached') unless requests_remaining? begin with_retry { yield } - rescue ::Octokit::TooManyRequests - raise_or_wait_for_rate_limit + rescue ::Octokit::TooManyRequests => e + raise_or_wait_for_rate_limit(e.response_body) # This retry will only happen when running in sequential mode as we'll # raise an error in parallel mode. @@ -213,11 +213,11 @@ module Gitlab octokit.rate_limit.limit end - def raise_or_wait_for_rate_limit + def raise_or_wait_for_rate_limit(message) rate_limit_counter.increment if parallel? - raise RateLimitError + raise RateLimitError, message else sleep(rate_limit_resets_in) end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index b960df581e4..0780ba6119f 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -11,6 +11,7 @@ module Gitlab # The base cache key to use for storing/retrieving issuable IDs. CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}' + CACHE_OBJECT_NOT_FOUND = -1 # project - An instance of `Project`. # object - The object to look up or set a database ID for. @@ -23,9 +24,18 @@ module Gitlab # # This method will return `nil` if no ID could be found. def database_id - val = Gitlab::Cache::Import::Caching.read(cache_key, timeout: timeout) + val = Gitlab::Cache::Import::Caching.read_integer(cache_key, timeout: timeout) - val.to_i if val.present? + return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) + + return if val == CACHE_OBJECT_NOT_FOUND + return val if val.present? + + object_id = cache_key_type.safe_constantize&.find_by(project_id: project.id, iid: cache_key_iid)&.id || + CACHE_OBJECT_NOT_FOUND + + cache_database_id(object_id) + object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id end # Associates the given database ID with the current object. diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb index 52b211c92d6..077a27df16c 100644 --- a/lib/gitlab/github_import/job_delay_calculator.rb +++ b/lib/gitlab/github_import/job_delay_calculator.rb @@ -15,7 +15,7 @@ module Gitlab def calculate_job_delay(job_index) multiplier = (job_index / parallel_import_batch[:size]) - (multiplier * parallel_import_batch[:delay]) + 1.second + (multiplier * parallel_import_batch[:delay]).to_i + 1 end end end diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index 39e669dbba4..d0bbd2bc7cf 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -7,6 +7,7 @@ module Gitlab # The base cache key to use for storing/retrieving label IDs. CACHE_KEY = 'github-import/label-finder/%{project}/%{name}' + CACHE_OBJECT_NOT_FOUND = -1 # project - An instance of `Project`. def initialize(project) @@ -15,7 +16,18 @@ module Gitlab # Returns the label ID for the given name. def id_for(name) - Gitlab::Cache::Import::Caching.read_integer(cache_key_for(name)) + cache_key = cache_key_for(name) + val = Gitlab::Cache::Import::Caching.read_integer(cache_key) + + return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) + + return if val == CACHE_OBJECT_NOT_FOUND + return val if val.present? + + object_id = project.labels.with_title(name).pick(:id) || CACHE_OBJECT_NOT_FOUND + + Gitlab::Cache::Import::Caching.write(cache_key, object_id) + object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id end # rubocop: disable CodeReuse/ActiveRecord @@ -32,7 +44,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(name) - CACHE_KEY % { project: project.id, name: name } + format(CACHE_KEY, project: project.id, name: name) end end end diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index d9290e36ea1..dcb679fda6d 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -7,6 +7,7 @@ module Gitlab # The base cache key to use for storing/retrieving milestone IDs. CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}' + CACHE_OBJECT_NOT_FOUND = -1 # project - An instance of `Project` def initialize(project) @@ -18,7 +19,20 @@ module Gitlab def id_for(issuable) return unless issuable.milestone_number - Gitlab::Cache::Import::Caching.read_integer(cache_key_for(issuable.milestone_number)) + milestone_iid = issuable.milestone_number + cache_key = cache_key_for(milestone_iid) + + val = Gitlab::Cache::Import::Caching.read_integer(cache_key) + + return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) + + return if val == CACHE_OBJECT_NOT_FOUND + return val if val.present? + + object_id = project.milestones.by_iid(milestone_iid).pick(:id) || CACHE_OBJECT_NOT_FOUND + + Gitlab::Cache::Import::Caching.write(cache_key, object_id) + object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id end # rubocop: disable CodeReuse/ActiveRecord @@ -35,7 +49,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(iid) - CACHE_KEY % { project: project.id, iid: iid } + format(CACHE_KEY, project: project.id, iid: iid) end end end diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index 88e91800cee..5618cfc6044 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -52,7 +52,7 @@ module Gitlab .sort .each do |counter| object_type = counter.split('/').last - result[operation][object_type] = CACHING.read_integer(counter) || 0 + result[operation][object_type] = CACHING.read_integer(counter, timeout: IMPORT_CACHING_TIMEOUT) || 0 end end end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index cccd99f48b1..ce93b5203df 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -6,7 +6,7 @@ module Gitlab include JobDelayCalculator attr_reader :project, :client, :page_counter, :already_imported_cache_key, - :job_waiter_cache_key, :job_waiter_remaining_cache_key + :job_waiter_cache_key, :job_waiter_remaining_cache_key # The base cache key to use for tracking already imported objects. ALREADY_IMPORTED_CACHE_KEY = @@ -26,12 +26,11 @@ module Gitlab @client = client @parallel = parallel @page_counter = PageCounter.new(project, collection_method) - @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % - { project: project.id, collection: collection_method } - @job_waiter_cache_key = JOB_WAITER_CACHE_KEY % - { project: project.id, collection: collection_method } - @job_waiter_remaining_cache_key = JOB_WAITER_REMAINING_CACHE_KEY % - { project: project.id, collection: collection_method } + @already_imported_cache_key = format(ALREADY_IMPORTED_CACHE_KEY, project: project.id, + collection: collection_method) + @job_waiter_cache_key = format(JOB_WAITER_CACHE_KEY, project: project.id, collection: collection_method) + @job_waiter_remaining_cache_key = format(JOB_WAITER_REMAINING_CACHE_KEY, project: project.id, + collection: collection_method) end def parallel? @@ -57,7 +56,8 @@ module Gitlab # still scheduling duplicates while. Since all work has already been # completed those jobs will just cycle through any remaining pages while # not scheduling anything. - Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) + Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, + Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) info(project.id, message: "importer finished") retval @@ -97,7 +97,7 @@ module Gitlab repr = object_representation(object) job_delay = calculate_job_delay(enqueued_job_counter) - sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash, job_waiter.key) + sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash.deep_stringify_keys, job_waiter.key.to_s) enqueued_job_counter += 1 job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key) diff --git a/lib/gitlab/github_import/representation/to_hash.rb b/lib/gitlab/github_import/representation/to_hash.rb index 4a0f36ab8f0..54faa51a787 100644 --- a/lib/gitlab/github_import/representation/to_hash.rb +++ b/lib/gitlab/github_import/representation/to_hash.rb @@ -16,11 +16,15 @@ module Gitlab hash end + # This method allow objects to be safely passed directly to Sidekiq without errors. + # It returns JSON datatypes: string, integer, float, boolean, null(nil), array and hash. def convert_value_for_to_hash(value) if value.is_a?(Array) value.map { |v| convert_value_for_to_hash(v) } elsif value.respond_to?(:to_hash) value.to_hash + elsif value.respond_to?(:strftime) || value.is_a?(Symbol) + value.to_s else value end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index e057b4bb6f1..59813e4f5a0 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -50,6 +50,7 @@ module Gitlab gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week gon.time_display_relative = true + gon.time_display_format = 0 gon.ee = Gitlab.ee? gon.jh = Gitlab.jh? gon.dot_com = Gitlab.com? @@ -67,6 +68,7 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url gon.time_display_relative = current_user.time_display_relative + gon.time_display_format = current_user.time_display_format end # Initialize gon.features with any flags that should be @@ -75,7 +77,6 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) - push_frontend_feature_flag(:unbatch_graphql_queries, current_user) # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) push_frontend_feature_flag(:custom_emoji) diff --git a/lib/gitlab/graphql/tracers/timer_tracer.rb b/lib/gitlab/graphql/tracers/timer_tracer.rb index 8e058621110..2cf06086a3c 100644 --- a/lib/gitlab/graphql/tracers/timer_tracer.rb +++ b/lib/gitlab/graphql/tracers/timer_tracer.rb @@ -15,11 +15,11 @@ module Gitlab end def trace(key, data) - start_time = Gitlab::Metrics::System.monotonic_time + start_time = ::Gitlab::Metrics::System.monotonic_time yield ensure - data[:duration_s] = Gitlab::Metrics::System.monotonic_time - start_time + data[:duration_s] = ::Gitlab::Metrics::System.monotonic_time - start_time end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 8ca88859b22..6fe7a0030f0 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -13,7 +13,7 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def users groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user) - groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/417455") + groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/427108") members = GroupMember.where(group: groups).non_invite users = super diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 08d44184bb6..720f8748cba 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# Detect user based on identifier like +# Detect user or keys based on identifier like # key-13 or user-36 module Gitlab module Identifier @@ -35,6 +35,13 @@ module Gitlab end end + # Tries to identify a deploy key using a SSH key identifier (e.g. "key-123"). + def identify_using_deploy_key(identifier) + key_id = identifier.gsub("key-", "") + + DeployKey.find_by_id(key_id) + end + def identify_with_cache(category, key) if identification_cache[category].key?(key) identification_cache[category][key] diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index ea91b01afdb..523df1f9d5e 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -40,13 +40,12 @@ module Gitlab cmd = %W[gzip #{filepath}] cmd << "-#{options}" if options - _, status = Gitlab::Popen.popen(cmd) + output, status = Gitlab::Popen.popen(cmd) - if status == 0 - status - else - raise Gitlab::ImportExport::Error.file_compression_error - end + return status if status == 0 + + message = cmd_error_message(output, status) + raise Gitlab::ImportExport::Error.file_compression_error(message) end def mkdir_p(path) @@ -104,9 +103,7 @@ module Gitlab return true if status == 0 - output = output&.strip - message = "command exited with error code #{status}" - message += ": #{output}" if output.present? + message = cmd_error_message(output, status) if @shared.respond_to?(:error) @shared.error(Gitlab::ImportExport::Error.new(message)) @@ -149,6 +146,12 @@ module Gitlab FileUtils.remove_dir(dir) raise end + + def cmd_error_message(output, status) + message = "Command exited with error code #{status}" + message << ": #{output.strip}" unless output.blank? + message + end end end end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index fa179f584eb..9b8e6374b5a 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -14,8 +14,8 @@ module Gitlab self.new('Unknown object type') end - def self.file_compression_error - self.new('File compression/decompression failed') + def self.file_compression_error(error) + self.new(format('File compression or decompression failed. %{error}', error: error)) end def self.incompatible_import_file_error diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb index 543fd25d883..0cb0eb32a23 100644 --- a/lib/gitlab/import_export/project/sample/date_calculator.rb +++ b/lib/gitlab/import_export/project/sample/date_calculator.rb @@ -25,7 +25,7 @@ module Gitlab end def calculate_by_closest_date_to_average(date) - return date unless closest_date_to_average && closest_date_to_average < Time.current + return date unless closest_date_to_average && closest_date_to_average.past? date + (Time.current - closest_date_to_average).seconds end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index e39bbb36680..88991495a10 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -90,7 +90,7 @@ module Gitlab result = ::Gitlab::Instrumentation::RedisClusterValidator.validate(commands) return true if result.nil? - if !result[:valid] && !result[:allowed] && (Rails.env.development? || Rails.env.test?) + if !result[:valid] && !result[:allowed] && raise_cross_slot_validation_errors? raise RedisClusterValidator::CrossSlotError, "Redis command #{result[:command_name]} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end @@ -189,6 +189,10 @@ module Gitlab redirection_type, _, target_node_key = err_msg.split { redirection_type: redirection_type, target_node_key: target_node_key } end + + def raise_cross_slot_validation_errors? + Rails.env.development? || Rails.env.test? + end end end end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 20ba1ab82a7..5934204bd0f 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -31,7 +31,7 @@ module Gitlab private def instrument_call(commands, pipelined = false) - start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined + start = ::Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request(commands.size) instrumentation_class.instance_count_pipelined_request(commands.size) if pipelined @@ -50,7 +50,7 @@ module Gitlab instrumentation_class.log_exception(ex) raise ex ensure - duration = Gitlab::Metrics::System.monotonic_time - start + duration = ::Gitlab::Metrics::System.monotonic_time - start unless exclude_from_apdex?(commands) commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 2a3c4db5ffa..49078a7ccd0 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -12,7 +12,6 @@ module Gitlab def add_instrumentation_data(payload) instrument_gitaly(payload) - instrument_rugged(payload) instrument_redis(payload) instrument_elasticsearch(payload) instrument_zoekt(payload) @@ -40,15 +39,6 @@ module Gitlab payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time end - def instrument_rugged(payload) - rugged_calls = Gitlab::RuggedInstrumentation.query_count - - return if rugged_calls == 0 - - payload[:rugged_calls] = rugged_calls - payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time - end - def instrument_redis(payload) payload.merge! ::Gitlab::Instrumentation::Redis.payload end diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb index 2790bc8ee24..e2e4ea75dbf 100644 --- a/lib/gitlab/internal_events.rb +++ b/lib/gitlab/internal_events.rb @@ -23,8 +23,6 @@ module Gitlab private def increase_total_counter(event_name) - return unless ::ServicePing::ServicePingSettings.enabled? - redis_counter_key = Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name) Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb index 12cc5f6e5dd..c60dac6f571 100644 --- a/lib/gitlab/issues/rebalancing/state.rb +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -100,7 +100,7 @@ module Gitlab def refresh_keys_expiration with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipeline| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| pipeline.expire(issue_ids_key, REDIS_EXPIRY_TIME) pipeline.expire(current_index_key, REDIS_EXPIRY_TIME) pipeline.expire(current_project_key, REDIS_EXPIRY_TIME) diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index 7abfe8e38e8..2b8b01e2023 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -34,6 +34,17 @@ module Gitlab request_params[:headers][:Cookie] = get_cookies if options[:use_cookies] request_params[:base_uri] = uri.to_s request_params.merge!(auth_params) + # Setting defaults here so we can also set `timeout` which prevents setting defaults in the HTTP gem's code + request_params[:open_timeout] = options[:open_timeout] || default_timeout_for(:open_timeout) + request_params[:read_timeout] = options[:read_timeout] || default_timeout_for(:read_timeout) + request_params[:write_timeout] = options[:write_timeout] || default_timeout_for(:write_timeout) + # Global timeout. Needs to be at least as high as the maximum defined in other timeouts + request_params[:timeout] = [ + Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT, + request_params[:open_timeout], + request_params[:read_timeout], + request_params[:write_timeout] + ].max result = Gitlab::HTTP.public_send(http_method, path, **request_params) # rubocop:disable GitlabSecurity/PublicSend @authenticated = result.response.is_a?(Net::HTTPOK) @@ -52,6 +63,10 @@ module Gitlab private + def default_timeout_for(param) + Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS[param] + end + def auth_params return {} unless @options[:username] && @options[:password] diff --git a/lib/gitlab/jira/middleware.rb b/lib/gitlab/jira/middleware.rb deleted file mode 100644 index 8a74729da49..00000000000 --- a/lib/gitlab/jira/middleware.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Jira - class Middleware - def self.jira_dvcs_connector?(env) - env['HTTP_USER_AGENT']&.downcase&.start_with?('jira dvcs connector') - end - - def initialize(app) - @app = app - end - - def call(env) - if self.class.jira_dvcs_connector?(env) - env['HTTP_AUTHORIZATION'] = env['HTTP_AUTHORIZATION']&.sub('token', 'Bearer') - end - - @app.call(env) - end - end - end -end diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb index 2b83f0492cb..04ef1a0ef68 100644 --- a/lib/gitlab/jira_import/base_importer.rb +++ b/lib/gitlab/jira_import/base_importer.rb @@ -5,7 +5,7 @@ module Gitlab class BaseImporter attr_reader :project, :client, :formatter, :jira_project_key, :running_import - def initialize(project) + def initialize(project, client = nil) Gitlab::JiraImport.validate_project_settings!(project) @running_import = project.latest_jira_import @@ -14,7 +14,7 @@ module Gitlab raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key @project = project - @client = project.jira_integration.client + @client = client || project.jira_integration.client @formatter = Gitlab::ImportFormatter.new end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 458f7c3f470..54ececc4938 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :imported_items_cache_key, :start_at, :job_waiter - def initialize(project) + def initialize(project, client = nil) super # get cached start_at value, or zero if not cached yet @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id) diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index e53bfb40654..7b491b3e14d 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -19,9 +19,6 @@ module Gitlab class JobWaiter KEY_PREFIX = "gitlab:job_waiter" - STARTED_METRIC = :gitlab_job_waiter_started_total - TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total - # This TTL needs to be long enough to allow whichever Sidekiq job calls # JobWaiter#wait to reach BLPOP. DEFAULT_TTL = 6.hours.to_i @@ -48,16 +45,15 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.del(key) } if key?(key) end - attr_reader :key, :finished, :worker_label + attr_reader :key, :finished attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for # key - The key of this waiter. - def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}", worker_label: nil) + def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}") @key = key @jobs_remaining = jobs_remaining @finished = [] - @worker_label = worker_label end # Waits for all the jobs to be completed. @@ -67,7 +63,6 @@ module Gitlab # long to process, or is never processed. def wait(timeout = 10) deadline = Time.now.utc + timeout - increment_counter(STARTED_METRIC) Gitlab::Redis::SharedState.with do |redis| while jobs_remaining > 0 @@ -81,10 +76,7 @@ module Gitlab list, jid = redis.blpop(key, timeout: seconds_left) # timed out - unless list && jid - increment_counter(TIMEOUTS_METRIC) - break - end + break unless list && jid @finished << jid @jobs_remaining -= 1 @@ -93,20 +85,5 @@ module Gitlab finished end - - private - - def increment_counter(metric) - return unless worker_label - - metrics[metric].increment(worker: worker_label) - end - - def metrics - @metrics ||= { - STARTED_METRIC => Gitlab::Metrics.counter(STARTED_METRIC, 'JobWaiter attempts started'), - TIMEOUTS_METRIC => Gitlab::Metrics.counter(TIMEOUTS_METRIC, 'JobWaiter attempts timed out') - } - end end end diff --git a/lib/gitlab/kubernetes/kubeconfig/template.rb b/lib/gitlab/kubernetes/kubeconfig/template.rb index d40b9ce117e..844472f9c8e 100644 --- a/lib/gitlab/kubernetes/kubeconfig/template.rb +++ b/lib/gitlab/kubernetes/kubeconfig/template.rb @@ -44,7 +44,7 @@ module Gitlab ) end kubeconfig_yaml[:clusters].each do |cluster| - ca_pem = cluster.dig(:cluster, :'certificate-authority-data')&.yield_self do |data| + ca_pem = cluster.dig(:cluster, :'certificate-authority-data')&.then do |data| Base64.strict_decode64(data) end diff --git a/lib/gitlab/legacy_http.rb b/lib/gitlab/legacy_http.rb index f38b2819c15..cf6ab80d37f 100644 --- a/lib/gitlab/legacy_http.rb +++ b/lib/gitlab/legacy_http.rb @@ -35,8 +35,8 @@ module Gitlab read_total_timeout = options.fetch(:timeout, Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT) httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| - start_time ||= Gitlab::Metrics::System.monotonic_time - elapsed = Gitlab::Metrics::System.monotonic_time - start_time + start_time ||= ::Gitlab::Metrics::System.monotonic_time + elapsed = ::Gitlab::Metrics::System.monotonic_time - start_time if elapsed > read_total_timeout raise Gitlab::HTTP::ReadTotalTimeout, "Request timed out after #{elapsed} seconds" diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb index db0fd24983b..8d32745ac34 100644 --- a/lib/gitlab/memory/reporter.rb +++ b/lib/gitlab/memory/reporter.rb @@ -26,13 +26,13 @@ module Gitlab perf_report: report.name )) - start_monotonic_time = Gitlab::Metrics::System.monotonic_time - start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + start_monotonic_time = ::Gitlab::Metrics::System.monotonic_time + start_thread_cpu_time = ::Gitlab::Metrics::System.thread_cpu_time report_file = store_report(report) - cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) - duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + cpu_s = ::Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) + duration_s = ::Gitlab::Metrics::System.monotonic_time - start_monotonic_time @logger.info( log_labels( diff --git a/lib/gitlab/memory/reports_uploader.rb b/lib/gitlab/memory/reports_uploader.rb index 76c3e0862e2..17230414a6a 100644 --- a/lib/gitlab/memory/reports_uploader.rb +++ b/lib/gitlab/memory/reports_uploader.rb @@ -13,11 +13,11 @@ module Gitlab def upload(path) log_upload_requested(path) - start_monotonic_time = Gitlab::Metrics::System.monotonic_time + start_monotonic_time = ::Gitlab::Metrics::System.monotonic_time File.open(path.to_s) { |file| fog.put_object(gcs_bucket, File.basename(path), file) } - duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + duration_s = ::Gitlab::Metrics::System.monotonic_time - start_monotonic_time log_upload_success(path, duration_s) rescue StandardError, Errno::ENOENT => error log_exception(error) diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb index e18909d8f17..075a897478b 100644 --- a/lib/gitlab/merge_requests/mergeability/check_result.rb +++ b/lib/gitlab/merge_requests/mergeability/check_result.rb @@ -5,6 +5,7 @@ module Gitlab class CheckResult SUCCESS_STATUS = :success FAILED_STATUS = :failed + INACTIVE_STATUS = :inactive attr_reader :status, :payload @@ -20,6 +21,10 @@ module Gitlab new(status: FAILED_STATUS, payload: default_payload.merge(**payload)) end + def self.inactive(payload: {}) + new(status: INACTIVE_STATUS, payload: default_payload.merge(**payload)) + end + def self.from_hash(data) new( status: data.fetch(:status).to_sym, diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb index 258b655229e..b80a8c503e8 100644 --- a/lib/gitlab/metrics/exporter/metrics_middleware.rb +++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb @@ -9,10 +9,10 @@ module Gitlab default_labels = { pid: pid } - @requests_total = Gitlab::Metrics.counter( + @requests_total = ::Gitlab::Metrics.counter( :exporter_http_requests_total, 'Total number of HTTP requests', default_labels ) - @request_durations = Gitlab::Metrics.histogram( + @request_durations = ::Gitlab::Metrics.histogram( :exporter_http_request_duration_seconds, 'HTTP request duration histogram (seconds)', default_labels, @@ -21,9 +21,9 @@ module Gitlab end def call(env) - start = Gitlab::Metrics::System.monotonic_time + start = ::Gitlab::Metrics::System.monotonic_time @app.call(env).tap do |response| - duration = Gitlab::Metrics::System.monotonic_time - start + duration = ::Gitlab::Metrics::System.monotonic_time - start labels = { method: env['REQUEST_METHOD'].downcase, diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index d2336ec4bb2..5a0612be88e 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -141,7 +141,7 @@ module Gitlab return empty_result unless has_basic_credentials?(request) login, password = user_name_and_password(request) - auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip) + auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, request: request) return empty_result unless auth_result.success? return empty_result unless auth_result.can?(:access_git) diff --git a/lib/gitlab/middleware/path_traversal_check.rb b/lib/gitlab/middleware/path_traversal_check.rb index 79465f3cb30..6fef247b708 100644 --- a/lib/gitlab/middleware/path_traversal_check.rb +++ b/lib/gitlab/middleware/path_traversal_check.rb @@ -5,6 +5,28 @@ module Gitlab class PathTraversalCheck PATH_TRAVERSAL_MESSAGE = 'Potential path traversal attempt detected' + EXCLUDED_EXACT_PATHS = %w[/search].freeze + EXCLUDED_PATH_PREFIXES = %w[/search/].freeze + + EXCLUDED_API_PATHS = %w[/search].freeze + EXCLUDED_PROJECT_API_PATHS = %w[/search].freeze + EXCLUDED_GROUP_API_PATHS = %w[/search].freeze + + API_PREFIX = %r{/api/[^/]+} + API_SUFFIX = %r{(?:\.[^/]+)?} + + EXCLUDED_API_PATHS_REGEX = [ + EXCLUDED_API_PATHS.map do |path| + %r{\A#{API_PREFIX}#{path}#{API_SUFFIX}\z} + end.freeze, + EXCLUDED_PROJECT_API_PATHS.map do |path| + %r{\A#{API_PREFIX}/projects/[^/]+(?:/-)?#{path}#{API_SUFFIX}\z} + end.freeze, + EXCLUDED_GROUP_API_PATHS.map do |path| + %r{\A#{API_PREFIX}/groups/[^/]+(?:/-)?#{path}#{API_SUFFIX}\z} + end.freeze + ].flatten.freeze + def initialize(app) @app = app end @@ -14,7 +36,8 @@ module Gitlab log_params = {} execution_time = measure_execution_time do - check(env, log_params) + request = ::Rack::Request.new(env.dup) + check(request, log_params) unless excluded?(request) end log_params[:duration_ms] = execution_time.round(5) if execution_time @@ -37,17 +60,25 @@ module Gitlab end end - def check(env, log_params) - request = ::Rack::Request.new(env) - fullpath = request.fullpath - decoded_fullpath = CGI.unescape(fullpath) + def check(request, log_params) + decoded_fullpath = CGI.unescape(request.fullpath) ::Gitlab::PathTraversal.check_path_traversal!(decoded_fullpath, skip_decoding: true) - rescue ::Gitlab::PathTraversal::PathTraversalAttackError - log_params[:fullpath] = fullpath + log_params[:method] = request.request_method + log_params[:fullpath] = request.fullpath log_params[:message] = PATH_TRAVERSAL_MESSAGE end + def excluded?(request) + path = request.path + + return true if path.in?(EXCLUDED_EXACT_PATHS) + return true if EXCLUDED_PATH_PREFIXES.any? { |p| path.start_with?(p) } + return true if EXCLUDED_API_PATHS_REGEX.any? { |r| path.match?(r) } + + false + end + def log(payload) Gitlab::AppLogger.warn( payload.merge(class_name: self.class.name) diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 81ad7a7f9e1..0bcd5b1196a 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -29,6 +29,8 @@ module Gitlab { authorize_params: { gl_auth_type: 'login' } } + when ->(provider_name) { AuthHelper.saml_providers.include?(provider_name.to_sym) } + { attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements } else {} end @@ -61,7 +63,7 @@ module Gitlab provider_arguments.concat arguments provider_arguments << defaults unless defaults.empty? when Hash, GitlabSettings::Options - hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults) + hash_arguments = merge_hash_defaults_and_args(defaults, arguments) normalized = normalize_hash_arguments(hash_arguments) # A Hash from the configuration will be passed as is. @@ -80,6 +82,13 @@ module Gitlab provider_arguments end + def merge_hash_defaults_and_args(defaults, arguments) + return arguments.to_hash if defaults.empty? + return defaults.deep_merge(arguments.deep_symbolize_keys) if Feature.enabled?(:invert_omniauth_args_merging) + + arguments.to_hash.deep_symbolize_keys.deep_merge(defaults) + end + def normalize_hash_arguments(args) args.deep_symbolize_keys! diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 3c8ac55f70b..adc417f287c 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -7,7 +7,7 @@ module Gitlab module_function def retry_lock(subject, max_retries = MAX_RETRIES, name:, &block) - start_time = Gitlab::Metrics::System.monotonic_time + start_time = ::Gitlab::Metrics::System.monotonic_time retry_attempts = 0 # prevent scope override, see https://gitlab.com/gitlab-org/gitlab/-/issues/391186 @@ -39,7 +39,7 @@ module Gitlab def log_optimistic_lock_retries(name:, retry_attempts:, start_time:) return unless retry_attempts > 0 - elapsed_time = Gitlab::Metrics::System.monotonic_time - start_time + elapsed_time = ::Gitlab::Metrics::System.monotonic_time - start_time retry_lock_logger.info( message: "Optimistic Lock released with retries", diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb index 6845f5d88ec..bf6ac3a056d 100644 --- a/lib/gitlab/pages/deployment_update.rb +++ b/lib/gitlab/pages/deployment_update.rb @@ -89,14 +89,10 @@ module Gitlab project.actual_limits.pages_file_entries end + # If a newer pipeline already build a PagesDeployment def validate_outdated_sha return if latest? - - # use pipeline_id in case the build is retried - last_deployed_pipeline_id = project.pages_metadatum&.pages_deployment&.ci_build&.pipeline_id - - return unless last_deployed_pipeline_id - return if last_deployed_pipeline_id <= build.pipeline_id + return if latest_pipeline_id <= build.pipeline_id errors.add(:base, 'build SHA is outdated for this ref') end @@ -111,6 +107,13 @@ module Gitlab def sha build.sha end + + def latest_pipeline_id + project + .active_pages_deployments + .with_path_prefix(build.pages&.dig(:path_prefix)) + .latest_pipeline_id + end end end end diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 81dcc54ff35..9e8c0c530a9 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -34,8 +34,10 @@ module Gitlab order_satisfied?(relation, cursor_based_request_context) end - def self.enforced_for_type?(relation) - ENFORCED_TYPES.include?(relation.klass) + def self.enforced_for_type?(request_scope, relation) + enforced = ENFORCED_TYPES + enforced += [::Ci::Build] if ::Feature.enabled?(:enforce_ci_builds_pagination_limit, request_scope, type: :ops) + enforced.include?(relation.klass) end def self.order_satisfied?(relation, cursor_based_request_context) diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb index c9eae2f899f..8f1fbf53161 100644 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -7,7 +7,7 @@ require 'sidekiq/version' require 'sidekiq/cron/version' -if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.7') +if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.12') raise 'New version of sidekiq detected, please remove or update this patch' end diff --git a/lib/gitlab/patch/sidekiq_scheduled_enq.rb b/lib/gitlab/patch/sidekiq_scheduled_enq.rb index de0e8465f97..b5a40c19923 100644 --- a/lib/gitlab/patch/sidekiq_scheduled_enq.rb +++ b/lib/gitlab/patch/sidekiq_scheduled_enq.rb @@ -15,10 +15,8 @@ module Gitlab # this portion swaps out Sidekiq.redis for Gitlab::Redis::Queues Gitlab::Redis::Queues.with do |conn| # rubocop:disable Cop/RedisQueueUsage sorted_sets.each do |sorted_set| - # adds namespace if `super` polls with a non-namespaced Sidekiq.redis - if Gitlab::Utils.to_boolean(ENV['SIDEKIQ_ENQUEUE_NON_NAMESPACED']) - sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage - end + # adds namespace since `super` polls with a non-namespaced Sidekiq.redis + sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s])) # rubocop:disable Gitlab/ModuleWithInstanceVariables, Lint/AssignmentInCondition Sidekiq::Client.push(Sidekiq.load_json(job)) # rubocop:disable Cop/SidekiqApiUsage @@ -28,7 +26,6 @@ module Gitlab end end - # calls original enqueue_jobs which may or may not be namespaced depending on SIDEKIQ_ENQUEUE_NON_NAMESPACED super end end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index c9ed4720e83..5f2084ce011 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -60,14 +60,14 @@ module Gitlab ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/third-party-logos/dotnet.svg'), ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'), - ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/gitlab-org/project-templates/bridgetown'), + ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/pages/bridgetown'), ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby'), 'https://gitlab.com/pages/gatsby', 'illustrations/third-party-logos/gatsby.svg'), ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'), ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'), ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML'), 'https://gitlab.com/pages/plain-html'), ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), - ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/gitlab-org/project-templates/middleman', 'illustrations/logos/middleman.svg'), + ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/pages/middleman', 'illustrations/logos/middleman.svg'), ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'), ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), @@ -81,7 +81,8 @@ module Gitlab ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'), ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg'), - ProjectTemplate.new('laravel', 'Laravel Framework', _('A basic folder structure of a Laravel application, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/laravel', 'illustrations/logos/laravel.svg') + ProjectTemplate.new('laravel', 'Laravel Framework', _('A basic folder structure of a Laravel application, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/laravel', 'illustrations/logos/laravel.svg'), + ProjectTemplate.new('astro_tailwind', 'Astro Tailwind', _('A basic folder structure of Astro Starter Kit, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/astro-tailwind') ] end # rubocop:enable Metrics/AbcSize diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 4471d21b9ac..e817f2130f4 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -14,6 +14,7 @@ module Gitlab :milestone, :remove_source_branch, :target, + :target_project, :title, :unassign, :unlabel diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 9798b0eca2c..72bec159226 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -172,6 +172,25 @@ module Gitlab end end + desc { _('Request changes') } + explanation { _('Request changes to the current merge request.') } + types MergeRequest + condition do + Feature.enabled?(:mr_request_changes, current_user) && + quick_action_target.persisted? && + quick_action_target.find_reviewer(current_user) + end + command :request_changes do + result = ::MergeRequests::UpdateReviewerStateService.new(project: quick_action_target.project, current_user: current_user) + .execute(quick_action_target, "requested_changes") + + @execution_message[:request_changes] = if result[:status] == :success + _('Changes requested to the current merge request.') + else + result[:message] + end + end + desc { _('Approve a merge request') } explanation { _('Approve the current merge request.') } types MergeRequest @@ -197,6 +216,10 @@ module Gitlab next unless success + ::MergeRequests::UpdateReviewerStateService + .new(project: quick_action_target.project, current_user: current_user) + .execute(quick_action_target, "unreviewed") + @execution_message[:unapprove] = _('Unapproved the current merge request.') end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 89ec996488f..9f7599d2500 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -14,7 +14,6 @@ module Gitlab Gitlab::Redis::FeatureFlag, Gitlab::Redis::Queues, Gitlab::Redis::QueuesMetadata, - Gitlab::Redis::Pubsub, Gitlab::Redis::RateLimiting, Gitlab::Redis::RepositoryCache, Gitlab::Redis::Sessions, diff --git a/lib/gitlab/redis/cluster_util.rb b/lib/gitlab/redis/cluster_util.rb index 5f1f39b5237..9e307940de3 100644 --- a/lib/gitlab/redis/cluster_util.rb +++ b/lib/gitlab/redis/cluster_util.rb @@ -26,6 +26,15 @@ module Gitlab end expired_count end + + # Redis cluster alternative to mget + def batch_get(keys, redis) + keys.each_slice(1000).flat_map do |subset| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + subset.map { |key| pipeline.get(key) } + end + end + end end end end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index bbe5a8add4b..6acbf83df24 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -63,8 +63,12 @@ module Gitlab hlen hmget hscan_each + llen + lrange mapped_hmget mget + pfcount + pttl scan scan_each scard @@ -72,20 +76,32 @@ module Gitlab smembers sscan sscan_each + strlen ttl + type + zcard + zcount + zrange + zrangebyscore + zrevrange zscan_each + zscore ].freeze WRITE_COMMANDS = %i[ + decr del eval expire flushdb hdel + hincrby hset incr incrby mapped_hmset + pfadd + pfmerge publish rpush sadd @@ -93,8 +109,15 @@ module Gitlab set setex setnx + spop srem + srem? unlink + zadd + zpopmin + zrem + zremrangebyrank + zremrangebyscore memory ].freeze @@ -254,11 +277,27 @@ module Gitlab # # Let's define it explicitly instead of propagating it to method_missing def close - if use_primary_and_secondary_stores? - [primary_store, secondary_store].map(&:close).first + if same_redis_store? + # if same_redis_store?, `use_primary_store_as_default?` returns false + # but we should avoid a feature-flag check in `.close` to avoid checking out + # an ActiveRecord connection during clean up. + secondary_store.close else - default_store.close + [primary_store, secondary_store].map(&:close).first + end + end + + # blpop blocks until an element to be popped exist in the list or after a timeout. + def blpop(*args) + result = default_store.blpop(*args) + if !!result && use_primary_and_secondary_stores? + # special case to accommodate Gitlab::JobWaiter as blpop is only used in JobWaiter + # 1s should be sufficient wait time to account for delays between 1st and 2nd lpush + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2520#note_1630893702 + non_default_store.blpop(args.first, timeout: 1) end + + result end private @@ -380,7 +419,7 @@ module Gitlab end def redis_store?(store) - store.is_a?(::Redis) || store.is_a?(::Redis::Namespace) + store.is_a?(::Redis) end def validate_stores! diff --git a/lib/gitlab/redis/pubsub.rb b/lib/gitlab/redis/pubsub.rb deleted file mode 100644 index b5022f467a2..00000000000 --- a/lib/gitlab/redis/pubsub.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - class Pubsub < ::Gitlab::Redis::Wrapper - class << self - def config_fallback - SharedState - end - end - end - end -end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index fb3a143121b..d12d3e8c6aa 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -3,6 +3,12 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper + def self.redis + primary_store = ::Redis.new(ClusterSharedState.params) + secondary_store = ::Redis.new(params) + + MultiStore.new(primary_store, secondary_store, store_name) + end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 2bcf4769b5a..d5470bc0016 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -19,7 +19,7 @@ module Gitlab InvalidPathError = Class.new(StandardError) class << self - delegate :params, :url, :store, to: :new + delegate :params, :url, :store, :encrypted_secrets, to: :new def with pool.with { |redis| yield redis } @@ -110,6 +110,14 @@ module Gitlab raw_config_hash[:sentinels] end + def secret_file + if raw_config_hash[:secret_file].blank? + File.join(Settings.encrypted_settings['path'], 'redis.yaml.enc') + else + Settings.absolute(raw_config_hash[:secret_file]) + end + end + def sentinels? sentinels && !sentinels.empty? end @@ -118,22 +126,44 @@ module Gitlab ::Redis::Store::Factory.create(redis_store_options.merge(extras)) end + def encrypted_secrets + # In rake tasks, we have to populate the encrypted_secrets even if the + # file does not exist, as it is the job of one of those tasks to create + # the file. In other cases, like when being loaded as part of spinning + # up test environment via `scripts/setup-test-env`, we should gate on + # the presence of the specified secret file so that + # `Settings.encrypted`, which might not be loadable does not gets + # called. + Settings.encrypted(secret_file) if File.exist?(secret_file) || ::Gitlab::Runtime.rake? + end + private def redis_store_options config = raw_config_hash config[:instrumentation_class] ||= self.class.instrumentation_class - result = if config[:cluster].present? - config[:db] = 0 # Redis Cluster only supports db 0 - config + decrypted_config = parse_encrypted_config(config) + + result = if decrypted_config[:cluster].present? + decrypted_config[:db] = 0 # Redis Cluster only supports db 0 + decrypted_config else - parse_redis_url(config) + parse_redis_url(decrypted_config) end parse_client_tls_options(result) end + def parse_encrypted_config(encrypted_config) + encrypted_config.delete(:secret_file) + + decrypted_secrets = encrypted_secrets&.config + encrypted_config.merge!(decrypted_secrets) if decrypted_secrets + + encrypted_config + end + def parse_redis_url(config) redis_url = config.delete(:url) redis_uri = URI.parse(redis_url) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 2fd9dc9fa09..6ac37986d5c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -5,6 +5,7 @@ module Gitlab extend self extend MergeRequests extend Packages + extend Packages::Protection::Rules def project_name_regex # The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff} diff --git a/lib/gitlab/regex/packages.rb b/lib/gitlab/regex/packages.rb index 6b178933a25..a0038d39318 100644 --- a/lib/gitlab/regex/packages.rb +++ b/lib/gitlab/regex/packages.rb @@ -3,6 +3,8 @@ module Gitlab module Regex module Packages + include ::Gitlab::Utils::StrongMemoize + CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze @@ -74,8 +76,10 @@ module Gitlab maven_app_name_regex end - def npm_package_name_regex - @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z}o + def npm_package_name_regex(other_accepted_chars = nil) + strong_memoize_with(:npm_package_name_regex, other_accepted_chars) do + %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9#{other_accepted_chars}]+\z} + end end def npm_package_name_regex_message diff --git a/lib/gitlab/regex/packages/protection/rules.rb b/lib/gitlab/regex/packages/protection/rules.rb new file mode 100644 index 00000000000..383f26fe92d --- /dev/null +++ b/lib/gitlab/regex/packages/protection/rules.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Regex + module Packages + module Protection + module Rules + def protection_rules_npm_package_name_pattern_regex + @protection_rules_npm_package_name_pattern_regex ||= npm_package_name_regex('*') + end + end + end + end + end +end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index 3a389d3363f..d5e80053772 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -6,7 +6,8 @@ module Gitlab module RequestForgeryProtection - class Controller < BaseActionController + # rubocop:disable Rails/ApplicationController + class Controller < ActionController::Base protect_from_forgery with: :exception, prepend: true def initialize @@ -39,5 +40,6 @@ module Gitlab rescue ActionController::InvalidAuthenticityToken false end + # rubocop:enable Rails/ApplicationController end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb deleted file mode 100644 index 36a3a491de6..00000000000 --- a/lib/gitlab/rugged_instrumentation.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RuggedInstrumentation - def self.query_time - query_time = SafeRequestStore[:rugged_query_time] || 0 - query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) - end - - def self.add_query_time(duration) - SafeRequestStore[:rugged_query_time] ||= 0 - SafeRequestStore[:rugged_query_time] += duration - end - - def self.query_time_ms - (self.query_time * 1000).round(2) - end - - def self.query_count - SafeRequestStore[:rugged_call_count] ||= 0 - end - - def self.increment_query_count - SafeRequestStore[:rugged_call_count] ||= 0 - SafeRequestStore[:rugged_call_count] += 1 - end - - def self.active? - SafeRequestStore.active? - end - - def self.add_call_details(details) - return unless Gitlab::PerformanceBar.enabled_for_request? - - Gitlab::SafeRequestStore[:rugged_call_details] ||= [] - Gitlab::SafeRequestStore[:rugged_call_details] << details - end - - def self.list_call_details - return [] unless Gitlab::PerformanceBar.enabled_for_request? - - Gitlab::SafeRequestStore[:rugged_call_details] || [] - end - end -end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index d06f414bd9a..fada3b84401 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -191,9 +191,7 @@ module Gitlab unless default_project_filter project_ids = project_ids_relation - if Feature.enabled?(:search_issues_hide_archived_projects, current_user) && !filters[:include_archived] - project_ids = project_ids.non_archived - end + project_ids = project_ids.non_archived unless filters[:include_archived] issues = issues.in_projects(project_ids) .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/420046') @@ -218,9 +216,7 @@ module Gitlab unless default_project_filter project_ids = project_ids_relation - if Feature.enabled?(:search_merge_requests_hide_archived_projects, current_user) && !filters[:include_archived] - project_ids = project_ids.non_archived - end + project_ids = project_ids.non_archived unless filters[:include_archived] merge_requests = merge_requests.of_projects(project_ids) end diff --git a/lib/gitlab/seeders/ci/catalog/resource_seeder.rb b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb new file mode 100644 index 00000000000..2971dabe044 --- /dev/null +++ b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + module Catalog + class ResourceSeeder + # This is currently disabled until it gets fixed: https://gitlab.com/gitlab-org/gitlab/-/issues/429649 + # Initializes the class + # + # @param [String] Path of the group to find + # @param [Integer] Number of resources to create + def initialize(group_path:, seed_count:) + @group = Group.find_by_full_path(group_path) + @seed_count = seed_count + @current_user = @group&.first_owner + end + + def seed + if @group.nil? + warn 'ERROR: Group was not found.' + return + end + + @seed_count.times do |i| + create_ci_catalog_resource(i) + end + end + + private + + def create_project(name, index) + project = ::Projects::CreateService.new( + @current_user, + description: "This is Catalog resource ##{index}", + name: name, + namespace_id: @group.id, + path: name, + visibility_level: @group.visibility_level + ).execute + + if project.saved? + project + else + warn project.errors.full_messages.to_sentence + nil + end + end + + def create_template_yml(project) + template_content = <<~YAML + spec: + inputs: + stage: + default: test + --- + component-job: + script: echo job 1 + stage: $[[ inputs.stage ]] + YAML + + project.repository.create_file( + @current_user, + 'template.yml', + template_content, + message: 'Add template.yml', + branch_name: project.default_branch_or_main + ) + end + + def create_readme(project, index) + project.repository.create_file( + @current_user, + '/README.md', + "## Component stuff #{index}", + message: 'Add README.md', + branch_name: project.default_branch_or_main + ) + end + + def create_ci_catalog(project) + result = ::Ci::Catalog::Resources::CreateService.new(project, @current_user).execute + if result.success? + result.payload + else + warn "Project '#{project.name}' could not be converted to a Catalog resource" + nil + end + end + + def create_ci_catalog_resource(index) + name = "ci_seed_resource_#{index}" + + if Project.find_by_name(name).present? + warn "Project '#{name}' already exists!" + return + end + + project = create_project(name, index) + + return unless project + + create_readme(project, index) + create_template_yml(project) + + return unless create_ci_catalog(project) + + warn "Project '#{name}' was saved successfully!" + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index a1363e7b6b2..10a69acc037 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -21,6 +21,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize DEFAULT_DUPLICATE_KEY_TTL = 6.hours + SHORT_DUPLICATE_KEY_TTL = 10.minutes DEFAULT_STRATEGY = :until_executing STRATEGY_NONE = :none @@ -134,7 +135,7 @@ module Gitlab jid != existing_jid end - def set_deduplicated_flag!(expiry = duplicate_key_ttl) + def set_deduplicated_flag! return unless reschedulable? with_redis { |redis| redis.eval(DEDUPLICATED_SCRIPT, keys: [cookie_key]) } @@ -173,7 +174,7 @@ module Gitlab end def duplicate_key_ttl - options[:ttl] || DEFAULT_DUPLICATE_KEY_TTL + options[:ttl] || default_duplicate_key_ttl end private @@ -182,6 +183,12 @@ module Gitlab attr_reader :queue_name, :job attr_writer :existing_jid + def default_duplicate_key_ttl + return SHORT_DUPLICATE_KEY_TTL if Feature.enabled?(:reduce_duplicate_job_key_ttl) + + DEFAULT_DUPLICATE_KEY_TTL + end + def worker_klass @worker_klass ||= worker_class_name.to_s.safe_constantize end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb index b065190f656..e7ce837de29 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb @@ -20,7 +20,7 @@ module Gitlab if duplicate_job.idempotent? duplicate_job.update_latest_wal_location! - duplicate_job.set_deduplicated_flag!(expiry) + duplicate_job.set_deduplicated_flag! Gitlab::SidekiqLogging::DeduplicationLogger.instance.deduplicated_log( job, strategy_name, duplicate_job.options) diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index a8b3683e09f..37a9ed37891 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -25,11 +25,6 @@ module Gitlab def metrics metrics = { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), - sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), @@ -43,9 +38,24 @@ module Gitlab metrics[:sidekiq_jobs_completion_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS) metrics[:sidekiq_jobs_queue_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS) metrics[:sidekiq_jobs_failed_total] = ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed') + + # resource usage + metrics[:sidekiq_jobs_cpu_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS) + metrics[:sidekiq_jobs_db_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS) + metrics[:sidekiq_jobs_gitaly_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS) + metrics[:sidekiq_redis_requests_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS) + metrics[:sidekiq_elasticsearch_requests_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS) else - # The sum metric is still used in GitLab.com for dashboards + # These metrics are used in GitLab.com dashboards metrics[:sidekiq_jobs_completion_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_completion_seconds_sum, 'Total of seconds to complete Sidekiq job') + metrics[:sidekiq_jobs_completion_count] = ::Gitlab::Metrics.counter(:sidekiq_jobs_completion_count, 'Number of Sidekiq jobs completed') + + # resource usage sums + metrics[:sidekiq_jobs_cpu_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_cpu_seconds_sum, 'Total seconds this Sidekiq job spent on the CPU') + metrics[:sidekiq_jobs_db_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_db_seconds_sum, 'Total seconds of database time to run Sidekiq job') + metrics[:sidekiq_jobs_gitaly_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_gitaly_seconds_sum, 'Total seconds Gitaly time to run Sidekiq job') + metrics[:sidekiq_redis_requests_duration_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_redis_requests_duration_seconds_sum, 'Total duration in seconds that a Sidekiq job spent in requests to a Redis server') + metrics[:sidekiq_elasticsearch_requests_duration_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_duration_seconds_sum, 'Total duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server') end metrics @@ -89,8 +99,9 @@ module Gitlab # in metrics and can use them in the `ThreadsSampler` for setting a label Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME - labels = create_labels(worker.class, queue, job) - instrument(job, labels) do + @job = job + @labels = create_labels(worker.class, queue, job) + instrument do yield end end @@ -99,8 +110,8 @@ module Gitlab attr_reader :metrics - def instrument(job, labels) - queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) + def instrument + @queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds]&.observe(labels, queue_duration) if queue_duration @@ -114,43 +125,33 @@ module Gitlab @metrics[:sidekiq_jobs_interrupted_total].increment(labels, 1) end - job_succeeded = false + @job_succeeded = false monotonic_time_start = Gitlab::Metrics::System.monotonic_time job_thread_cputime_start = get_thread_cputime begin transaction = Gitlab::Metrics::BackgroundTransaction.new transaction.run { yield } - job_succeeded = true + @job_succeeded = true ensure monotonic_time_end = Gitlab::Metrics::System.monotonic_time job_thread_cputime_end = get_thread_cputime - monotonic_time = monotonic_time_end - monotonic_time_start - job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start + @monotonic_time = monotonic_time_end - monotonic_time_start + @job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start - # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label @metrics[:sidekiq_running_jobs].increment(labels, -1) - if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops) - @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded - else - # we don't need job_status label here - @metrics[:sidekiq_jobs_completion_seconds_sum].increment(labels, monotonic_time) - end + @instrumentation = job[:instrumentation] || {} + + record_resource_usage_counters # job_status: done, fail match the job_status attribute in structured logging labels[:job_status] = job_succeeded ? "done" : "fail" - instrumentation = job[:instrumentation] || {} - @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) - @metrics[:sidekiq_jobs_completion_seconds]&.observe(labels, monotonic_time) + record_histograms - @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) - @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation)) @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation)) - @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation)) @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) - @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) @metrics[:sidekiq_mem_total_bytes].set(labels, get_thread_memory_total_allocations(instrumentation)) with_load_balancing_settings(job) do |settings| @@ -162,15 +163,50 @@ module Gitlab @metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1) end - sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) - Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded - Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) - Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration + @sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) + record_execution_sli + record_queueing_sli end end private + attr_reader :labels, :job, :queue_duration, :job_succeeded, :monotonic_time, :job_thread_cputime, :instrumentation, :sli_labels + + def record_resource_usage_counters + if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops) + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded + else + @metrics[:sidekiq_jobs_completion_seconds_sum].increment(labels, monotonic_time) + @metrics[:sidekiq_jobs_completion_count].increment(labels, 1) + @metrics[:sidekiq_jobs_cpu_seconds_sum].increment(labels, job_thread_cputime) + @metrics[:sidekiq_jobs_db_seconds_sum].increment(labels, ActiveRecord::LogSubscriber.runtime / 1000) + @metrics[:sidekiq_jobs_gitaly_seconds_sum].increment(labels, get_gitaly_time(instrumentation)) + @metrics[:sidekiq_redis_requests_duration_seconds_sum].increment(labels, get_redis_time(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_duration_seconds_sum].increment(labels, get_elasticsearch_time(instrumentation)) + end + end + + def record_histograms + @metrics[:sidekiq_jobs_cpu_seconds]&.observe(labels, job_thread_cputime) + + @metrics[:sidekiq_jobs_completion_seconds]&.observe(labels, monotonic_time) + + @metrics[:sidekiq_jobs_db_seconds]&.observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) + @metrics[:sidekiq_jobs_gitaly_seconds]&.observe(labels, get_gitaly_time(instrumentation)) + @metrics[:sidekiq_redis_requests_duration_seconds]&.observe(labels, get_redis_time(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_duration_seconds]&.observe(labels, get_elasticsearch_time(instrumentation)) + end + + def record_queueing_sli + Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration + end + + def record_execution_sli + Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded + Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) + end + def with_load_balancing_settings(job) keys = %w[load_balancing_strategy worker_data_consistency] return unless keys.all? { |k| job.key?(k) } diff --git a/lib/gitlab/sidekiq_middleware/skip_jobs.rb b/lib/gitlab/sidekiq_middleware/skip_jobs.rb index 34ad843e8ee..56b150116a3 100644 --- a/lib/gitlab/sidekiq_middleware/skip_jobs.rb +++ b/lib/gitlab/sidekiq_middleware/skip_jobs.rb @@ -80,13 +80,20 @@ module Gitlab end health_check_attrs = worker_class.database_health_check_attrs - job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first + + tables, schema = health_check_attrs.values_at(:tables, :gitlab_schema) + + if health_check_attrs[:block].respond_to?(:call) + schema, tables = health_check_attrs[:block].call(job['args'], schema, tables) + end + + job_base_model = Gitlab::Database.schemas_to_base_models[schema].first health_context = Gitlab::Database::HealthStatus::Context.new( DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), job_base_model.connection, - health_check_attrs[:tables], - health_check_attrs[:gitlab_schema] + tables, + schema ) Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 778d278146d..ae4aca7ff92 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -94,8 +94,17 @@ module Gitlab keys = job_ids.map { |jid| key_for(jid) } - with_redis { |redis| redis.mget(*keys) } - .map { |result| !result.nil? } + status = with_redis do |redis| + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_get(keys, redis) + else + redis.mget(*keys) + end + end + end + + status.map { |result| !result.nil? } end # Returns the JIDs that are completed diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index f127e14243c..3bbcd59f45e 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# WARNING: This module has been deprecated and will be removed in the future +# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html + module Gitlab module Tracking class << self diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 1b7dcaa5cf4..a9b8dc313d0 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -40,6 +40,8 @@ module Gitlab note_url(object, **options) when Release instance.release_url(object, **options) + when Organizations::Organization + instance.organization_url(object, **options) when Project instance.project_url(object, **options) when Snippet diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 7252283d1b9..941c2f793c4 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -3,7 +3,7 @@ module Gitlab module Usage class MetricDefinition - METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') + METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema', '**', '*.json') AVAILABLE_STATUSES = %w[active broken].to_set.freeze VALID_SERVICE_PING_STATUSES = %w[active broken].to_set.freeze @@ -52,7 +52,7 @@ module Gitlab end def validate! - self.class.schemer.validate(attributes.deep_stringify_keys).each do |error| + errors.each do |error| error_message = <<~ERROR_MSG Error type: #{error['type']} Data: #{error['data']} @@ -104,8 +104,10 @@ module Gitlab definitions[key_path]&.to_context end - def schemer - @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) + def schemers + @schemers ||= Dir[METRIC_SCHEMA_PATH].map do |path| + ::JSONSchemer.schema(Pathname.new(path)) + end end def dump_metrics_yaml @@ -145,6 +147,19 @@ module Gitlab private + def errors + result = [] + + self.class.schemers.each do |schemer| + # schemer.validate returns an Enumerator object + schemer.validate(attributes.deep_stringify_keys).each do |error| + result << error + end + end + + result + end + def method_missing(method, *args) attributes[method] || super end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b2027791e9d..5f819f060e4 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -174,7 +174,6 @@ module Gitlab prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? }, prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::IncomingEmail.enabled? }, - web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { false }, signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index f9dc8bd8a3c..185b49d4a68 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# WARNING: This module has been deprecated and will be removed in the future +# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html + module Gitlab module UsageDataCounters module HLLRedisCounter @@ -53,8 +56,6 @@ module Gitlab private def track(values, event_name, time: Time.zone.now) - return unless ::ServicePing::ServicePingSettings.enabled? - event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb index 591e431c871..3f16681b642 100644 --- a/lib/gitlab/usage_data_counters/redis_counter.rb +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -1,17 +1,16 @@ # frozen_string_literal: true +# WARNING: This module has been deprecated and will be removed in the future +# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html + module Gitlab module UsageDataCounters module RedisCounter def increment(redis_counter_key) - return unless ::ServicePing::ServicePingSettings.enabled? - Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end def increment_by(redis_counter_key, incr) - return unless ::ServicePing::ServicePingSettings.enabled? - Gitlab::Redis::SharedState.with { |redis| redis.incrby(redis_counter_key, incr) } end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index f2db7e3c9b9..057e89a2a97 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -20,7 +20,7 @@ module Gitlab include JwtAuthenticatable class << self - def git_http_ok(repository, repo_type, user, action, show_all_refs: false) + def git_http_ok(repository, repo_type, user, action, show_all_refs: false, need_audit: false) raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) attrs = { @@ -28,6 +28,7 @@ module Gitlab GL_REPOSITORY: repo_type.identifier_for_container(repository.container), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, + NeedAudit: need_audit, Repository: repository.gitaly_repository.to_h, GitConfigOptions: [], GitalyServer: { diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb deleted file mode 100644 index 3ed54a010f8..00000000000 --- a/lib/peek/views/rugged.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Peek - module Views - class Rugged < DetailedView - def results - return {} unless calls > 0 - - super - end - - private - - def duration - ::Gitlab::RuggedInstrumentation.query_time_ms - end - - def calls - ::Gitlab::RuggedInstrumentation.query_count - end - - def call_details - ::Gitlab::RuggedInstrumentation.list_call_details - end - - def format_call_details(call) - super.merge(args: format_args(call[:args])) - end - - def format_args(args) - args.map do |arg| - # ActiveSupport::JSON recursively calls as_json on all - # instance variables, and if that instance variable points to - # something that refers back to the same instance, we can wind - # up in an infinite loop. Currently this only seems to happen with - # Gitlab::Git::Repository and ::Repository. - if arg.instance_variables.present? - arg.to_s - else - arg - end - end - end - end - end -end diff --git a/lib/sbom/purl_type/converter.rb b/lib/sbom/purl_type/converter.rb index bfcfb414180..bc08083fdae 100644 --- a/lib/sbom/purl_type/converter.rb +++ b/lib/sbom/purl_type/converter.rb @@ -18,6 +18,7 @@ module Sbom 'nuget' => 'nuget', 'pip' => 'pypi', 'pipenv' => 'pypi', + 'poetry' => 'pypi', 'setuptools' => 'pypi', 'python-pkg' => 'pypi' # this package manager is generated by trivy }.with_indifferent_access.freeze diff --git a/lib/sidebars/admin/menus/admin_settings_menu.rb b/lib/sidebars/admin/menus/admin_settings_menu.rb index 4656e0f33e2..4d2d19c60f7 100644 --- a/lib/sidebars/admin/menus/admin_settings_menu.rb +++ b/lib/sidebars/admin/menus/admin_settings_menu.rb @@ -12,7 +12,6 @@ module Sidebars add_item(ci_cd_menu_item) add_item(reporting_menu_item) add_item(metrics_and_profiling_menu_item) - add_item(service_usage_data_menu_item) add_item(network_settings_menu_item) add_item(appearance_menu_item) add_item(preferences_menu_item) @@ -102,15 +101,6 @@ module Sidebars ) end - def service_usage_data_menu_item - ::Sidebars::MenuItem.new( - title: _('Service usage data'), - link: service_usage_data_admin_application_settings_path, - active_routes: { path: 'admin/application_settings#service_usage_data' }, - item_id: :admin_service_usage - ) - end - def network_settings_menu_item ::Sidebars::MenuItem.new( title: _('Network'), diff --git a/lib/sidebars/explore/menus/catalog_menu.rb b/lib/sidebars/explore/menus/catalog_menu.rb new file mode 100644 index 00000000000..2d8e8bba08b --- /dev/null +++ b/lib/sidebars/explore/menus/catalog_menu.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sidebars + module Explore + module Menus + class CatalogMenu < ::Sidebars::Menu + override :link + def link + explore_catalog_index_path + end + + override :title + def title + _('CI/CD Catalog') + end + + override :sprite_icon + def sprite_icon + 'catalog-checkmark' + end + + override :render? + def render? + Feature.enabled?(:global_ci_catalog, current_user) + end + + override :active_routes + def active_routes + { controller: ['explore/catalog'] } + end + end + end + end +end diff --git a/lib/sidebars/explore/panel.rb b/lib/sidebars/explore/panel.rb index 6260df6bb5f..3559f7d9627 100644 --- a/lib/sidebars/explore/panel.rb +++ b/lib/sidebars/explore/panel.rb @@ -28,6 +28,7 @@ module Sidebars def add_menus add_menu(Sidebars::Explore::Menus::ProjectsMenu.new(context)) add_menu(Sidebars::Explore::Menus::GroupsMenu.new(context)) + add_menu(Sidebars::Explore::Menus::CatalogMenu.new(context)) add_menu(Sidebars::Explore::Menus::TopicsMenu.new(context)) add_menu(Sidebars::Explore::Menus::SnippetsMenu.new(context)) end diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb index ee02429baf3..8fcb373c9dc 100644 --- a/lib/sidebars/menu.rb +++ b/lib/sidebars/menu.rb @@ -80,6 +80,7 @@ module Sidebars is_active = @context.route_is_active.call(active_routes) || items.any? { |item| item[:is_active] } { + id: self.class.name.demodulize.underscore, title: title, icon: sprite_icon, avatar: avatar, diff --git a/lib/sidebars/organizations/menus/manage_menu.rb b/lib/sidebars/organizations/menus/manage_menu.rb index 0df716cdd3f..7c342002c31 100644 --- a/lib/sidebars/organizations/menus/manage_menu.rb +++ b/lib/sidebars/organizations/menus/manage_menu.rb @@ -30,6 +30,15 @@ module Sidebars item_id: :organization_groups_and_projects ) ) + add_item( + ::Sidebars::MenuItem.new( + title: _('Users'), + link: users_organization_path(context.container), + super_sidebar_parent: ::Sidebars::Organizations::Menus::ManageMenu, + active_routes: { path: 'organizations/organizations#users' }, + item_id: :organization_users + ) + ) end end end diff --git a/lib/sidebars/projects/menus/ci_cd_menu.rb b/lib/sidebars/projects/menus/ci_cd_menu.rb index 02596b16cfa..c77e8e996b0 100644 --- a/lib/sidebars/projects/menus/ci_cd_menu.rb +++ b/lib/sidebars/projects/menus/ci_cd_menu.rb @@ -18,7 +18,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options { - class: 'shortcuts-pipelines rspec-link-pipelines' + class: 'shortcuts-pipelines' } end diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index b08845a37e6..d3c9f3a6466 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -70,7 +70,7 @@ module Sidebars highlight: Users::CalloutsHelper::GKE_CLUSTER_INTEGRATION, highlight_priority: Users::Callout.feature_names[:GKE_CLUSTER_INTEGRATION], dismiss_endpoint: callouts_path, - auto_devops_help_path: help_page_path('topics/autodevops/index.md') } } + auto_devops_help_path: help_page_path('topics/autodevops/index') } } end def terraform_states_menu_item diff --git a/lib/sidebars/projects/menus/scope_menu.rb b/lib/sidebars/projects/menus/scope_menu.rb index f388c814bd7..d03abfdfb7e 100644 --- a/lib/sidebars/projects/menus/scope_menu.rb +++ b/lib/sidebars/projects/menus/scope_menu.rb @@ -22,7 +22,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options { - class: 'shortcuts-project rspec-project-link' + class: 'shortcuts-project' } end diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 8fed1c46425..077eebf58b9 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -57,10 +57,6 @@ module Sidebars monitor_menu_item, usage_quotas_menu_item ] - elsif context.current_user && can?(context.current_user, :manage_resource_access_tokens, context.project) - [ - access_tokens_menu_item - ] else [] end diff --git a/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb b/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb index 0441d3b4a03..d4ecf132c44 100644 --- a/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/monitor_menu.rb @@ -18,6 +18,7 @@ module Sidebars def configure_menu_items [ :tracing, + :metrics, :error_tracking, :alert_management, :incidents, diff --git a/lib/sidebars/user_settings/menus/comment_templates_menu.rb b/lib/sidebars/user_settings/menus/comment_templates_menu.rb index da37c42bbd4..1e9aea8ec9a 100644 --- a/lib/sidebars/user_settings/menus/comment_templates_menu.rb +++ b/lib/sidebars/user_settings/menus/comment_templates_menu.rb @@ -23,7 +23,7 @@ module Sidebars override :render? def render? - !!context.current_user && saved_replies_enabled? + !!context.current_user end override :active_routes diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index 240b808baf3..917fce42762 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -3,7 +3,7 @@ namespace :gitlab do namespace :import do desc "GitLab | Import | Add all users to all projects (admin users are added as maintainers)" - task all_users_to_all_projects: :environment do |t, args| + task all_users_to_all_projects: :environment do |t, args| user_ids = User.where(admin: false).pluck(:id) admin_ids = User.where(admin: true).pluck(:id) projects = Project.all diff --git a/lib/tasks/gitlab/click_house/migration.rake b/lib/tasks/gitlab/click_house/migration.rake new file mode 100644 index 00000000000..ddac81ec98f --- /dev/null +++ b/lib/tasks/gitlab/click_house/migration.rake @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :clickhouse do + task :prepare_schema_migration_table, [:database] => :environment do |_t, args| + require_relative '../../../../lib/click_house/migration_support/schema_migration' + + ClickHouse::MigrationSupport::SchemaMigration.create_table(args.database&.to_sym || :main) + end + + desc 'GitLab | ClickHouse | Migrate' + task migrate: [:prepare_schema_migration_table] do + migrate(:up) + end + + desc 'GitLab | ClickHouse | Rollback' + task rollback: [:prepare_schema_migration_table] do + migrate(:down) + end + + private + + def check_target_version + return unless target_version + + version = ENV['VERSION'] + + return if ClickHouse::Migration::MIGRATION_FILENAME_REGEXP.match?(version) || /\A\d+\z/.match?(version) + + raise "Invalid format of target version: `VERSION=#{version}`" + end + + def target_version + ENV['VERSION'].to_i if ENV['VERSION'] && !ENV['VERSION'].empty? + end + + def migrate(direction) + require_relative '../../../../lib/click_house/migration_support/schema_migration' + require_relative '../../../../lib/click_house/migration_support/migration_context' + require_relative '../../../../lib/click_house/migration_support/migrator' + + check_target_version + + scope = ENV['SCOPE'] + verbose_was = ClickHouse::Migration.verbose + ClickHouse::Migration.verbose = ENV['VERBOSE'] ? ENV['VERBOSE'] != 'false' : true + + migrations_paths = ClickHouse::MigrationSupport::Migrator.migrations_paths + schema_migration = ClickHouse::MigrationSupport::SchemaMigration + migration_context = ClickHouse::MigrationSupport::MigrationContext.new(migrations_paths, schema_migration) + migrations_ran = migration_context.public_send(direction, target_version) do |migration| + scope.blank? || scope == migration.scope + end + + puts('No migrations ran.') unless migrations_ran&.any? + ensure + ClickHouse::Migration.verbose = verbose_was + end + end +end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index cf52a219e83..d89ab548419 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -107,7 +107,10 @@ namespace :gitlab do end end - Rake::Task['db:seed_fu'].invoke if databases_loaded.present? && databases_loaded.all? + if databases_loaded.present? && databases_loaded.all? + Rake::Task["gitlab:db:lock_writes"].invoke + Rake::Task['db:seed_fu'].invoke + end end def configure_database(connection, database_name: nil) @@ -454,7 +457,12 @@ namespace :gitlab do ActiveRecord::Base.establish_connection(config) # rubocop: disable Database/EstablishConnection Gitlab::Database.check_for_non_superuser - Rake::Task['db:migrate'].invoke + + if Rake::Task.task_defined?("db:migrate:#{db_config.name}") + Rake::Task["db:migrate:#{db_config.name}"].invoke + else + Rake::Task["db:migrate"].invoke + end end end diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake deleted file mode 100644 index e44328e0de1..00000000000 --- a/lib/tasks/gitlab/features.rake +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -namespace :gitlab do - namespace :features do - desc 'GitLab | Features | Enable direct Git access via Rugged for NFS' - task enable_rugged: :environment do - set_rugged_feature_flags(true) - puts 'All Rugged feature flags were enabled.' - end - - task disable_rugged: :environment do - set_rugged_feature_flags(false) - puts 'All Rugged feature flags were disabled.' - end - - task unset_rugged: :environment do - set_rugged_feature_flags(nil) - puts 'All Rugged feature flags were unset.' - end - end - - def set_rugged_feature_flags(status) - Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag| - case status - when nil - Feature.remove(flag) - when true - Feature.enable(flag) - when false - Feature.disable(flag) - end - end - end -end diff --git a/lib/tasks/gitlab/redis.rake b/lib/tasks/gitlab/redis.rake new file mode 100644 index 00000000000..6983c5fc318 --- /dev/null +++ b/lib/tasks/gitlab/redis.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :redis do + namespace :secret do + desc "GitLab | Redis | Secret | Show Redis secret" + task :show, [:instance_name] => [:environment] do |_t, args| + Gitlab::EncryptedRedisCommand.show(args: args) + end + + desc "GitLab | Redis | Secret | Edit Redis secret" + task :edit, [:instance_name] => [:environment] do |_t, args| + Gitlab::EncryptedRedisCommand.edit(args: args) + end + + desc "GitLab | Redis | Secret | Write Redis secret" + task :write, [:instance_name] => [:environment] do |_t, args| + content = $stdin.tty? ? $stdin.gets : $stdin.read + Gitlab::EncryptedRedisCommand.write(content, args: args) + end + end + end +end diff --git a/lib/tasks/gitlab/seed/ci_catalog_resources.rake b/lib/tasks/gitlab/seed/ci_catalog_resources.rake new file mode 100644 index 00000000000..1db995aa801 --- /dev/null +++ b/lib/tasks/gitlab/seed/ci_catalog_resources.rake @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# This task should be enabled when the seeder gets fixed: +# https://gitlab.com/gitlab-org/gitlab/-/issues/429649 +# +# Seed CI/CD catalog resources +# +# @param group_path - Group name under which to create the projects +# @param seed_count - Total number of Catalog resources to create (default: 30) +# +# @example +# bundle exec rake "gitlab:seed:ci_catalog_resources[root, 50]" +# +# namespace :gitlab do +# namespace :seed do +# desc 'Seed CI Catalog resources' +# task :ci_catalog_resources, +# [:group_path, :seed_count] => :gitlab_environment do |_t, args| +# Gitlab::Seeders::Ci::Catalog::ResourceSeeder.new( +# group_path: args.group_path, +# seed_count: args.seed_count&.to_i +# ).seed +# puts "Task finished!" +# end +# end +# end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 495d7a339b8..de1401feb8a 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -26,32 +26,31 @@ namespace :tw do CodeOwnerRule.new('Analytics Instrumentation', '@lciutacu'), CodeOwnerRule.new('Anti-Abuse', '@phillipwells'), CodeOwnerRule.new('Cloud Connector', '@jglassman1'), - CodeOwnerRule.new('Authentication and Authorization', '@jglassman1'), + CodeOwnerRule.new('Authentication', '@jglassman1'), + CodeOwnerRule.new('Authorization', '@jglassman1'), # CodeOwnerRule.new('Billing and Subscription Management', ''), CodeOwnerRule.new('Code Creation', '@jglassman1'), CodeOwnerRule.new('Code Review', '@aqualls'), CodeOwnerRule.new('Compliance', '@eread'), CodeOwnerRule.new('Composition Analysis', '@rdickenson'), - CodeOwnerRule.new('Environments', '@phillipwells'), CodeOwnerRule.new('Container Registry', '@marcel.amirault'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Database', '@aqualls'), CodeOwnerRule.new('DataOps', '@sselhorn'), # CodeOwnerRule.new('Delivery', ''), - CodeOwnerRule.new('Development', '@sselhorn'), CodeOwnerRule.new('Distribution', '@axil'), CodeOwnerRule.new('Distribution (Charts)', '@axil'), CodeOwnerRule.new('Distribution (Omnibus)', '@eread'), - CodeOwnerRule.new('Documentation Guidelines', '@sselhorn'), CodeOwnerRule.new('Duo Chat', '@sselhorn'), CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'), CodeOwnerRule.new('Editor Extensions', '@aqualls'), + CodeOwnerRule.new('Environments', '@phillipwells'), CodeOwnerRule.new('Foundations', '@sselhorn'), # CodeOwnerRule.new('Fulfillment Platform', ''), CodeOwnerRule.new('Fuzz Testing', '@rdickenson'), CodeOwnerRule.new('Geo', '@axil'), CodeOwnerRule.new('Gitaly', '@eread'), - # CodeOwnerRule.new('GitLab Dedicated', ''), + CodeOwnerRule.new('GitLab Dedicated', '@lyspin'), CodeOwnerRule.new('Global Search', '@ashrafkhamis'), CodeOwnerRule.new('IDE', '@ashrafkhamis'), CodeOwnerRule.new('Import and Integrate', '@eread @ashrafkhamis'), @@ -75,9 +74,9 @@ namespace :tw do CodeOwnerRule.new('Runner', '@fneill'), CodeOwnerRule.new('Runner SaaS', '@fneill'), CodeOwnerRule.new('Security Policies', '@rdickenson'), + CodeOwnerRule.new('Solutions Architecture', '@jfullam @brianwald @Darwinjs'), CodeOwnerRule.new('Source Code', '@msedlakjakubowski'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), - CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Tenant Scale', '@lciutacu'), CodeOwnerRule.new('Testing', '@eread'), CodeOwnerRule.new('Threat Insights', '@rdickenson'), @@ -87,6 +86,33 @@ namespace :tw do # CodeOwnerRule.new('Vulnerability Research', '') ].freeze + CONTRIBUTOR_DOCS_PATH = '/doc/development/' + CONTRIBUTOR_DOCS_CODE_OWNER_RULES = [ + CodeOwnerRule.new('Analytics Instrumentation', + '@gitlab-org/analytics-section/product-analytics/engineers/frontend ' \ + '@gitlab-org/analytics-section/analytics-instrumentation/engineers'), + CodeOwnerRule.new('Authentication', '@gitlab-org/govern/authentication/approvers'), + CodeOwnerRule.new('Authorization', '@gitlab-org/govern/authorization/approvers'), + CodeOwnerRule.new('Compliance', + '@gitlab-org/govern/security-policies-frontend @gitlab-org/govern/threat-insights-frontend-team ' \ + '@gitlab-org/govern/threat-insights-backend-team'), + CodeOwnerRule.new('Composition Analysis', + '@gitlab-org/secure/composition-analysis-be @gitlab-org/secure/static-analysis'), + CodeOwnerRule.new('Distribution', '@gitlab-org/distribution'), + CodeOwnerRule.new('Documentation Guidelines', '@sselhorn'), + CodeOwnerRule.new('Engineering Productivity', '@gl-quality/eng-prod'), + CodeOwnerRule.new('Foundations', '@gitlab-org/manage/foundations/engineering'), + CodeOwnerRule.new('Gitaly', '@proglottis @toon'), + CodeOwnerRule.new('Global Search', '@gitlab-org/search-team/migration-maintainers'), + CodeOwnerRule.new('IDE', + '@gitlab-org/maintainers/remote-development/backend @gitlab-org/maintainers/remote-development/frontend'), + CodeOwnerRule.new('Pipeline Authoring', '@gitlab-org/maintainers/cicd-verify'), + CodeOwnerRule.new('Pipeline Execution', '@gitlab-org/maintainers/cicd-verify'), + CodeOwnerRule.new('Product Analytics', '@gitlab-org/analytics-section/product-analytics/engineers/frontend'), + CodeOwnerRule.new('Tenant Scale', '@abdwdd @alexpooley @manojmj'), + CodeOwnerRule.new('Threat Insights', '@gitlab-org/govern/threat-insights-frontend-team') + ].freeze + ERRORS_EXCLUDED_FILES = [ '/doc/architecture' ].freeze @@ -105,7 +131,8 @@ namespace :tw do end def self.writer_for_group(category, path) - writer = CODE_OWNER_RULES.find { |rule| rule.category == category }&.writer + rules = path.start_with?(CONTRIBUTOR_DOCS_PATH) ? CONTRIBUTOR_DOCS_CODE_OWNER_RULES : CODE_OWNER_RULES + writer = rules.find { |rule| rule.category == category }&.writer if writer.is_a?(String) || writer.nil? writer diff --git a/lib/tasks/tanuki_emoji.rake b/lib/tasks/tanuki_emoji.rake index b3099853434..aec7d3c1bf6 100644 --- a/lib/tasks/tanuki_emoji.rake +++ b/lib/tasks/tanuki_emoji.rake @@ -30,9 +30,9 @@ namespace :tanuki_emoji do require 'digest/sha2' digest_emoji_map = {} - emojis_map = {} + emojis_array = [] - TanukiEmoji.index.all.each do |emoji| + TanukiEmoji.index.all.sort_by(&:sort_key).each do |emoji| emoji_path = Gitlab::Emoji.emoji_public_absolute_path.join("#{emoji.name}.png") digest_entry = { @@ -47,13 +47,14 @@ namespace :tanuki_emoji do # Our new map is only characters to make the json substantially smaller emoji_entry = { + n: emoji.name, c: emoji.category, e: emoji.codepoints, d: emoji.description, u: emoji.unicode_version } - emojis_map[emoji.name] = emoji_entry + emojis_array << emoji_entry end digests_json = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') @@ -63,7 +64,7 @@ namespace :tanuki_emoji do emojis_json = Gitlab::Emoji.emoji_public_absolute_path.join('emojis.json') File.open(emojis_json, 'w') do |handle| - handle.write(Gitlab::Json.pretty_generate(emojis_map)) + handle.write(Gitlab::Json.pretty_generate(emojis_array)) end end diff --git a/lib/unnested_in_filters/rewriter.rb b/lib/unnested_in_filters/rewriter.rb index 9eb1c0b8273..2e334eb147b 100644 --- a/lib/unnested_in_filters/rewriter.rb +++ b/lib/unnested_in_filters/rewriter.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# rubocop:disable CodeReuse/ActiveRecord (This module is generating ActiveRecord relations therefore using AR methods is necessary) +# rubocop:disable CodeReuse/ActiveRecord -- This module is generating ActiveRecord relations therefore using AR methods is necessary module UnnestedInFilters class Rewriter include Gitlab::Utils::StrongMemoize @@ -295,3 +295,4 @@ module UnnestedInFilters end end end +# rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/vs_code/settings.rb b/lib/vs_code/settings.rb index 30b91ebb16f..0cc2245eae1 100644 --- a/lib/vs_code/settings.rb +++ b/lib/vs_code/settings.rb @@ -15,7 +15,7 @@ module VsCode } ] }.freeze - SETTINGS_TYPES = %w[settings extensions globalState machines keybindings snippets tasks].freeze + SETTINGS_TYPES = %w[settings extensions globalState machines keybindings snippets tasks profiles].freeze DEFAULT_SESSION = "1" NO_CONTENT_ETAG = "0" end |