diff options
Diffstat (limited to 'lib')
614 files changed, 17082 insertions, 3978 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 74f6515f07f..38a9856ca58 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -19,6 +19,7 @@ module API desc "Gets a list of access requests for a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester + tags %w[access_requests] end params do use :pagination @@ -37,6 +38,24 @@ module API desc "Requests access for the authenticated user to a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester + success [ + { + code: 200, + model: Entities::AccessRequester, + message: 'successful operation', + examples: { + successfull_response: { + "id" => 1, + "username" => "raymond_smith", + "name" => "Raymond Smith", + "state" => "active", + "created_at" => "2012-10-22T14:13:35Z", + "access_level" => 20 + } + } + } + ] + tags %w[access_requests] end post ":id/access_requests" do source = find_source(source_type, params[:id]) @@ -51,7 +70,24 @@ module API desc 'Approves an access request for the given user.' do detail 'This feature was introduced in GitLab 8.11.' - success Entities::Member + success [ + { + code: 200, + model: Entities::AccessRequester, + message: 'successful operation', + examples: { + successfull_response: { + "id" => 1, + "username" => "raymond_smith", + "name" => "Raymond Smith", + "state" => "active", + "created_at" => "2012-10-22T14:13:35Z", + "access_level" => 20 + } + } + } + ] + tags %w[access_requests] end params do requires :user_id, type: Integer, desc: 'The user ID of the access requester' @@ -74,6 +110,7 @@ module API desc 'Denies an access request for the given user.' do detail 'This feature was introduced in GitLab 8.11.' + tags %w[access_requests] end params do requires :user_id, type: Integer, desc: 'The user ID of the access requester' diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb index 0462878c90c..bc351e27f99 100644 --- a/lib/api/admin/ci/variables.rb +++ b/lib/api/admin/ci/variables.rb @@ -13,8 +13,9 @@ module API namespace 'admin' do namespace 'ci' do namespace 'variables' do - desc 'Get instance-level variables' do + desc 'List all instance-level variables' do success Entities::Ci::Variable + tags %w[ci_variables] end params do use :pagination @@ -25,11 +26,13 @@ module API present paginate(variables), with: Entities::Ci::Variable end - desc 'Get a specific variable from a group' do + desc 'Get the details of a specific instance-level variable' do success Entities::Ci::Variable + failure [{ code: 404, message: 'Instance Variable Not Found' }] + tags %w[ci_variables] end params do - requires :key, type: String, desc: 'The key of the variable' + requires :key, type: String, desc: 'The key of a variable' end get ':key' do key = params[:key] @@ -42,28 +45,35 @@ module API desc 'Create a new instance-level variable' do success Entities::Ci::Variable + failure [{ code: 400, message: '400 Bad Request' }] + tags %w[ci_variables] end + route_setting :log_safety, { safe: %w[key], unsafe: %w[value] } params do requires :key, type: String, - desc: 'The key of the variable' + desc: 'The key of the variable. Max 255 characters' requires :value, type: String, - desc: 'The value of the variable' + desc: 'The value of a variable' optional :protected, - type: String, + type: Boolean, desc: 'Whether the variable is protected' optional :masked, - type: String, + type: Boolean, desc: 'Whether the variable is masked' + optional :raw, + type: Boolean, + desc: 'Whether the variable will be expanded' + optional :variable_type, type: String, values: ::Ci::InstanceVariable.variable_types.keys, - desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' + desc: 'The type of a variable. Available types are: env_var (default) and file' end post '/' do variable_params = declared_params(include_missing: false) @@ -77,30 +87,37 @@ module API end end - desc 'Update an existing instance-variable' do + desc 'Update an instance-level variable' do success Entities::Ci::Variable + failure [{ code: 404, message: 'Instance Variable Not Found' }] + tags %w[ci_variables] end + route_setting :log_safety, { safe: %w[key], unsafe: %w[value] } params do optional :key, type: String, - desc: 'The key of the variable' + desc: 'The key of a variable' optional :value, type: String, - desc: 'The value of the variable' + desc: 'The value of a variable' optional :protected, - type: String, + type: Boolean, desc: 'Whether the variable is protected' optional :masked, - type: String, + type: Boolean, desc: 'Whether the variable is masked' + optional :raw, + type: Boolean, + desc: 'Whether the variable will be expanded' + optional :variable_type, type: String, values: ::Ci::InstanceVariable.variable_types.keys, - desc: 'The type of variable, must be one of env_var or file' + desc: 'The type of a variable. Available types are: env_var (default) and file' end put ':key' do variable = ::Ci::InstanceVariable.find_by_key(params[:key]) @@ -118,9 +135,11 @@ module API desc 'Delete an existing instance-level variable' do success Entities::Ci::Variable + failure [{ code: 404, message: 'Instance Variable Not Found' }] + tags %w[ci_variables] end params do - requires :key, type: String, desc: 'The key of the variable' + requires :key, type: String, desc: 'The key of a variable' end delete ':key' do variable = ::Ci::InstanceVariable.find_by_key(params[:key]) diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index 7163225777a..f848103d9a0 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -14,16 +14,28 @@ module API end namespace 'admin' do - desc "Get list of all instance clusters" do - detail "This feature was introduced in GitLab 13.2." + desc 'List instance clusters' do + detail 'This feature was introduced in GitLab 13.2. Returns a list of instance clusters.' + success Entities::Cluster + failure [ + { code: 403, message: 'Forbidden' } + ] + is_array true + tags %w[clusters] end get '/clusters' do authorize! :read_cluster, clusterable_instance present paginate(clusters_for_current_user), with: Entities::Cluster end - desc "Get a single instance cluster" do - detail "This feature was introduced in GitLab 13.2." + desc 'Get a single instance cluster' do + detail 'This feature was introduced in GitLab 13.2. Returns a single instance cluster.' + success Entities::Cluster + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: "The cluster ID" @@ -34,8 +46,15 @@ module API present cluster, with: Entities::Cluster end - desc "Add an instance cluster" do - detail "This feature was introduced in GitLab 13.2." + desc 'Add existing instance cluster' do + detail 'This feature was introduced in GitLab 13.2. Adds an existing Kubernetes instance cluster.' + success Entities::Cluster + failure [ + { code: 400, message: 'Validation error' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :name, type: String, desc: 'Cluster name' @@ -67,8 +86,15 @@ module API end end - desc "Update an instance cluster" do - detail "This feature was introduced in GitLab 13.2." + desc 'Edit instance cluster' do + detail 'This feature was introduced in GitLab 13.2. Updates an existing instance cluster.' + success Entities::Cluster + failure [ + { code: 400, message: 'Validation error' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: 'The cluster ID' @@ -98,8 +124,14 @@ module API end end - desc "Remove a cluster" do - detail "This feature was introduced in GitLab 13.2." + desc 'Delete instance cluster' do + detail 'This feature was introduced in GitLab 13.2. Deletes an existing instance cluster. Does not remove existing resources within the connected Kubernetes cluster.' + success Entities::Cluster + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: "The cluster ID" diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb index 7ce70d85d46..49b41b44a18 100644 --- a/lib/api/admin/plan_limits.rb +++ b/lib/api/admin/plan_limits.rb @@ -5,6 +5,8 @@ module API class PlanLimits < ::API::Base before { authenticated_as_admin! } + PLAN_LIMITS_TAGS = %w[plan_limits].freeze + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned helpers do @@ -17,10 +19,17 @@ module API end desc 'Get current plan limits' do + detail 'List the current limits of a plan on the GitLab instance.' success Entities::PlanLimit + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' } + ] + tags PLAN_LIMITS_TAGS end params do - optional :plan_name, type: String, values: Plan.all_plans, default: Plan::DEFAULT, desc: 'Name of the plan' + optional :plan_name, type: String, values: Plan.all_plans, default: Plan::DEFAULT, + desc: 'Name of the plan to get the limits from. Default: default.' end get "application/plan_limits" do params = declared_params(include_missing: false) @@ -29,16 +38,24 @@ module API present plan.actual_limits, with: Entities::PlanLimit end - desc 'Modify plan limits' do + desc 'Change plan limits' do + detail 'Modify the limits of a plan on the GitLab instance.' success Entities::PlanLimit + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' } + ] + tags PLAN_LIMITS_TAGS end params do - requires :plan_name, type: String, values: Plan.all_plans, desc: 'Name of the plan' + requires :plan_name, type: String, values: Plan.all_plans, desc: 'Name of the plan to update' optional :ci_pipeline_size, type: Integer, desc: 'Maximum number of jobs in a single pipeline' optional :ci_active_jobs, type: Integer, desc: 'Total number of jobs in currently active pipelines' optional :ci_active_pipelines, type: Integer, desc: 'Maximum number of active pipelines per project' - optional :ci_project_subscriptions, type: Integer, desc: 'Maximum number of pipeline subscriptions to and from a project' + optional :ci_project_subscriptions, type: Integer, + desc: 'Maximum number of pipeline subscriptions to and from a project' optional :ci_pipeline_schedules, type: Integer, desc: 'Maximum number of pipeline schedules' optional :ci_needs_size_limit, type: Integer, desc: 'Maximum number of DAG dependencies that a job can have' optional :ci_registered_group_runners, type: Integer, desc: 'Maximum number of runners registered per group' @@ -50,7 +67,8 @@ module API optional :npm_max_file_size, type: Integer, desc: 'Maximum NPM package file size in bytes' optional :nuget_max_file_size, type: Integer, desc: 'Maximum NuGet package file size in bytes' optional :pypi_max_file_size, type: Integer, desc: 'Maximum PyPI package file size in bytes' - optional :terraform_module_max_file_size, type: Integer, desc: 'Maximum Terraform Module package file size in bytes' + optional :terraform_module_max_file_size, type: Integer, + desc: 'Maximum Terraform Module package file size in bytes' optional :storage_size_limit, type: Integer, desc: 'Maximum storage size for the root namespace in megabytes' end put "application/plan_limits" do diff --git a/lib/api/alert_management_alerts.rb b/lib/api/alert_management_alerts.rb index f03f133f6f7..f57b7d00c81 100644 --- a/lib/api/alert_management_alerts.rb +++ b/lib/api/alert_management_alerts.rb @@ -6,7 +6,7 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' requires :alert_iid, type: Integer, desc: 'The IID of the Alert' end diff --git a/lib/api/api.rb b/lib/api/api.rb index 933c3f69075..ffb0cdf8991 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -13,13 +13,14 @@ module API USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze LOG_FILTERS = ::Rails.application.config.filter_parameters + [/^output$/] LOG_FORMATTER = Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new + LOGGER = Logger.new(LOG_FILENAME) insert_before Grape::Middleware::Error, GrapeLogging::Middleware::RequestLogger, - logger: Logger.new(LOG_FILENAME), + logger: LOGGER, formatter: LOG_FORMATTER, include: [ - GrapeLogging::Loggers::FilterParameters.new(LOG_FILTERS), + Gitlab::GrapeLogging::Loggers::FilterParameters.new(LOG_FILTERS), Gitlab::GrapeLogging::Loggers::ClientEnvLogger.new, Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, @@ -168,42 +169,111 @@ module API # Mount endpoints to include in the OpenAPI V2 documentation here namespace do + # Keep in alphabetical order + mount ::API::AccessRequests + mount ::API::Admin::Ci::Variables + mount ::API::Admin::InstanceClusters + mount ::API::Admin::PlanLimits + mount ::API::Appearance + mount ::API::Applications + mount ::API::Avatar + mount ::API::Badges + mount ::API::Branches + mount ::API::BroadcastMessages + mount ::API::BulkImports + mount ::API::Ci::Jobs + mount ::API::Ci::ResourceGroups + mount ::API::Ci::Runner + mount ::API::Ci::Runners + mount ::API::Ci::Pipelines + mount ::API::Ci::PipelineSchedules + mount ::API::Ci::Triggers + mount ::API::Ci::Variables + mount ::API::Clusters::AgentTokens + mount ::API::Clusters::Agents + mount ::API::Commits + mount ::API::CommitStatuses + mount ::API::DependencyProxy + mount ::API::DeployKeys + mount ::API::DeployTokens + mount ::API::Deployments + mount ::API::Environments + mount ::API::ErrorTracking::ClientKeys + mount ::API::ErrorTracking::ProjectSettings + mount ::API::FeatureFlags + mount ::API::FeatureFlagsUserLists + mount ::API::Features + mount ::API::Files + mount ::API::FreezePeriods + mount ::API::Geo + mount ::API::GoProxy + mount ::API::GroupAvatar + mount ::API::GroupClusters + mount ::API::GroupContainerRepositories + mount ::API::GroupExport + mount ::API::GroupImport + mount ::API::GroupPackages + mount ::API::GroupVariables + mount ::API::ImportBitbucketServer + mount ::API::ImportGithub + mount ::API::Integrations + mount ::API::Invitations + mount ::API::IssueLinks + mount ::API::Keys + mount ::API::Lint + mount ::API::Markdown + mount ::API::MergeRequestApprovals + mount ::API::MergeRequestDiffs mount ::API::Metadata + mount ::API::Metrics::Dashboard::Annotations + mount ::API::Metrics::UserStarredDashboards + mount ::API::PackageFiles + mount ::API::PersonalAccessTokens::SelfInformation + mount ::API::PersonalAccessTokens + mount ::API::ProjectClusters + mount ::API::ProjectEvents + mount ::API::ProjectExport + mount ::API::ProjectHooks + mount ::API::ProjectImport + mount ::API::ProjectRepositoryStorageMoves + mount ::API::ProjectSnippets + mount ::API::ProjectSnapshots + mount ::API::ProjectStatistics + mount ::API::ProjectTemplates + mount ::API::ProtectedBranches + mount ::API::ProtectedTags + mount ::API::Releases + mount ::API::Release::Links + mount ::API::RemoteMirrors + mount ::API::Repositories + mount ::API::ResourceAccessTokens + mount ::API::ResourceMilestoneEvents + mount ::API::Snippets + mount ::API::SnippetRepositoryStorageMoves + mount ::API::Statistics + mount ::API::Submodules + mount ::API::Suggestions + mount ::API::SystemHooks + mount ::API::Tags + mount ::API::Terraform::Modules::V1::Packages + mount ::API::Terraform::State + mount ::API::Terraform::StateVersion + mount ::API::Topics + mount ::API::Unleash + mount ::API::UserCounts + mount ::API::Wikis add_open_api_documentation! end # Keep in alphabetical order - mount ::API::AccessRequests mount ::API::Admin::BatchedBackgroundMigrations - mount ::API::Admin::Ci::Variables - mount ::API::Admin::InstanceClusters - mount ::API::Admin::PlanLimits mount ::API::Admin::Sidekiq mount ::API::AlertManagementAlerts - mount ::API::Appearance - mount ::API::Applications - mount ::API::Avatar mount ::API::AwardEmoji - mount ::API::Badges mount ::API::Boards - mount ::API::Branches - mount ::API::BroadcastMessages - mount ::API::BulkImports mount ::API::Ci::JobArtifacts - mount ::API::Ci::Jobs - mount ::API::Ci::PipelineSchedules - mount ::API::Ci::Pipelines - mount ::API::Ci::ResourceGroups - mount ::API::Ci::Runner - mount ::API::Ci::Runners mount ::API::Ci::SecureFiles - mount ::API::Ci::Triggers - mount ::API::Ci::Variables - mount ::API::Clusters::Agents - mount ::API::Clusters::AgentTokens - mount ::API::CommitStatuses - mount ::API::Commits mount ::API::ComposerPackages mount ::API::ConanInstancePackages mount ::API::ConanProjectPackages @@ -211,55 +281,22 @@ module API mount ::API::ContainerRepositories mount ::API::DebianGroupPackages mount ::API::DebianProjectPackages - mount ::API::DependencyProxy - mount ::API::DeployKeys - mount ::API::DeployTokens - mount ::API::Deployments mount ::API::Discussions - mount ::API::Environments - mount ::API::ErrorTracking::ClientKeys mount ::API::ErrorTracking::Collector - mount ::API::ErrorTracking::ProjectSettings mount ::API::Events - mount ::API::FeatureFlags - mount ::API::FeatureFlagsUserLists - mount ::API::Features - mount ::API::Files - mount ::API::FreezePeriods mount ::API::GenericPackages - mount ::API::Geo - mount ::API::GoProxy - mount ::API::GroupAvatar mount ::API::GroupBoards - mount ::API::GroupClusters - mount ::API::GroupContainerRepositories mount ::API::GroupDebianDistributions - mount ::API::GroupExport - mount ::API::GroupImport mount ::API::GroupLabels mount ::API::GroupMilestones - mount ::API::GroupPackages - mount ::API::GroupVariables mount ::API::Groups mount ::API::HelmPackages - mount ::API::ImportBitbucketServer - mount ::API::ImportGithub - mount ::API::Integrations mount ::API::Integrations::JiraConnect::Subscriptions - mount ::API::Invitations - mount ::API::IssueLinks mount ::API::Issues - mount ::API::Keys mount ::API::Labels - mount ::API::Lint - mount ::API::Markdown mount ::API::MavenPackages mount ::API::Members - mount ::API::MergeRequestApprovals - mount ::API::MergeRequestDiffs mount ::API::MergeRequests - mount ::API::Metrics::Dashboard::Annotations - mount ::API::Metrics::UserStarredDashboards mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings @@ -267,63 +304,31 @@ module API mount ::API::NpmProjectPackages mount ::API::NugetGroupPackages mount ::API::NugetProjectPackages - mount ::API::PackageFiles mount ::API::Pages mount ::API::PagesDomains - mount ::API::PersonalAccessTokens::SelfInformation - mount ::API::PersonalAccessTokens - mount ::API::ProjectClusters mount ::API::ProjectContainerRepositories mount ::API::ProjectDebianDistributions mount ::API::ProjectEvents - mount ::API::ProjectExport - mount ::API::ProjectHooks - mount ::API::ProjectImport mount ::API::ProjectMilestones mount ::API::ProjectPackages - mount ::API::ProjectRepositoryStorageMoves - mount ::API::ProjectSnapshots - mount ::API::ProjectSnippets - mount ::API::ProjectStatistics - mount ::API::ProjectTemplates mount ::API::Projects - mount ::API::ProtectedBranches mount ::API::ProtectedTags mount ::API::PypiPackages - mount ::API::Release::Links - mount ::API::Releases - mount ::API::RemoteMirrors - mount ::API::Repositories - mount ::API::ResourceAccessTokens mount ::API::ResourceLabelEvents - mount ::API::ResourceMilestoneEvents mount ::API::ResourceStateEvents mount ::API::RpmProjectPackages mount ::API::RubygemPackages mount ::API::Search mount ::API::Settings mount ::API::SidekiqMetrics - mount ::API::SnippetRepositoryStorageMoves - mount ::API::Snippets - mount ::API::Statistics - mount ::API::Submodules mount ::API::Subscriptions - mount ::API::Suggestions - mount ::API::SystemHooks mount ::API::Tags mount ::API::Templates - mount ::API::Terraform::Modules::V1::Packages - mount ::API::Terraform::State - mount ::API::Terraform::StateVersion mount ::API::Todos - mount ::API::Topics - mount ::API::Unleash mount ::API::UsageData mount ::API::UsageDataNonSqlMetrics mount ::API::UsageDataQueries - mount ::API::UserCounts mount ::API::Users - mount ::API::Wikis mount ::API::Ml::Mlflow end diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb index e599abf4aaf..69f1521ef2a 100644 --- a/lib/api/appearance.rb +++ b/lib/api/appearance.rb @@ -22,6 +22,7 @@ module API desc 'Modify appearance' do success Entities::Appearance + consumes ['multipart/form-data'] end params do optional :title, type: String, desc: 'Instance title on the sign in / sign up page' diff --git a/lib/api/applications.rb b/lib/api/applications.rb index 4048215160f..6fc9408a570 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -10,17 +10,21 @@ module API resource :applications do desc 'Create a new application' do detail 'This feature was introduced in GitLab 10.5' - success Entities::ApplicationWithSecret + success code: 200, model: Entities::ApplicationWithSecret end params do - requires :name, type: String, desc: 'Application name' - requires :redirect_uri, type: String, desc: 'Application redirect URI' - requires :scopes, type: String, desc: 'Application scopes', allow_blank: false + requires :name, type: String, desc: 'Name of the application.', documentation: { example: 'MyApplication' } + requires :redirect_uri, type: String, desc: 'Redirect URI of the application.', documentation: { example: 'https://redirect.uri' } + requires :scopes, type: String, + desc: 'Scopes of the application. You can specify multiple scopes by separating\ + each scope using a space', + allow_blank: false optional :confidential, type: Boolean, default: true, - desc: 'Application will be used where the client secret is confidential' + desc: 'The application is used where the client secret can be kept confidential. Native mobile apps \ + and Single Page Apps are considered non-confidential. Defaults to true if not supplied' end post do application = Doorkeeper::Application.new(declared_params) @@ -33,14 +37,19 @@ module API end desc 'Get applications' do + detail 'List all registered applications' success Entities::Application + is_array true end get do applications = ApplicationsFinder.new.execute present applications, with: Entities::Application end - desc 'Delete an application' + desc 'Delete an application' do + detail 'Delete a specific application' + success code: 204 + end params do requires :id, type: Integer, desc: 'The ID of the application (not the application_id)' end diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 0a3f247ffd6..020ba53b9ee 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -28,6 +28,8 @@ module API desc "Gets a list of #{source_type} badges viewable by the authenticated user." do detail 'This feature was introduced in GitLab 10.6.' success Entities::Badge + is_array true + tags %w[badges] end params do use :pagination @@ -46,6 +48,7 @@ module API desc "Preview a badge from a #{source_type}." do detail 'This feature was introduced in GitLab 10.6.' success Entities::BasicBadgeDetails + tags %w[badges] end params do requires :link_url, type: String, desc: 'URL of the badge link' @@ -69,6 +72,7 @@ module API desc "Gets a badge of a #{source_type}." do detail 'This feature was introduced in GitLab 10.6.' success Entities::Badge + tags %w[badges] end params do requires :badge_id, type: Integer, desc: 'The badge ID' @@ -86,6 +90,7 @@ module API desc "Adds a badge to a #{source_type}." do detail 'This feature was introduced in GitLab 10.6.' success Entities::Badge + tags %w[badges] end params do requires :link_url, type: String, desc: 'URL of the badge link' @@ -107,6 +112,7 @@ module API desc "Updates a badge of a #{source_type}." do detail 'This feature was introduced in GitLab 10.6.' success Entities::Badge + tags %w[badges] end params do optional :link_url, type: String, desc: 'URL of the badge link' @@ -127,8 +133,9 @@ module API end end - desc 'Removes a badge from a project or group.' do + desc "Removes a badge from the #{source_type}." do detail 'This feature was introduced in GitLab 10.6.' + tags %w[badges] end params do requires :badge_id, type: Integer, desc: 'The badge ID' diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 6e3005ce676..0e0f6441da7 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -19,7 +19,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do segment ':id/boards' do diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 7e6b0214c03..845e42c2ed8 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -14,7 +14,7 @@ module API before do require_repository_enabled! - authorize! :download_code, user_project + authorize! :read_code, user_project end rescue_from Gitlab::Git::Repository::NoRepository do @@ -29,17 +29,21 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project repository branches' do success Entities::Branch + success code: 200, model: Entities::Branch + failure [{ code: 404, message: '404 Project Not Found' }] + tags %w[branches] + is_array true end params do use :pagination use :filter_params - optional :page_token, type: String, desc: 'Name of branch to start the paginaition from' + optional :page_token, type: String, desc: 'Name of branch to start the pagination from' end get ':id/repository/branches', urgency: :low do cache_action([user_project, :branches, current_user, declared_params], expires_in: 30.seconds) do @@ -65,15 +69,23 @@ module API end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do - desc 'Get a single branch' do - success Entities::Branch - end params do requires :branch, type: String, desc: 'The name of the branch' end + desc 'Check if a branch exists' do + success [{ code: 204, message: 'No Content' }] + failure [{ code: 404, message: 'Not Found' }] + tags %w[branches] + end head do user_project.repository.branch_exists?(params[:branch]) ? no_content! : not_found! end + desc 'Get a single repository branch' do + success Entities::Branch + success code: 200, model: Entities::Branch + failure [{ code: 404, message: 'Branch Not Found' }, { code: 404, message: 'Project Not Found' }] + tags %w[branches] + end get '/', urgency: :low do branch = find_branch!(params[:branch]) @@ -87,6 +99,9 @@ module API # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. desc 'Protect a single branch' do success Entities::Branch + success code: 200, model: Entities::Branch + failure [{ code: 404, message: '404 Branch Not Found' }] + tags %w[branches] end params do requires :branch, type: String, desc: 'The name of the branch', allow_blank: false @@ -126,6 +141,9 @@ module API # Note: This API will be deprecated in favor of the protected branches API. desc 'Unprotect a single branch' do success Entities::Branch + success code: 200, model: Entities::Branch + failure [{ code: 404, message: '404 Project Not Found' }, { code: 404, message: '404 Branch Not Found' }] + tags %w[branches] end params do requires :branch, type: String, desc: 'The name of the branch', allow_blank: false @@ -145,6 +163,9 @@ module API desc 'Create branch' do success Entities::Branch + success code: 201, model: Entities::Branch + failure [{ code: 400, message: 'Failed to create branch' }, { code: 400, message: 'Branch already exists' }] + tags %w[branches] end params do requires :branch, type: String, desc: 'The name of the branch', allow_blank: false @@ -166,7 +187,11 @@ module API end end - desc 'Delete a branch' + desc 'Delete a branch' do + success code: 204 + failure [{ code: 404, message: 'Branch Not Found' }] + tags %w[branches] + end params do requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end @@ -187,7 +212,11 @@ module API end end - desc 'Delete all merged branches' + desc 'Delete all merged branches' do + success code: 202, message: '202 Accepted' + failure [{ code: 404, message: '404 Project Not Found' }] + tags %w[branches] + end delete ':id/repository/merged_branches' do ::Branches::DeleteMergedService.new(user_project, current_user).async_execute diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index c54632919be..a28db321348 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -41,7 +41,15 @@ module API resource :bulk_imports do desc 'Start a new GitLab Migration' do detail 'This feature was introduced in GitLab 14.2.' - success Entities::BulkImport + success code: 200, model: Entities::BulkImport + consumes ['application/x-www-form-urlencoded'] + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' }, + { code: 503, message: 'Service unavailable' } + ] end params do requires :configuration, type: Hash, desc: 'The source GitLab instance configuration' do @@ -88,7 +96,13 @@ module API desc 'List all GitLab Migrations' do detail 'This feature was introduced in GitLab 14.1.' - success Entities::BulkImport + is_array true + success code: 200, model: Entities::BulkImport + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end params do use :pagination @@ -103,7 +117,13 @@ module API desc "List all GitLab Migrations' entities" do detail 'This feature was introduced in GitLab 14.1.' - success Entities::BulkImports::Entity + is_array true + success code: 200, model: Entities::BulkImports::Entity + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end params do use :pagination @@ -123,7 +143,12 @@ module API desc 'Get GitLab Migration details' do detail 'This feature was introduced in GitLab 14.1.' - success Entities::BulkImport + success code: 200, model: Entities::BulkImport + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end params do requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" @@ -134,7 +159,13 @@ module API desc "List GitLab Migration entities" do detail 'This feature was introduced in GitLab 14.1.' - success Entities::BulkImports::Entity + is_array true + success code: 200, model: Entities::BulkImports::Entity + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end params do requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" @@ -148,7 +179,12 @@ module API desc 'Get GitLab Migration entity details' do detail 'This feature was introduced in GitLab 14.1.' - success Entities::BulkImports::Entity + success code: 200, model: Entities::BulkImports::Entity + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end params do requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index 269f2fa7ddc..be4d82bc500 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -53,7 +53,7 @@ module API # HTTP status codes to terminate the job on GitLab Runner: # - 403 - def authenticate_job!(require_running: true, heartbeat_runner: false) + def authenticate_job!(heartbeat_runner: false) job = current_job # 404 is not returned here because we want to terminate the job if it's @@ -66,10 +66,7 @@ module API forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete? forbidden!('Job has been erased!') if job.erased? - - if require_running - job_forbidden!(job, 'Job is not running') unless job.running? - end + job_forbidden!(job, 'Job is not running') unless job.running? # Only some requests (like updating the job or patching the trace) should trigger # runner heartbeat. Operations like artifacts uploading are executed in context of @@ -87,9 +84,9 @@ module API end def authenticate_job_via_dependent_job! - forbidden! unless current_authenticated_job + authenticate! forbidden! unless current_job - forbidden! unless can?(current_authenticated_job.user, :read_build, current_job) + forbidden! unless can?(current_user, :read_build, current_job) end def current_job @@ -106,21 +103,6 @@ module API end end - # TODO: Replace this with `#current_authenticated_job from API::Helpers` - # after the feature flag `ci_authenticate_running_job_token_for_artifacts` - # is removed. - # - # For the time being, this needs to be overridden because the API - # GET api/v4/jobs/:id/artifacts - # needs to allow requests using token whose job is not running. - # - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83713#note_942368526 - def current_authenticated_job - strong_memoize(:current_authenticated_job) do - ::Ci::AuthJobFinder.new(token: job_token).execute - end - end - # The token used by runner to authenticate a request. # In most cases, the runner uses the token belonging to the requested job. # However, when requesting for job artifacts, the runner would use @@ -151,10 +133,6 @@ module API { config: attributes_for_keys(%w(gpus), params.dig('info', 'config')) } end - def request_using_running_job_token? - current_job.present? && current_authenticated_job.present? && current_job != current_authenticated_job - end - def metrics strong_memoize(:metrics) { ::Gitlab::Ci::Runner::Metrics.new } end diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb index 37c7cc73c46..352ad04c982 100644 --- a/lib/api/ci/job_artifacts.rb +++ b/lib/api/ci/job_artifacts.rb @@ -19,7 +19,7 @@ module API prepend_mod_with('API::Ci::JobArtifacts') # rubocop: disable Cop/InjectEnterpriseEditionModule params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Download the artifacts archive from a job' do @@ -38,7 +38,7 @@ module API latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) authorize_read_job_artifacts!(latest_build) - present_artifacts_file!(latest_build.artifacts_file, project: latest_build.project) + present_artifacts_file!(latest_build.artifacts_file) end desc 'Download a specific file from artifacts archive from a ref' do @@ -80,7 +80,7 @@ module API build = find_build!(params[:job_id]) authorize_read_job_artifacts!(build) - present_artifacts_file!(build.artifacts_file, project: build.project) + present_artifacts_file!(build.artifacts_file) end desc 'Download a specific file from artifacts archive' do diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index 6049993bf6f..9e41e1c0d8f 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -11,12 +11,12 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end helpers do params :optional_scope do - optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + optional :scope, type: Array[String], desc: 'The scope of builds to show', values: ::CommitStatus::AVAILABLE_STATUSES, coerce_with: ->(scope) { case scope @@ -29,12 +29,19 @@ module API else ['unknown'] end - } + }, + documentation: { example: %w[pending running] } end end desc 'Get a projects jobs' do - success Entities::Ci::Job + success code: 200, model: Entities::Ci::Job + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true end params do use :optional_scope @@ -53,10 +60,15 @@ module API # rubocop: enable CodeReuse/ActiveRecord desc 'Get a specific job of a project' do - success Entities::Ci::Job + success code: 200, model: Entities::Ci::Job + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :job_id, type: Integer, desc: 'The ID of a job' + requires :job_id, type: Integer, desc: 'The ID of a job', documentation: { example: 88 } end get ':id/jobs/:job_id', urgency: :low, feature_category: :continuous_integration do authorize_read_builds! @@ -69,9 +81,16 @@ module API # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. - desc 'Get a trace of a specific job of a project' + desc 'Get a trace of a specific job of a project' do + success code: 200, model: Entities::Ci::Job + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + end params do - requires :job_id, type: Integer, desc: 'The ID of a job' + requires :job_id, type: Integer, desc: 'The ID of a job', documentation: { example: 88 } end get ':id/jobs/:job_id/trace', urgency: :low, feature_category: :continuous_integration do authorize_read_builds! @@ -90,10 +109,15 @@ module API end desc 'Cancel a specific job of a project' do - success Entities::Ci::Job + success code: 201, model: Entities::Ci::Job + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :job_id, type: Integer, desc: 'The ID of a job' + requires :job_id, type: Integer, desc: 'The ID of a job', documentation: { example: 88 } end post ':id/jobs/:job_id/cancel', urgency: :low, feature_category: :continuous_integration do authorize_update_builds! @@ -107,10 +131,15 @@ module API end desc 'Retry a specific build of a project' do - success Entities::Ci::Job + success code: 201, model: Entities::Ci::Job + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :job_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a build', documentation: { example: 88 } end post ':id/jobs/:job_id/retry', urgency: :low, feature_category: :continuous_integration do authorize_update_builds! @@ -128,10 +157,16 @@ module API end desc 'Erase job (remove artifacts and the trace)' do - success Entities::Ci::Job + success code: 201, model: Entities::Ci::Job + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' } + ] end params do - requires :job_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a build', documentation: { example: 88 } end post ':id/jobs/:job_id/erase', urgency: :low, feature_category: :continuous_integration do authorize_update_builds! @@ -148,15 +183,21 @@ module API end desc 'Trigger an actionable job (manual, delayed, etc)' do - success Entities::Ci::JobBasic detail 'This feature was added in GitLab 8.11' + success code: 200, model: Entities::Ci::JobBasic + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :job_id, type: Integer, desc: 'The ID of a Job' + requires :job_id, type: Integer, desc: 'The ID of a Job', documentation: { example: 88 } optional :job_variables_attributes, type: Array, desc: 'User defined variables that will be included when running the job' do - requires :key, type: String, desc: 'The name of the variable' - requires :value, type: String, desc: 'The value of the variable' + requires :key, type: String, desc: 'The name of the variable', documentation: { example: 'foo' } + requires :value, type: String, desc: 'The value of the variable', documentation: { example: 'bar' } end end @@ -183,7 +224,12 @@ module API resource :job do desc 'Get current job using job token' do - success Entities::Ci::Job + success code: 200, model: Entities::Ci::Job + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] end route_setting :authentication, job_token_allowed: true get '', feature_category: :continuous_integration, urgency: :low do @@ -194,6 +240,12 @@ module API desc 'Get current agents' do detail 'Retrieves a list of agents for the given job token' + success code: 200, model: Entities::Ci::Job + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] end route_setting :authentication, job_token_allowed: true get '/allowed_agents', urgency: :low, feature_category: :kubernetes_management do @@ -210,7 +262,7 @@ module API .select { |_role, role_access_level| role_access_level <= user_access_level } .map(&:first) - environment = if persisted_environment = current_authenticated_job.persisted_environment + environment = if persisted_environment = current_authenticated_job.actual_persisted_environment { tier: persisted_environment.tier, slug: persisted_environment.slug } end @@ -244,6 +296,8 @@ module API # current_authenticated_job will be nil if user is using # a valid authentication (like PRIVATE-TOKEN) that is not CI_JOB_TOKEN not_found!('Job') unless current_authenticated_job + + ::Gitlab::ApplicationContext.push(job: current_authenticated_job) end end end diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb index 886c3509c51..afb3754f2ae 100644 --- a/lib/api/ci/pipeline_schedules.rb +++ b/lib/api/ci/pipeline_schedules.rb @@ -11,16 +11,24 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project', + documentation: { example: 18 } end resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all pipeline schedules' do - success Entities::Ci::PipelineSchedule + success code: 200, model: Entities::Ci::PipelineSchedule + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true end params do use :pagination optional :scope, type: String, values: %w[active inactive], - desc: 'The scope of pipeline schedules' + desc: 'The scope of pipeline schedules', + documentation: { example: 'active' } end # rubocop: disable CodeReuse/ActiveRecord get ':id/pipeline_schedules' do @@ -33,34 +41,51 @@ module API # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single pipeline schedule' do - success Entities::Ci::PipelineScheduleDetails + success code: 200, model: Entities::Ci::PipelineScheduleDetails + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 } end get ':id/pipeline_schedules/:pipeline_schedule_id' do present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails, user: current_user end desc 'Get all pipelines triggered from a pipeline schedule' do - success Entities::Ci::PipelineBasic + success code: 200, model: Entities::Ci::PipelineBasic + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule ID' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule ID', documentation: { example: 13 } end get ':id/pipeline_schedules/:pipeline_schedule_id/pipelines' do present paginate(pipeline_schedule.pipelines), with: Entities::Ci::PipelineBasic end desc 'Create a new pipeline schedule' do - success Entities::Ci::PipelineScheduleDetails + success code: 201, model: Entities::Ci::PipelineScheduleDetails + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :description, type: String, desc: 'The description of pipeline schedule' - requires :ref, type: String, desc: 'The branch/tag name will be triggered', allow_blank: false - requires :cron, type: String, desc: 'The cron' - optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone' - optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule' + requires :description, type: String, desc: 'The description of pipeline schedule', documentation: { example: 'Test schedule pipeline' } + requires :ref, type: String, desc: 'The branch/tag name will be triggered', allow_blank: false, documentation: { example: 'develop' } + requires :cron, type: String, desc: 'The cron', documentation: { example: '* * * * *' } + optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone', documentation: { example: 'Asia/Tokyo' } + optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule', documentation: { example: true } end post ':id/pipeline_schedules' do authorize! :create_pipeline_schedule, user_project @@ -77,15 +102,21 @@ module API end desc 'Edit a pipeline schedule' do - success Entities::Ci::PipelineScheduleDetails + success code: 200, model: Entities::Ci::PipelineScheduleDetails + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' - optional :description, type: String, desc: 'The description of pipeline schedule' - optional :ref, type: String, desc: 'The branch/tag name will be triggered' - optional :cron, type: String, desc: 'The cron' - optional :cron_timezone, type: String, desc: 'The timezone' - optional :active, type: Boolean, desc: 'The activation of pipeline schedule' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 } + optional :description, type: String, desc: 'The description of pipeline schedule', documentation: { example: 'Test schedule pipeline' } + optional :ref, type: String, desc: 'The branch/tag name will be triggered', documentation: { example: 'develop' } + optional :cron, type: String, desc: 'The cron', documentation: { example: '* * * * *' } + optional :cron_timezone, type: String, desc: 'The timezone', documentation: { example: 'Asia/Tokyo' } + optional :active, type: Boolean, desc: 'The activation of pipeline schedule', documentation: { example: true } end put ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :update_pipeline_schedule, pipeline_schedule @@ -98,10 +129,16 @@ module API end desc 'Take ownership of a pipeline schedule' do - success Entities::Ci::PipelineScheduleDetails + success code: 201, model: Entities::Ci::PipelineScheduleDetails + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 } end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do authorize! :take_ownership_pipeline_schedule, pipeline_schedule @@ -114,10 +151,16 @@ module API end desc 'Delete a pipeline schedule' do - success Entities::Ci::PipelineScheduleDetails + success code: 204 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 412, message: 'Precondition Failed' } + ] end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 } end delete ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :admin_pipeline_schedule, pipeline_schedule @@ -127,9 +170,15 @@ module API desc 'Play a scheduled pipeline immediately' do detail 'This feature was added in GitLab 12.8' + success code: 201 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 } end post ':id/pipeline_schedules/:pipeline_schedule_id/play' do authorize! :play_pipeline_schedule, pipeline_schedule @@ -145,13 +194,20 @@ module API end desc 'Create a new pipeline schedule variable' do - success Entities::Ci::Variable + success code: 201, model: Entities::Ci::Variable + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' - requires :key, type: String, desc: 'The key of the variable' - requires :value, type: String, desc: 'The value of the variable' - optional :variable_type, type: String, values: ::Ci::PipelineScheduleVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 } + requires :key, type: String, desc: 'The key of the variable', documentation: { example: 'NEW_VARIABLE' } + requires :value, type: String, desc: 'The value of the variable', documentation: { example: 'new value' } + optional :variable_type, type: String, values: ::Ci::PipelineScheduleVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var', + documentation: { default: 'env_var' } end post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do authorize! :update_pipeline_schedule, pipeline_schedule @@ -166,13 +222,20 @@ module API end desc 'Edit a pipeline schedule variable' do - success Entities::Ci::Variable + success code: 200, model: Entities::Ci::Variable + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' - requires :key, type: String, desc: 'The key of the variable' - optional :value, type: String, desc: 'The value of the variable' - optional :variable_type, type: String, values: ::Ci::PipelineScheduleVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 } + requires :key, type: String, desc: 'The key of the variable', documentation: { example: 'NEW_VARIABLE' } + optional :value, type: String, desc: 'The value of the variable', documentation: { example: 'new value' } + optional :variable_type, type: String, values: ::Ci::PipelineScheduleVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file', + documentation: { default: 'env_var' } end put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do authorize! :update_pipeline_schedule, pipeline_schedule @@ -185,11 +248,16 @@ module API end desc 'Delete a pipeline schedule variable' do - success Entities::Ci::Variable + success code: 202, model: Entities::Ci::Variable + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' - requires :key, type: String, desc: 'The key of the variable' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id', documentation: { example: 13 } + requires :key, type: String, desc: 'The key of the variable', documentation: { example: 'NEW_VARIABLE' } end delete ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do authorize! :admin_pipeline_schedule, pipeline_schedule diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 72a81330e71..c055512e54e 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -10,12 +10,17 @@ module API before { authenticate_non_get! } params do - requires :id, type: String, desc: 'The project ID' + requires :id, type: String, desc: 'The project ID or URL-encoded path', documentation: { example: 11 } end resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all Pipelines of the project' do detail 'This feature was introduced in GitLab 8.11.' - success Entities::Ci::PipelineBasic + success status: 200, model: Entities::Ci::PipelineBasic + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' } + ] + is_array true end helpers do @@ -31,27 +36,39 @@ module API else ['unknown'] end - } + }, + documentation: { example: %w[pending running] } end end params do use :pagination optional :scope, type: String, values: %w[running pending finished branches tags], - desc: 'The scope of pipelines' + desc: 'The scope of pipelines', + documentation: { example: 'pending' } optional :status, type: String, values: ::Ci::HasStatus::AVAILABLE_STATUSES, - desc: 'The status of pipelines' - optional :ref, type: String, desc: 'The ref of pipelines' - optional :sha, type: String, desc: 'The sha of pipelines' - optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' - optional :username, type: String, desc: 'The username of the user who triggered pipelines' - optional :updated_before, type: DateTime, desc: 'Return pipelines updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' - optional :updated_after, type: DateTime, desc: 'Return pipelines updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' + desc: 'The status of pipelines', + documentation: { example: 'pending' } + optional :ref, type: String, desc: 'The ref of pipelines', + documentation: { example: 'develop' } + optional :sha, type: String, desc: 'The sha of pipelines', + documentation: { example: 'a91957a858320c0e17f3a0eca7cfacbff50ea29a' } + optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations', + documentation: { example: false } + optional :username, type: String, desc: 'The username of the user who triggered pipelines', + documentation: { example: 'root' } + optional :updated_before, type: DateTime, desc: 'Return pipelines updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ', + documentation: { example: '2015-12-24T15:51:21.880Z' } + optional :updated_after, type: DateTime, desc: 'Return pipelines updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ', + documentation: { example: '2015-12-24T15:51:21.880Z' } optional :order_by, type: String, values: ::Ci::PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id', - desc: 'Order pipelines' + desc: 'Order pipelines', + documentation: { example: 'status' } optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Sort pipelines' - optional :source, type: String, values: ::Ci::Pipeline.sources.keys + desc: 'Sort pipelines', + documentation: { example: 'asc' } + optional :source, type: String, values: ::Ci::Pipeline.sources.keys, + documentation: { example: 'push' } end get ':id/pipelines', urgency: :low, feature_category: :continuous_integration do authorize! :read_pipeline, user_project @@ -63,11 +80,22 @@ module API desc 'Create a new pipeline' do detail 'This feature was introduced in GitLab 8.14' - success Entities::Ci::Pipeline + success status: 201, model: Entities::Ci::Pipeline + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :ref, type: String, desc: 'Reference' - optional :variables, Array, desc: 'Array of variables available in the pipeline' + requires :ref, type: String, desc: 'Reference', + documentation: { example: 'develop' } + optional :variables, type: Array, desc: 'Array of variables available in the pipeline' do + optional :key, type: String, desc: 'The key of the variable', documentation: { example: 'UPLOAD_TO_S3' } + optional :value, type: String, desc: 'The value of the variable', documentation: { example: 'true' } + optional :variable_type, type: String, values: ::Ci::PipelineVariable.variable_types.keys, default: 'env_var', desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' + end end post ':id/pipeline', urgency: :low, feature_category: :continuous_integration do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20711') @@ -89,12 +117,18 @@ module API end end - desc 'Gets a the latest pipeline for the project branch' do + desc 'Gets the latest pipeline for the project branch' do detail 'This feature was introduced in GitLab 12.3' - success Entities::Ci::Pipeline + success status: 200, model: Entities::Ci::Pipeline + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - optional :ref, type: String, desc: 'branch ref of pipeline' + optional :ref, type: String, desc: 'Branch ref of pipeline. Uses project default branch if not specified.', + documentation: { example: 'develop' } end get ':id/pipelines/latest', urgency: :low, feature_category: :continuous_integration do authorize! :read_pipeline, latest_pipeline @@ -104,10 +138,15 @@ module API desc 'Gets a specific pipeline for the project' do detail 'This feature was introduced in GitLab 8.11' - success Entities::Ci::Pipeline + success status: 200, model: Entities::Ci::Pipeline + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } end get ':id/pipelines/:pipeline_id', urgency: :low, feature_category: :continuous_integration do authorize! :read_pipeline, pipeline @@ -116,10 +155,16 @@ module API end desc 'Get pipeline jobs' do - success Entities::Ci::Job + success status: 200, model: Entities::Ci::Job + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } optional :include_retried, type: Boolean, default: false, desc: 'Includes retried jobs' use :optional_scope use :pagination @@ -140,10 +185,16 @@ module API end desc 'Get pipeline bridge jobs' do - success Entities::Ci::Bridge + success status: 200, model: Entities::Ci::Bridge + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } use :optional_scope use :pagination end @@ -163,10 +214,16 @@ module API desc 'Gets the variables for a given pipeline' do detail 'This feature was introduced in GitLab 11.11' - success Entities::Ci::Variable + success status: 200, model: Entities::Ci::Variable + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } end get ':id/pipelines/:pipeline_id/variables', feature_category: :pipeline_authoring, urgency: :low do authorize! :read_pipeline_variable, pipeline @@ -176,10 +233,15 @@ module API desc 'Gets the test report for a given pipeline' do detail 'This feature was introduced in GitLab 13.0.' - success TestReportEntity + success status: 200, model: TestReportEntity + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } end get ':id/pipelines/:pipeline_id/test_report', feature_category: :code_testing, urgency: :low do authorize! :read_build, pipeline @@ -189,10 +251,15 @@ module API desc 'Gets the test report summary for a given pipeline' do detail 'This feature was introduced in GitLab 14.2' - success TestReportSummaryEntity + success status: 200, model: TestReportSummaryEntity + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } end get ':id/pipelines/:pipeline_id/test_report_summary', feature_category: :code_testing do authorize! :read_build, pipeline @@ -205,7 +272,7 @@ module API http_codes [[204, 'Pipeline was deleted'], [403, 'Forbidden']] end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } end delete ':id/pipelines/:pipeline_id', urgency: :low, feature_category: :continuous_integration do authorize! :destroy_pipeline, pipeline @@ -219,10 +286,15 @@ module API desc 'Retry builds in the pipeline' do detail 'This feature was introduced in GitLab 8.11.' - success Entities::Ci::Pipeline + success status: 201, model: Entities::Ci::Pipeline + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } end post ':id/pipelines/:pipeline_id/retry', urgency: :low, feature_category: :continuous_integration do authorize! :update_pipeline, pipeline @@ -238,10 +310,15 @@ module API desc 'Cancel all builds in the pipeline' do detail 'This feature was introduced in GitLab 8.11.' - success Entities::Ci::Pipeline + success status: 200, model: Entities::Ci::Pipeline + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } end post ':id/pipelines/:pipeline_id/cancel', urgency: :low, feature_category: :continuous_integration do authorize! :update_pipeline, pipeline diff --git a/lib/api/ci/resource_groups.rb b/lib/api/ci/resource_groups.rb index ea6d3cc8fd4..79a9fe58a7d 100644 --- a/lib/api/ci/resource_groups.rb +++ b/lib/api/ci/resource_groups.rb @@ -5,17 +5,30 @@ module API class ResourceGroups < ::API::Base include PaginationParams + ci_resource_groups_tags = %w[ci_resource_groups] + + RESOURCE_GROUP_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS + .merge(key: API::NO_SLASH_URL_PART_REGEX) + before { authenticate! } feature_category :continuous_delivery urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, + types: [String, Integer], + desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get all resource groups for this project' do + desc 'Get all resource groups for a project' do success Entities::Ci::ResourceGroup + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags ci_resource_groups_tags end params do use :pagination @@ -26,27 +39,38 @@ module API present paginate(user_project.resource_groups), with: Entities::Ci::ResourceGroup end - desc 'Get a single resource group' do + desc 'Get a specific resource group' do success Entities::Ci::ResourceGroup + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags ci_resource_groups_tags end params do requires :key, type: String, desc: 'The key of the resource group' end - get ':id/resource_groups/:key' do + get ':id/resource_groups/:key', requirements: RESOURCE_GROUP_ENDPOINT_REQUIREMENTS do authorize! :read_resource_group, resource_group present resource_group, with: Entities::Ci::ResourceGroup end - desc 'List upcoming jobs of a resource group' do + desc 'List upcoming jobs for a specific resource group' do success Entities::Ci::JobBasic + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags ci_resource_groups_tags end params do requires :key, type: String, desc: 'The key of the resource group' use :pagination end - get ':id/resource_groups/:key/upcoming_jobs' do + get ':id/resource_groups/:key/upcoming_jobs', requirements: RESOURCE_GROUP_ENDPOINT_REQUIREMENTS do authorize! :read_resource_group, resource_group authorize! :read_build, user_project @@ -57,15 +81,25 @@ module API present paginate(upcoming_processables), with: Entities::Ci::JobBasic end - desc 'Edit a resource group' do + desc 'Edit an existing resource group' do + detail "Updates an existing resource group's properties." success Entities::Ci::ResourceGroup + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags ci_resource_groups_tags end params do requires :key, type: String, desc: 'The key of the resource group' - optional :process_mode, type: String, desc: 'The process mode', - values: ::Ci::ResourceGroup.process_modes.keys + + optional :process_mode, + type: String, + desc: 'The process mode of the resource group', + values: ::Ci::ResourceGroup.process_modes.keys end - put ':id/resource_groups/:key' do + put ':id/resource_groups/:key', requirements: RESOURCE_GROUP_ENDPOINT_REQUIREMENTS do authorize! :update_resource_group, resource_group if resource_group.update(declared_params(include_missing: false)) diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 2d2dcc544f9..c7d1887638a 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -8,25 +8,38 @@ module API content_type :txt, 'text/plain' resource :runners do - desc 'Registers a new Runner' do + desc 'Register a new runner' do + detail "Register a new runner for the instance" success Entities::Ci::RunnerRegistrationDetails - http_codes [[201, 'Runner was created'], [403, 'Forbidden']] + failure [[400, 'Bad Request'], [403, 'Forbidden']] end params do requires :token, type: String, desc: 'Registration token' optional :description, type: String, desc: %q(Runner's description) - optional :maintainer_note, type: String, desc: %q(Deprecated: Use :maintenance_note instead. Runner's maintenance notes) - optional :maintenance_note, type: String, desc: %q(Runner's maintenance notes) - optional :info, type: Hash, desc: %q(Runner's metadata) - optional :active, type: Boolean, desc: 'Deprecated: Use `:paused` instead. Should runner be active' - optional :paused, type: Boolean, desc: 'Whether the runner should ignore new jobs' - optional :locked, type: Boolean, desc: 'Whether the runner should be locked for current project' + optional :maintainer_note, type: String, desc: %q(Deprecated: see `maintenance_note`) + optional :maintenance_note, type: String, + desc: %q(Free-form maintenance notes for the runner (1024 characters)) + optional :info, type: Hash, desc: %q(Runner's metadata) do + optional :name, type: String, desc: %q(Runner's name) + optional :version, type: String, desc: %q(Runner's version) + optional :revision, type: String, desc: %q(Runner's revision) + optional :platform, type: String, desc: %q(Runner's platform) + optional :architecture, type: String, desc: %q(Runner's architecture) + end + optional :active, type: Boolean, + desc: 'Deprecated: Use `paused` instead. Specifies whether the runner is allowed ' \ + 'to receive new jobs' + optional :paused, type: Boolean, desc: 'Specifies whether the runner should ignore new jobs' + optional :locked, type: Boolean, desc: 'Specifies whether the runner should be locked for the current project' optional :access_level, type: String, values: ::Ci::Runner.access_levels.keys, - desc: 'The access_level of the runner; `not_protected` or `ref_protected`' - optional :run_untagged, type: Boolean, desc: 'Whether the runner should handle untagged jobs' - optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: %q(List of Runner's tags) - optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this runner handles the job' - mutually_exclusive :maintainer_note, :maintainer_note + desc: 'The access level of the runner' + optional :run_untagged, type: Boolean, desc: 'Specifies whether the runner should handle untagged jobs' + optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + desc: %q(A list of runner tags) + optional :maximum_timeout, type: Integer, + desc: 'Maximum timeout that limits the amount of time (in seconds) ' \ + 'that runners can run jobs' + mutually_exclusive :maintainer_note, :maintenance_note mutually_exclusive :active, :paused end post '/', urgency: :low, feature_category: :runner do @@ -49,11 +62,12 @@ module API end end - desc 'Deletes a registered Runner' do - http_codes [[204, 'Runner was deleted'], [403, 'Forbidden']] + desc 'Delete a registered runner' do + summary "Delete a runner by authentication token" + failure [[403, 'Forbidden']] end params do - requires :token, type: String, desc: %q(Runner's authentication token) + requires :token, type: String, desc: %q(The runner's authentication token) end delete '/', urgency: :low, feature_category: :runner do authenticate_runner! @@ -61,11 +75,12 @@ module API destroy_conditionally!(current_runner) { ::Ci::Runners::UnregisterRunnerService.new(current_runner, params[:token]).execute } end - desc 'Validates authentication credentials' do + desc 'Validate authentication credentials' do + summary "Verify authentication for a registered runner" http_codes [[200, 'Credentials are valid'], [403, 'Forbidden']] end params do - requires :token, type: String, desc: %q(Runner's authentication token) + requires :token, type: String, desc: %q(The runner's authentication token) end post '/verify', urgency: :low, feature_category: :runner do authenticate_runner! @@ -75,6 +90,7 @@ module API desc 'Reset runner authentication token with current token' do success Entities::Ci::ResetTokenResult + failure [[403, 'Forbidden']] end params do requires :token, type: String, desc: 'The current authentication token of the runner' @@ -94,7 +110,8 @@ module API success Entities::Ci::JobRequest::Response http_codes [[201, 'Job was scheduled'], [204, 'No job for Runner'], - [403, 'Forbidden']] + [403, 'Forbidden'], + [409, 'Conflict']] end params do requires :token, type: String, desc: %q(Runner's authentication token) @@ -168,14 +185,14 @@ module API end end - desc 'Updates a job' do + desc 'Update a job' do http_codes [[200, 'Job was updated'], [202, 'Update accepted'], [400, 'Unknown parameters'], [403, 'Forbidden']] end params do - requires :token, type: String, desc: %q(Runners's authentication token) + requires :token, type: String, desc: %q(Runner's authentication token) requires :id, type: Integer, desc: %q(Job's ID) optional :state, type: String, desc: %q(Job's status: success, failed) optional :checksum, type: String, desc: %q(Job's trace CRC32 checksum) @@ -203,7 +220,7 @@ module API end end - desc 'Appends a patch to the job trace' do + desc 'Append a patch to the job trace' do http_codes [[202, 'Trace was patched'], [400, 'Missing Content-Range header'], [403, 'Forbidden'], @@ -285,14 +302,14 @@ module API end params do requires :id, type: Integer, desc: %q(Job's ID) - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact file to store (generated by Multipart middleware)) + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact file to store (generated by Multipart middleware)), documentation: { type: 'file' } optional :token, type: String, desc: %q(Job's authentication token) optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) optional :artifact_type, type: String, desc: %q(The type of artifact), default: 'archive', values: ::Ci::JobArtifact.file_types.keys optional :artifact_format, type: String, desc: %q(The format of artifact), default: 'zip', values: ::Ci::JobArtifact.file_formats.keys - optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware)) + optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware)), documentation: { type: 'file' } end post '/:id/artifacts', feature_category: :build_artifacts, urgency: :low do not_allowed! unless Gitlab.config.artifacts.enabled @@ -317,6 +334,7 @@ module API desc 'Download the artifacts file for job' do http_codes [[200, 'Upload allowed'], + [401, 'Unauthorized'], [403, 'Forbidden'], [404, 'Artifact not found']] end @@ -325,14 +343,11 @@ module API optional :token, type: String, desc: %q(Job's authentication token) optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts) end + route_setting :authentication, job_token_allowed: true get '/:id/artifacts', feature_category: :build_artifacts do - if request_using_running_job_token? - authenticate_job_via_dependent_job! - else - authenticate_job!(require_running: false) - end + authenticate_job_via_dependent_job! - present_artifacts_file!(current_job.artifacts_file, project: current_job.project, supports_direct_download: params[:direct_download]) + present_artifacts_file!(current_job.artifacts_file, supports_direct_download: params[:direct_download]) end end end diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 4b578f8b7e5..988c3f4f566 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -10,20 +10,98 @@ module API feature_category :runner urgency :low + helpers do + params :deprecated_filter_params do + optional :scope, type: String, values: ::Ci::Runner::AVAILABLE_SCOPES, + desc: 'Deprecated: Use `type` or `status` instead. The scope of specific runners to return' + end + + params :filter_params do + optional :type, type: String, values: ::Ci::Runner::AVAILABLE_TYPES, desc: 'The type of runners to return' + optional :paused, type: Boolean, + desc: 'Whether to include only runners that are accepting or ignoring new jobs' + optional :status, type: String, values: ::Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of runners to return' + optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + desc: 'A list of runner tags', documentation: { example: "['macos', 'shell']" } + use :pagination + end + + def filter_runners(runners, scope, allowed_scopes: ::Ci::Runner::AVAILABLE_SCOPES) + return runners unless scope.present? + + unless allowed_scopes.include?(scope) + render_api_error!('Scope contains invalid value', 400) + end + + # Support deprecated scopes + if runners.respond_to?("deprecated_#{scope}") + scope = "deprecated_#{scope}" + end + + runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend + end + + def apply_filter(runners, params) + runners = filter_runners(runners, params[:type], allowed_scopes: ::Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: ::Ci::Runner::AVAILABLE_STATUSES) + runners = filter_runners(runners, params[:paused] ? 'paused' : 'active', allowed_scopes: %w[paused active]) if params.include?(:paused) + runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] + + runners + end + + def get_runner(id) + runner = ::Ci::Runner.find(id) + not_found!('Runner') unless runner + runner + end + + def authenticate_show_runner!(runner) + return if runner.instance_type? || current_user.admin? + + forbidden!("No access granted") unless can?(current_user, :read_runner, runner) + end + + def authenticate_update_runner!(runner) + return if current_user.admin? + + forbidden!("No access granted") unless can?(current_user, :update_runner, runner) + end + + def authenticate_delete_runner!(runner) + return if current_user.admin? + + forbidden!("Runner associated with more than one project") if runner.runner_projects.count > 1 + forbidden!("No access granted") unless can?(current_user, :delete_runner, runner) + end + + def authenticate_enable_runner!(runner) + forbidden!("Runner is a group runner") if runner.group_type? + + return if current_user.admin? + + forbidden!("Runner is locked") if runner.locked? + forbidden!("No access granted") unless can?(current_user, :assign_runner, runner) + end + + def authenticate_list_runners_jobs!(runner) + return if current_user.admin? + + forbidden!("No access granted") unless can?(current_user, :read_builds, runner) + end + end + resource :runners do desc 'Get runners available for user' do + summary 'List owned runners' success Entities::Ci::Runner + failure [[400, 'Scope contains invalid value'], [401, 'Unauthorized']] + tags %w[runners] end params do - optional :scope, type: String, values: ::Ci::Runner::AVAILABLE_STATUSES, - desc: 'The scope of specific runners to show' - optional :type, type: String, values: ::Ci::Runner::AVAILABLE_TYPES, - desc: 'The type of the runners to show' - optional :paused, type: Boolean, desc: 'Whether to include only runners that are accepting or ignoring new jobs' - optional :status, type: String, values: ::Ci::Runner::AVAILABLE_STATUSES, - desc: 'The status of the runners to show' - optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The tags of the runners to show' - use :pagination + use :deprecated_filter_params + use :filter_params end get do runners = current_user.ci_owned_runners @@ -34,18 +112,16 @@ module API end desc 'Get all runners - shared and specific' do + summary 'List all runners' + detail 'Get a list of all runners in the GitLab instance (specific and shared). ' \ + 'Access is restricted to users with administrator access.' success Entities::Ci::Runner + failure [[400, 'Scope contains invalid value'], [401, 'Unauthorized']] + tags %w[runners] end params do - optional :scope, type: String, values: ::Ci::Runner::AVAILABLE_SCOPES, - desc: 'The scope of specific runners to show' - optional :type, type: String, values: ::Ci::Runner::AVAILABLE_TYPES, - desc: 'The type of the runners to show' - optional :paused, type: Boolean, desc: 'Whether to include only runners that are accepting or ignoring new jobs' - optional :status, type: String, values: ::Ci::Runner::AVAILABLE_STATUSES, - desc: 'The status of the runners to show' - optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The tags of the runners to show' - use :pagination + use :deprecated_filter_params + use :filter_params end get 'all' do authenticated_as_admin! @@ -58,10 +134,13 @@ module API end desc "Get runner's details" do + detail 'At least the Maintainer role is required to get runner details at the project and group level. ' \ + 'Instance-level runner details via this endpoint are available to all signed in users.' success Entities::Ci::RunnerDetails + failure [[401, 'Unauthorized'], [403, 'No access granted'], [404, 'Runner not found']] end params do - requires :id, type: Integer, desc: 'The ID of the runner' + requires :id, type: Integer, desc: 'The ID of a runner' end get ':id' do runner = get_runner(params[:id]) @@ -71,19 +150,24 @@ module API end desc "Update runner's details" do + summary "Update details of a runner" success Entities::Ci::RunnerDetails + failure [[400, 'Bad Request'], [401, 'Unauthorized'], [403, 'No access granted'], [404, 'Runner not found']] end params do - requires :id, type: Integer, desc: 'The ID of the runner' + requires :id, type: Integer, desc: 'The ID of a runner' optional :description, type: String, desc: 'The description of the runner' - optional :active, type: Boolean, desc: 'Deprecated: Use `:paused` instead. Flag indicating whether the runner is allowed to receive jobs' - optional :paused, type: Boolean, desc: 'Flag indicating whether the runner should ignore new jobs' - optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The list of tags for a runner' - optional :run_untagged, type: Boolean, desc: 'Flag indicating whether the runner can execute untagged jobs' - optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' + optional :active, type: Boolean, desc: 'Deprecated: Use `paused` instead. Flag indicating whether the runner is allowed to receive jobs' + optional :paused, type: Boolean, desc: 'Specifies whether the runner should ignore new jobs' + optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + desc: 'The list of tags for a runner', documentation: { example: "['macos', 'shell']" } + optional :run_untagged, type: Boolean, desc: 'Specifies whether the runner can execute untagged jobs' + optional :locked, type: Boolean, desc: 'Specifies whether the runner is locked' optional :access_level, type: String, values: ::Ci::Runner.access_levels.keys, - desc: 'The access_level of the runner' - optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' + desc: 'The access level of the runner' + optional :maximum_timeout, type: Integer, + desc: 'Maximum timeout that limits the amount of time (in seconds) ' \ + 'that runners can run jobs' at_least_one_of :description, :active, :paused, :tag_list, :run_untagged, :locked, :access_level, :maximum_timeout mutually_exclusive :active, :paused end @@ -101,10 +185,15 @@ module API end desc 'Remove a runner' do + summary 'Delete a runner' success Entities::Ci::Runner + failure [[401, 'Unauthorized'], [403, 'No access granted'], + [403, 'Runner associated with more than one project'], [404, 'Runner not found'], + [412, 'Precondition Failed']] + tags %w[runners] end params do - requires :id, type: Integer, desc: 'The ID of the runner' + requires :id, type: Integer, desc: 'The ID of a runner' end delete ':id' do runner = get_runner(params[:id]) @@ -115,13 +204,19 @@ module API end desc 'List jobs running on a runner' do + summary "List runner's jobs" + detail 'List jobs that are being processed or were processed by the specified runner. ' \ + 'The list of jobs is limited to projects where the user has at least the Reporter role.' success Entities::Ci::JobBasicWithProject + failure [[401, 'Unauthorized'], [403, 'No access granted'], [404, 'Runner not found']] + tags %w[runners jobs] end params do - requires :id, type: Integer, desc: 'The ID of the runner' + requires :id, type: Integer, desc: 'The ID of a runner' optional :status, type: String, desc: 'Status of the job', values: ::Ci::Build::AVAILABLE_STATUSES - optional :order_by, type: String, desc: 'Order by `id` or not', values: ::Ci::RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS - optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort by asc (ascending) or desc (descending)' + optional :order_by, type: String, desc: 'Order by `id`', values: ::Ci::RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS + optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort by `asc` or `desc` order. ' \ + 'Specify `order_by` as well, including for `id`' use :pagination end get ':id/jobs' do @@ -143,7 +238,10 @@ module API end desc 'Reset runner authentication token' do + summary "Reset runner's authentication token" success Entities::Ci::ResetTokenResult + failure [[403, 'No access granted'], [404, 'Runner not found']] + tags %w[runners] end params do requires :id, type: Integer, desc: 'The ID of the runner' @@ -158,24 +256,24 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, + types: [String, Integer], + desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authorize_admin_project } desc 'Get runners available for project' do + summary "List project's runners" + detail 'List all runners available in the project, including from ancestor groups ' \ + 'and any allowed shared runners.' success Entities::Ci::Runner + failure [[400, 'Scope contains invalid value'], [403, 'No access granted']] + tags %w[runners projects] end params do - optional :scope, type: String, values: ::Ci::Runner::AVAILABLE_SCOPES, - desc: 'The scope of specific runners to show' - optional :type, type: String, values: ::Ci::Runner::AVAILABLE_TYPES, - desc: 'The type of the runners to show' - optional :paused, type: Boolean, desc: 'Whether to include only runners that are accepting or ignoring new jobs' - optional :status, type: String, values: ::Ci::Runner::AVAILABLE_STATUSES, - desc: 'The status of the runners to show' - optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The tags of the runners to show' - use :pagination + use :deprecated_filter_params + use :filter_params end get ':id/runners' do runners = ::Ci::Runner.owned_or_instance_wide(user_project.id) @@ -187,11 +285,16 @@ module API present paginate(runners), with: Entities::Ci::Runner end - desc 'Enable a runner for a project' do + desc 'Enable a runner in project' do + detail "Enable an available specific runner in the project." success Entities::Ci::Runner + failure [[400, 'Bad Request'], + [403, 'No access granted'], [403, 'Runner is a group runner'], [403, 'Runner is locked'], + [404, 'Runner not found']] + tags %w[runners projects] end params do - requires :runner_id, type: Integer, desc: 'The ID of the runner' + requires :runner_id, type: Integer, desc: 'The ID of a runner' end post ':id/runners' do runner = get_runner(params[:runner_id]) @@ -205,10 +308,17 @@ module API end desc "Disable project's runner" do + summary "Disable a specific runner from the project" + detail "It works only if the project isn't the only project associated with the specified runner. " \ + "If so, an error is returned. Use the call to delete a runner instead." success Entities::Ci::Runner + failure [[400, 'Bad Request'], + [403, 'Only one project associated with the runner. Please remove the runner instead'], + [404, 'Runner not found'], [412, 'Precondition Failed']] + tags %w[runners projects] end params do - requires :runner_id, type: Integer, desc: 'The ID of the runner' + requires :runner_id, type: Integer, desc: 'The ID of a runner' end # rubocop: disable CodeReuse/ActiveRecord delete ':id/runners/:runner_id' do @@ -230,16 +340,15 @@ module API before { authorize_admin_group } desc 'Get runners available for group' do + summary "List group's runners" + detail 'List all runners available in the group as well as its ancestor groups, ' \ + 'including any allowed shared runners.' success Entities::Ci::Runner + failure [[400, 'Scope contains invalid value'], [403, 'Forbidden']] + tags %w[runners groups] end params do - optional :type, type: String, values: ::Ci::Runner::AVAILABLE_TYPES, - desc: 'The type of the runners to show' - optional :paused, type: Boolean, desc: 'Whether to include only runners that are accepting or ignoring new jobs' - optional :status, type: String, values: ::Ci::Runner::AVAILABLE_STATUSES, - desc: 'The status of the runners to show' - optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The tags of the runners to show' - use :pagination + use :filter_params end get ':id/runners' do runners = ::Ci::Runner.group_or_instance_wide(user_group) @@ -252,8 +361,11 @@ module API resource :runners do before { authenticate_non_get! } - desc 'Resets runner registration token' do + desc 'Reset runner registration token' do + summary "Reset instance's runner registration token" success Entities::Ci::ResetTokenResult + failure [[403, 'Forbidden']] + tags %w[runners groups] end post 'reset_registration_token' do authorize! :update_runners_registration_token, ApplicationSetting.current @@ -269,8 +381,11 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authenticate_non_get! } - desc 'Resets runner registration token' do + desc 'Reset runner registration token' do + summary "Reset the runner registration token for a project" success Entities::Ci::ResetTokenResult + failure [[401, 'Unauthorized'], [403, 'Forbidden'], [404, 'Project Not Found']] + tags %w[runners projects] end post ':id/runners/reset_registration_token' do project = find_project! user_project.id @@ -287,8 +402,11 @@ module API resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authenticate_non_get! } - desc 'Resets runner registration token' do + desc 'Reset runner registration token' do + summary "Reset the runner registration token for a group" success Entities::Ci::ResetTokenResult + failure [[401, 'Unauthorized'], [403, 'Forbidden'], [404, 'Group Not Found']] + tags %w[runners groups] end post ':id/runners/reset_registration_token' do group = find_group! user_group.id @@ -298,72 +416,6 @@ module API present group.runners_token_with_expiration, with: Entities::Ci::ResetTokenResult end end - - helpers do - def filter_runners(runners, scope, allowed_scopes: ::Ci::Runner::AVAILABLE_SCOPES) - return runners unless scope.present? - - unless allowed_scopes.include?(scope) - render_api_error!('Scope contains invalid value', 400) - end - - # Support deprecated scopes - if runners.respond_to?("deprecated_#{scope}") - scope = "deprecated_#{scope}" - end - - runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend - end - - def apply_filter(runners, params) - runners = filter_runners(runners, params[:type], allowed_scopes: ::Ci::Runner::AVAILABLE_TYPES) - runners = filter_runners(runners, params[:status], allowed_scopes: ::Ci::Runner::AVAILABLE_STATUSES) - runners = filter_runners(runners, params[:paused] ? 'paused' : 'active', allowed_scopes: %w[paused active]) if params.include?(:paused) - runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] - - runners - end - - def get_runner(id) - runner = ::Ci::Runner.find(id) - not_found!('Runner') unless runner - runner - end - - def authenticate_show_runner!(runner) - return if runner.instance_type? || current_user.admin? - - forbidden!("No access granted") unless can?(current_user, :read_runner, runner) - end - - def authenticate_update_runner!(runner) - return if current_user.admin? - - forbidden!("No access granted") unless can?(current_user, :update_runner, runner) - end - - def authenticate_delete_runner!(runner) - return if current_user.admin? - - forbidden!("Runner associated with more than one project") if runner.runner_projects.count > 1 - forbidden!("No access granted") unless can?(current_user, :delete_runner, runner) - end - - def authenticate_enable_runner!(runner) - forbidden!("Runner is a group runner") if runner.group_type? - - return if current_user.admin? - - forbidden!("Runner is locked") if runner.locked? - forbidden!("No access granted") unless can?(current_user, :assign_runner, runner) - end - - def authenticate_list_runners_jobs!(runner) - return if current_user.admin? - - forbidden!("No access granted") unless can?(current_user, :read_builds, runner) - end - end end end end diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index 511b6e06cd3..dd628a3413f 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -16,7 +16,7 @@ module API default_format :json params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -61,7 +61,7 @@ module API desc 'Upload a Secure File' params do requires :name, type: String, desc: 'The name of the file' - requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The secure file to be uploaded' + requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The secure file to be uploaded', documentation: { type: 'file' } end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true post ':id/secure_files' do diff --git a/lib/api/ci/triggers.rb b/lib/api/ci/triggers.rb index c49f1c9e9e1..c202d188e43 100644 --- a/lib/api/ci/triggers.rb +++ b/lib/api/ci/triggers.rb @@ -11,16 +11,26 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project', + documentation: { example: 18 } end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Trigger a GitLab project pipeline' do - success Entities::Ci::Pipeline + success code: 201, model: Entities::Ci::Pipeline + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false - requires :token, type: String, desc: 'The unique token of trigger or job token' - optional :variables, type: Hash, desc: 'The list of variables to be injected into build' + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false, + documentation: { example: 'develop' } + requires :token, type: String, desc: 'The unique token of trigger or job token', + documentation: { example: '6d056f63e50fe6f8c5f8f4aa10edb7' } + optional :variables, type: Hash, desc: 'The list of variables to be injected into build', + documentation: { example: { VAR1: "value1", VAR2: "value2" } } end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758') @@ -47,7 +57,13 @@ module API end desc 'Get triggers list' do - success Entities::Trigger + success code: 200, model: Entities::Trigger + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true end params do use :pagination @@ -64,10 +80,15 @@ module API # rubocop: enable CodeReuse/ActiveRecord desc 'Get specific trigger of a project' do - success Entities::Trigger + success code: 200, model: Entities::Trigger + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' + requires :trigger_id, type: Integer, desc: 'The trigger ID', documentation: { example: 10 } end get ':id/triggers/:trigger_id' do authenticate! @@ -80,10 +101,17 @@ module API end desc 'Create a trigger' do - success Entities::Trigger + success code: 201, model: Entities::Trigger + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :description, type: String, desc: 'The trigger description' + requires :description, type: String, desc: 'The trigger description', + documentation: { example: 'my trigger description' } end post ':id/triggers' do authenticate! @@ -100,7 +128,13 @@ module API end desc 'Update a trigger' do - success Entities::Trigger + success code: 200, model: Entities::Trigger + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do requires :trigger_id, type: Integer, desc: 'The trigger ID' @@ -123,10 +157,16 @@ module API end desc 'Delete a trigger' do - success Entities::Trigger + success code: 204 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 412, message: 'Precondition Failed' } + ] end params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' + requires :trigger_id, type: Integer, desc: 'The trigger ID', documentation: { example: 10 } end delete ':id/triggers/:trigger_id' do authenticate! diff --git a/lib/api/ci/variables.rb b/lib/api/ci/variables.rb index c9e1d115d03..5a6b5987228 100644 --- a/lib/api/ci/variables.rb +++ b/lib/api/ci/variables.rb @@ -13,12 +13,13 @@ module API helpers ::API::Helpers::VariablesHelpers params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID of a project or URL-encoded NAMESPACE/PROJECT_NAME of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get project variables' do success Entities::Ci::Variable + tags %w[ci_variables] end params do use :pagination @@ -28,13 +29,15 @@ module API present paginate(variables), with: Entities::Ci::Variable end - desc 'Get a specific variable from a project' do + desc 'Get the details of a single variable from a project' do success Entities::Ci::Variable + failure [{ code: 404, message: 'Variable Not Found' }] + tags %w[ci_variables] end params do - requires :key, type: String, desc: 'The key of the variable' + requires :key, type: String, desc: 'The key of a variable' optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' do - optional :environment_scope, type: String, desc: 'The environment scope of the variable' + optional :environment_scope, type: String, desc: 'The environment scope of a variable' end end # rubocop: disable CodeReuse/ActiveRecord @@ -48,13 +51,17 @@ module API desc 'Create a new variable in a project' do success Entities::Ci::Variable + failure [{ code: 400, message: '400 Bad Request' }] + tags %w[ci_variables] end + route_setting :log_safety, { safe: %w[key], unsafe: %w[value] } params do - requires :key, type: String, desc: 'The key of the variable' - requires :value, type: String, desc: 'The value of the variable' + requires :key, type: String, desc: 'The key of a variable' + requires :value, type: String, desc: 'The value of a variable' optional :protected, type: Boolean, desc: 'Whether the variable is protected' optional :masked, type: Boolean, desc: 'Whether the variable is masked' - optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' + optional :raw, type: Boolean, desc: 'Whether the variable will be expanded' + optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of the variable. Default: env_var' optional :environment_scope, type: String, desc: 'The environment_scope of the variable' end post ':id/variables' do @@ -73,16 +80,20 @@ module API desc 'Update an existing variable from a project' do success Entities::Ci::Variable + failure [{ code: 404, message: 'Variable Not Found' }] + tags %w[ci_variables] end + route_setting :log_safety, { safe: %w[key], unsafe: %w[value] } params do - optional :key, type: String, desc: 'The key of the variable' - optional :value, type: String, desc: 'The value of the variable' + optional :key, type: String, desc: 'The key of a variable' + optional :value, type: String, desc: 'The value of a variable' optional :protected, type: Boolean, desc: 'Whether the variable is protected' optional :masked, type: Boolean, desc: 'Whether the variable is masked' - optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' - optional :environment_scope, type: String, desc: 'The environment_scope of the variable' + optional :environment_scope, type: String, desc: 'The environment_scope of a variable' + optional :raw, type: Boolean, desc: 'Whether the variable will be expanded' + optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of the variable. Default: env_var' optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' do - optional :environment_scope, type: String, desc: 'The environment scope of the variable' + optional :environment_scope, type: String, desc: 'The environment scope of a variable' end end # rubocop: disable CodeReuse/ActiveRecord @@ -106,9 +117,11 @@ module API desc 'Delete an existing variable from a project' do success Entities::Ci::Variable + failure [{ code: 404, message: 'Variable Not Found' }] + tags %w[ci_variables] end params do - requires :key, type: String, desc: 'The key of the variable' + requires :key, type: String, desc: 'The key of a variable' optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' do optional :environment_scope, type: String, desc: 'The environment scope of the variable' end diff --git a/lib/api/clusters/agent_tokens.rb b/lib/api/clusters/agent_tokens.rb index 1f9c8700d7a..f65ae465b3d 100644 --- a/lib/api/clusters/agent_tokens.rb +++ b/lib/api/clusters/agent_tokens.rb @@ -10,7 +10,7 @@ module API feature_category :kubernetes_management params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do @@ -18,22 +18,24 @@ module API end resource ':id/cluster_agents/:agent_id' do resource :tokens do - desc 'List agent tokens' do - detail 'This feature was introduced in GitLab 15.0.' + desc 'List tokens for an agent' do + detail 'This feature was introduced in GitLab 15.0. Returns a list of tokens for an agent.' success Entities::Clusters::AgentTokenBasic + tags %w[cluster_agents] end params do use :pagination end get do - agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) + agent_tokens = ::Clusters::AgentTokensFinder.new(user_project, current_user, params[:agent_id]).execute - present paginate(agent.agent_tokens), with: Entities::Clusters::AgentTokenBasic + present paginate(agent_tokens), with: Entities::Clusters::AgentTokenBasic end desc 'Get a single agent token' do - detail 'This feature was introduced in GitLab 15.0.' + detail 'This feature was introduced in GitLab 15.0. Gets a single agent token.' success Entities::Clusters::AgentToken + tags %w[cluster_agents] end params do requires :token_id, type: Integer, desc: 'The ID of the agent token' @@ -47,8 +49,9 @@ module API end desc 'Create an agent token' do - detail 'This feature was introduced in GitLab 15.0.' + detail 'This feature was introduced in GitLab 15.0. Creates a new token for an agent.' success Entities::Clusters::AgentTokenWithToken + tags %w[cluster_agents] end params do requires :name, type: String, desc: 'The name for the token' @@ -71,7 +74,8 @@ module API end desc 'Revoke an agent token' do - detail 'This feature was introduced in GitLab 15.0.' + detail 'This feature was introduced in GitLab 15.0. Revokes an agent token.' + tags %w[cluster_agents] end params do requires :token_id, type: Integer, desc: 'The ID of the agent token' diff --git a/lib/api/clusters/agents.rb b/lib/api/clusters/agents.rb index 2affd9680b6..62d4fb009c6 100644 --- a/lib/api/clusters/agents.rb +++ b/lib/api/clusters/agents.rb @@ -11,12 +11,13 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'List agents' do - detail 'This feature was introduced in GitLab 14.10.' + desc 'List the agents for a project' do + detail 'This feature was introduced in GitLab 14.10. Returns the list of agents registered for the project.' success Entities::Clusters::Agent + tags %w[cluster_agents] end params do use :pagination @@ -29,9 +30,10 @@ module API present paginate(agents), with: Entities::Clusters::Agent end - desc 'Get single agent' do - detail 'This feature was introduced in GitLab 14.10.' + desc 'Get details about an agent' do + detail 'This feature was introduced in GitLab 14.10. Gets a single agent details.' success Entities::Clusters::Agent + tags %w[cluster_agents] end params do requires :agent_id, type: Integer, desc: 'The ID of an agent' @@ -42,9 +44,10 @@ module API present agent, with: Entities::Clusters::Agent end - desc 'Add an agent to a project' do - detail 'This feature was introduced in GitLab 14.10.' + desc 'Register an agent with a project' do + detail 'This feature was introduced in GitLab 14.10. Registers an agent to the project.' success Entities::Clusters::Agent + tags %w[cluster_agents] end params do requires :name, type: String, desc: 'The name of the agent' @@ -61,8 +64,9 @@ module API present result[:cluster_agent], with: Entities::Clusters::Agent end - desc 'Delete an agent' do - detail 'This feature was introduced in GitLab 14.10.' + desc 'Delete a registered agent' do + detail 'This feature was introduced in GitLab 14.10. Deletes an existing agent registration.' + tags %w[cluster_agents] end params do requires :agent_id, type: Integer, desc: 'The ID of an agent' diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 7d8b58fd7b6..954b572c9b1 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -8,7 +8,7 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include PaginationParams @@ -16,14 +16,20 @@ module API before { authenticate! } desc "Get a commit's statuses" do - success Entities::CommitStatus + success code: 200, model: Entities::CommitStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true end params do - requires :sha, type: String, desc: 'The commit hash' - optional :ref, type: String, desc: 'The ref' - optional :stage, type: String, desc: 'The stage' - optional :name, type: String, desc: 'The name' - optional :all, type: String, desc: 'Show all statuses, default: false' + requires :sha, type: String, desc: 'The commit hash', documentation: { example: '18f3e63d05582537db6d183d9d557be09e1f90c8' } + optional :ref, type: String, desc: 'The ref', documentation: { example: 'develop' } + optional :stage, type: String, desc: 'The stage', documentation: { example: 'test' } + optional :name, type: String, desc: 'The name', documentation: { example: 'bundler:audit' } + optional :all, type: Boolean, desc: 'Show all statuses', documentation: { default: false } use :pagination end # rubocop: disable CodeReuse/ActiveRecord @@ -43,19 +49,32 @@ module API # rubocop: enable CodeReuse/ActiveRecord desc 'Post status to a commit' do - success Entities::CommitStatus + success code: 200, model: Entities::CommitStatus + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :sha, type: String, desc: 'The commit hash' - requires :state, type: String, desc: 'The state of the status', - values: %w(pending running success failed canceled) - optional :ref, type: String, desc: 'The ref' - optional :target_url, type: String, desc: 'The target URL to associate with this status' - optional :description, type: String, desc: 'A short description of the status' - optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"', documentation: { default: 'default' } - optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"', documentation: { default: 'default' } - optional :coverage, type: Float, desc: 'The total code coverage' - optional :pipeline_id, type: Integer, desc: 'An existing pipeline ID, when multiple pipelines on the same commit SHA have been triggered' + requires :sha, type: String, desc: 'The commit hash', + documentation: { example: '18f3e63d05582537db6d183d9d557be09e1f90c8' } + requires :state, type: String, desc: 'The state of the status', + values: %w(pending running success failed canceled), + documentation: { example: 'pending' } + optional :ref, type: String, desc: 'The ref', + documentation: { example: 'develop' } + optional :target_url, type: String, desc: 'The target URL to associate with this status', + documentation: { example: 'https://gitlab.example.com/janedoe/gitlab-foss/builds/91' } + optional :description, type: String, desc: 'A short description of the status' + optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems', + documentation: { example: 'coverage', default: 'default' } + optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems', + documentation: { example: 'coverage', default: 'default' } + optional :coverage, type: Float, desc: 'The total code coverage', + documentation: { example: 100.0 } + optional :pipeline_id, type: Integer, desc: 'An existing pipeline ID, when multiple pipelines on the same commit SHA have been triggered' end # rubocop: disable CodeReuse/ActiveRecord post ':id/statuses/:sha' do diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 50d0687ba75..63a13b83a9b 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -9,7 +9,7 @@ module API before do require_repository_enabled! - authorize! :download_code, user_project + authorize! :read_code, user_project verify_pagination_params! end @@ -27,17 +27,35 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, urgency: :low do desc 'Get a project repository commits' do - success Entities::Commit + success code: 200, model: Entities::Commit + tags %w[commits] + is_array true + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] end params do - optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' - optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' - optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' - optional :path, type: String, desc: 'The file path' + optional :ref_name, + type: String, + desc: 'The name of a repository branch or tag, if not given the default branch is used', + documentation: { example: 'v1.1.0' } + optional :since, + type: DateTime, + desc: 'Only commits after or on this date will be returned', + documentation: { example: '2021-09-20T11:50:22.001' } + optional :until, + type: DateTime, + desc: 'Only commits before or on this date will be returned', + documentation: { example: '2021-09-20T11:50:22.001' } + optional :path, + type: String, + desc: 'The file path', + documentation: { example: 'README.md' } optional :all, type: Boolean, desc: 'Every commit will be returned' optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response' optional :first_parent, type: Boolean, desc: 'Only include the first parent of merges' @@ -81,40 +99,87 @@ module API end desc 'Commit multiple file changes as one commit' do - success Entities::CommitDetail + success code: 200, model: Entities::CommitDetail + tags %w[commits] + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' } + ] detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide either `start_branch` or `start_sha`, and optionally `start_project`.', allow_blank: false - requires :commit_message, type: String, desc: 'Commit message' - requires :actions, type: Array, desc: 'Actions to perform in commit' do - requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze, allow_blank: false - requires :file_path, type: String, desc: 'Full path to the file. Ex. `lib/class.rb`' + requires :branch, + type: String, + desc: 'Name of the branch to commit into. To create a new branch, also provide either `start_branch` or `start_sha`, and optionally `start_project`.', + allow_blank: false, + documentation: { example: 'master' } + requires :commit_message, + type: String, + desc: 'Commit message', + documentation: { example: 'initial commit' } + requires :actions, + type: Array, + desc: 'Actions to perform in commit' do + requires :action, + type: String, + desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze, + allow_blank: false + requires :file_path, + type: String, + desc: 'Full path to the file.', + documentation: { example: 'lib/class.rb' } given action: ->(action) { action == 'move' } do - requires :previous_path, type: String, desc: 'Original full path to the file being moved. Ex. `lib/class1.rb`' + requires :previous_path, + type: String, + desc: 'Original full path to the file being moved.', + documentation: { example: 'lib/class.rb' } end given action: ->(action) { %w[create move].include? action } do - optional :content, type: String, desc: 'File content' + optional :content, + type: String, + desc: 'File content', + documentation: { example: 'Some file content' } end given action: ->(action) { action == 'update' } do - requires :content, type: String, desc: 'File content' + requires :content, + type: String, + desc: 'File content', + documentation: { example: 'Some file content' } end optional :encoding, type: String, desc: '`text` or `base64`', default: 'text', values: %w[text base64] given action: ->(action) { %w[update move delete].include? action } do - optional :last_commit_id, type: String, desc: 'Last known file commit id' + optional :last_commit_id, + type: String, + desc: 'Last known file commit id', + documentation: { example: '2695effb5807a22ff3d138d593fd856244e155e7' } end given action: ->(action) { action == 'chmod' } do requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' end end - optional :start_branch, type: String, desc: 'Name of the branch to start the new branch from' - optional :start_sha, type: String, desc: 'SHA of the commit to start the new branch from' + optional :start_branch, + type: String, + desc: 'Name of the branch to start the new branch from', + documentation: { example: 'staging' } + optional :start_sha, + type: String, + desc: 'SHA of the commit to start the new branch from', + documentation: { example: '2695effb5807a22ff3d138d593fd856244e155e7' } mutually_exclusive :start_branch, :start_sha - optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the new branch from' - optional :author_email, type: String, desc: 'Author email for commit' - optional :author_name, type: String, desc: 'Author name for commit' + optional :start_project, + types: [Integer, String], + desc: 'The ID or path of the project to start the new branch from', + documentation: { example: 1 } + optional :author_email, + type: String, + desc: 'Author email for commit', + documentation: { example: 'janedoe@example.com' } + optional :author_name, + type: String, + desc: 'Author name for commit', + documentation: { example: 'Jane Doe' } optional :stats, type: Boolean, default: true, desc: 'Include commit stats' optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch` or `start_sha`' end @@ -151,8 +216,11 @@ module API end desc 'Get a specific commit of a project' do - success Entities::CommitDetail - failure [[404, 'Commit Not Found']] + success code: 200, model: Entities::CommitDetail + tags %w[commits] + failure [ + { code: 404, message: 'Not found' } + ] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' @@ -167,7 +235,12 @@ module API end desc 'Get the diff for a specific commit of a project' do - failure [[404, 'Commit Not Found']] + success code: 200, model: Entities::Diff + tags %w[commits] + is_array true + failure [ + { code: 404, message: 'Not found' } + ] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' @@ -184,8 +257,12 @@ module API end desc "Get a commit's comments" do - success Entities::CommitNote - failure [[404, 'Commit Not Found']] + success code: 200, model: Entities::CommitNote + tags %w[commits] + is_array true + failure [ + { code: 404, message: 'Not found' } + ] end params do use :pagination @@ -202,13 +279,25 @@ module API desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' - success Entities::Commit + success code: 200, model: Entities::Commit + tags %w[commits] + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' } + ] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' - requires :branch, type: String, desc: 'The name of the branch', allow_blank: false + requires :branch, + type: String, + desc: 'The name of the branch', + allow_blank: false, + documentation: { example: 'master' } optional :dry_run, type: Boolean, default: false, desc: "Does not commit any changes" - optional :message, type: String, desc: 'A custom commit message to use for the picked commit' + optional :message, + type: String, + desc: 'A custom commit message to use for the picked commit', + documentation: { example: 'Initial commit' } end post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do authorize_push_to_branch!(params[:branch]) @@ -248,11 +337,20 @@ module API desc 'Revert a commit in a branch' do detail 'This feature was introduced in GitLab 11.5' - success Entities::Commit + success code: 200, model: Entities::Commit + tags %w[commits] + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' } + ] end params do requires :sha, type: String, desc: 'Commit SHA to revert' - requires :branch, type: String, desc: 'Target branch name', allow_blank: false + requires :branch, + type: String, + desc: 'Target branch name', + allow_blank: false, + documentation: { example: 'master' } optional :dry_run, type: Boolean, default: false, desc: "Does not commit any changes" end post ':id/repository/commits/:sha/revert', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do @@ -292,7 +390,12 @@ module API desc 'Get all references a commit is pushed to' do detail 'This feature was introduced in GitLab 10.6' - success Entities::BasicRef + success code: 200, model: Entities::BasicRef + tags %w[commits] + is_array true + failure [ + { code: 404, message: 'Not found' } + ] end params do requires :sha, type: String, desc: 'A commit sha' @@ -312,14 +415,28 @@ module API end desc 'Post comment to commit' do - success Entities::CommitNote + success code: 200, model: Entities::CommitNote + tags %w[commits] + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' } + ] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag on which to post a comment' - requires :note, type: String, desc: 'The text of the comment' - optional :path, type: String, desc: 'The file path' + requires :note, + type: String, + desc: 'The text of the comment', + documentation: { example: 'Nice code!' } + optional :path, + type: String, + desc: 'The file path', + documentation: { example: 'doc/update/5.4-to-6.0.md' } given :path do - requires :line, type: Integer, desc: 'The line number' + requires :line, + type: Integer, + desc: 'The line number', + documentation: { example: 11 } requires :line_type, type: String, values: %w[new old], default: 'new', desc: 'The type of the line' end end @@ -361,7 +478,12 @@ module API end desc 'Get Merge Requests associated with a commit' do - success Entities::MergeRequestBasic + success code: 200, model: Entities::MergeRequestBasic + tags %w[commits] + is_array true + failure [ + { code: 404, message: 'Not found' } + ] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag on which to find Merge Requests' @@ -383,7 +505,11 @@ module API end desc "Get a commit's signature" do - success Entities::CommitSignature + success code: 200, model: Entities::CommitSignature + tags %w[commits] + failure [ + { code: 404, message: 'Not found' } + ] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index d8c2eb4ff33..fdbffb1689b 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -29,7 +29,7 @@ module API CONAN_REVISION_REGEX = Gitlab::Regex.conan_revision_regex CONAN_REVISION_USER_CHANNEL_REGEX = Gitlab::Regex.conan_recipe_user_channel_regex - CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze + CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).uniq.freeze included do feature_category :package_registry @@ -307,7 +307,7 @@ module API end params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -358,7 +358,7 @@ module API end params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true diff --git a/lib/api/container_registry_event.rb b/lib/api/container_registry_event.rb index 66689f8d7c8..9acf2fca1b3 100644 --- a/lib/api/container_registry_event.rb +++ b/lib/api/container_registry_event.rb @@ -23,8 +23,20 @@ module API content_type :json, DOCKER_DISTRIBUTION_EVENTS_V1_JSON format :json + desc 'Receives notifications from the container registry when an operation occurs' do + detail 'This feature was introduced in GitLab 12.10' + consumes [:json, DOCKER_DISTRIBUTION_EVENTS_V1_JSON] + end params do - requires :events, type: Array + requires :events, type: Array, desc: 'Event notifications' do + requires :action, type: String, desc: 'The action to perform, `push`, `delete`', + values: %w[push delete].freeze + optional :target, type: Hash, desc: 'The target of the action' do + optional :tag, type: String, desc: 'The target tag' + optional :repository, type: String, desc: 'The target repository' + optional :digest, type: String, desc: 'Unique identifier for target image manifest' + end + end end # This endpoint is used by Docker Registry to push a set of event diff --git a/lib/api/container_repositories.rb b/lib/api/container_repositories.rb index d4fa6153a92..f2dd1fa21fd 100644 --- a/lib/api/container_repositories.rb +++ b/lib/api/container_repositories.rb @@ -14,7 +14,7 @@ module API namespace 'registry' do params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :repositories, requirements: { id: /[0-9]*/ } do desc 'Get a container repository' do diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 03f0f97b805..df3b6e774ae 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -34,7 +34,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end namespace ':id/packages/debian' do @@ -64,7 +64,7 @@ module API # PUT {projects|groups}/:id/packages/debian/:file_name params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true diff --git a/lib/api/dependency_proxy.rb b/lib/api/dependency_proxy.rb index 290a90934d7..fcf18a2792a 100644 --- a/lib/api/dependency_proxy.rb +++ b/lib/api/dependency_proxy.rb @@ -12,11 +12,18 @@ module API end params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, types: [String, Integer], + desc: 'The ID or URL-encoded path of the group owned by the authenticated user' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Deletes all dependency_proxy_blobs for a group' do - detail 'This feature was introduced in GitLab 12.10' + desc 'Purge the dependency proxy for a group' do + detail 'Schedules for deletion the cached manifests and blobs for a group.'\ + 'This endpoint requires the Owner role for the group.' + success code: 202 + failure [ + { code: 401, message: 'Unauthorized' } + ] + tags %w[dependency_proxy] end delete ':id/dependency_proxy/cache' do not_found! unless user_group.dependency_proxy_feature_available? diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index c53f4bca5a7..ffe0b6589bc 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -4,6 +4,8 @@ module API class DeployKeys < ::API::Base include PaginationParams + deploy_keys_tags = %w[deploy_keys] + before { authenticate! } feature_category :continuous_delivery @@ -21,7 +23,16 @@ module API # rubocop: enable CodeReuse/ActiveRecord end - desc 'Return all deploy keys' + desc 'List all deploy keys' do + detail 'Get a list of all deploy keys across all projects of the GitLab instance. This endpoint requires administrator access and is not available on GitLab.com.' + success Entities::DeployKey + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' } + ] + is_array true + tags deploy_keys_tags + end params do use :pagination optional :public, type: Boolean, default: false, desc: "Only return deploy keys that are public" @@ -35,13 +46,20 @@ module API end params do - requires :id, type: String, desc: 'The ID of the project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authorize_admin_project } - desc "Get a specific project's deploy keys" do + desc 'List deploy keys for project' do + detail "Get a list of a project's deploy keys." success Entities::DeployKeysProject + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags deploy_keys_tags end params do use :pagination @@ -54,8 +72,14 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'Get single deploy key' do + desc 'Get a single deploy key' do + detail 'Get a single key.' success Entities::DeployKeysProject + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_keys_tags end params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' @@ -66,12 +90,19 @@ module API present key, with: Entities::DeployKeysProject end - desc 'Add new deploy key to a project' do + desc 'Add deploy key' do + detail "Creates a new deploy key for a project. If the deploy key already exists in another project, it's joined to the current project only if the original one is accessible by the same user." success Entities::DeployKeysProject + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_keys_tags end params do - requires :key, type: String, desc: 'The new deploy key' - requires :title, type: String, desc: 'The name of the deploy key' + requires :key, type: String, desc: 'New deploy key' + requires :title, type: String, desc: "New deploy key's title" optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end # rubocop: disable CodeReuse/ActiveRecord @@ -109,12 +140,20 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'Update an existing deploy key for a project' do + desc 'Update deploy key' do + detail 'Updates a deploy key for a project.' success Entities::DeployKey + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags deploy_keys_tags end params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' - optional :title, type: String, desc: 'The name of the deploy key' + optional :title, type: String, desc: "New deploy key's title" optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" at_least_one_of :title, :can_push end @@ -143,9 +182,14 @@ module API end end - desc 'Enable a deploy key for a project' do - detail 'This feature was added in GitLab 8.11' + desc 'Enable a deploy key' do + detail 'Enables a deploy key for a project so this can be used. Returns the enabled key, with a status code 201 when successful. This feature was added in GitLab 8.11.' success Entities::DeployKey + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_keys_tags end params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' @@ -161,7 +205,14 @@ module API end end - desc 'Delete deploy key for a project' + desc 'Delete deploy key' do + detail "Removes a deploy key from the project. If the deploy key is used only for this project, it's deleted from the system." + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_keys_tags + end params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' end diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb index 3955e29621f..975a65af285 100644 --- a/lib/api/deploy_tokens.rb +++ b/lib/api/deploy_tokens.rb @@ -4,6 +4,8 @@ module API class DeployTokens < ::API::Base include PaginationParams + deploy_tokens_tags = %w[deploy_tokens] + feature_category :continuous_delivery urgency :low @@ -25,9 +27,15 @@ module API end end - desc 'Return all deploy tokens' do - detail 'This feature was introduced in GitLab 12.9.' + desc 'List all deploy tokens' do + detail 'Get a list of all deploy tokens across the GitLab instance. This endpoint requires administrator access. This feature was introduced in GitLab 12.9.' success Entities::DeployToken + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' } + ] + is_array true + tags deploy_tokens_tags end params do use :pagination @@ -46,16 +54,23 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do use :pagination use :filter_params end - desc 'List deploy tokens for a project' do - detail 'This feature was introduced in GitLab 12.9' + desc 'List project deploy tokens' do + detail "Get a list of a project's deploy tokens. This feature was introduced in GitLab 12.9." success Entities::DeployToken + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags deploy_tokens_tags end get ':id/deploy_tokens' do authorize!(:read_deploy_token, user_project) @@ -75,13 +90,19 @@ module API type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, values: ::DeployToken::AVAILABLE_SCOPES.map(&:to_s), - desc: 'Indicates the deploy token scopes. Must be at least one of "read_repository", "read_registry", "write_registry", "read_package_registry", or "write_package_registry".' - optional :expires_at, type: DateTime, desc: 'Expiration date for the deploy token. Does not expire if no value is provided.' + desc: 'Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, `write_registry`, `read_package_registry`, or `write_package_registry`.' + optional :expires_at, type: DateTime, desc: 'Expiration date for the deploy token. Does not expire if no value is provided. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`).' optional :username, type: String, desc: 'Username for deploy token. Default is `gitlab+deploy-token-{n}`' end desc 'Create a project deploy token' do - detail 'This feature was introduced in GitLab 12.9' + detail 'Creates a new deploy token for a project. This feature was introduced in GitLab 12.9.' success Entities::DeployTokenWithToken + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_tokens_tags end post ':id/deploy_tokens' do authorize!(:create_deploy_token, user_project) @@ -98,11 +119,16 @@ module API end desc 'Get a project deploy token' do - detail 'This feature was introduced in GitLab 14.9' + detail "Get a single project's deploy token by ID. This feature was introduced in GitLab 14.9." success Entities::DeployToken + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_tokens_tags end params do - requires :token_id, type: Integer, desc: 'The deploy token ID' + requires :token_id, type: Integer, desc: 'The ID of the deploy token' end get ':id/deploy_tokens/:token_id' do authorize!(:read_deploy_token, user_project) @@ -113,10 +139,15 @@ module API end desc 'Delete a project deploy token' do - detail 'This feature was introduced in GitLab 12.9' + detail 'This feature was introduced in GitLab 12.9.' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_tokens_tags end params do - requires :token_id, type: Integer, desc: 'The deploy token ID' + requires :token_id, type: Integer, desc: 'The ID of the deploy token' end delete ':id/deploy_tokens/:token_id' do authorize!(:destroy_deploy_token, user_project) @@ -130,16 +161,23 @@ module API end params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, types: [Integer, String], desc: 'The ID or URL-encoded path of the group owned by the authenticated user' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do use :pagination use :filter_params end - desc 'List deploy tokens for a group' do - detail 'This feature was introduced in GitLab 12.9' + desc 'List group deploy tokens' do + detail "Get a list of a group's deploy tokens. This feature was introduced in GitLab 12.9." success Entities::DeployToken + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags deploy_tokens_tags end get ':id/deploy_tokens' do authorize!(:read_deploy_token, user_group) @@ -154,18 +192,24 @@ module API end params do - requires :name, type: String, desc: 'The name of the deploy token' + requires :name, type: String, desc: "New deploy token's name" requires :scopes, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, values: ::DeployToken::AVAILABLE_SCOPES.map(&:to_s), - desc: 'Indicates the deploy token scopes. Must be at least one of "read_repository", "read_registry", "write_registry", "read_package_registry", or "write_package_registry".' - optional :expires_at, type: DateTime, desc: 'Expiration date for the deploy token. Does not expire if no value is provided.' + desc: 'Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, `write_registry`, `read_package_registry`, or `write_package_registry`' + optional :expires_at, type: DateTime, desc: 'Expiration date for the deploy token. Does not expire if no value is provided. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`)' optional :username, type: String, desc: 'Username for deploy token. Default is `gitlab+deploy-token-{n}`' end desc 'Create a group deploy token' do - detail 'This feature was introduced in GitLab 12.9' + detail 'Creates a new deploy token for a group. This feature was introduced in GitLab 12.9.' success Entities::DeployTokenWithToken + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_tokens_tags end post ':id/deploy_tokens' do authorize!(:create_deploy_token, user_group) @@ -182,11 +226,16 @@ module API end desc 'Get a group deploy token' do - detail 'This feature was introduced in GitLab 14.9' + detail "Get a single group's deploy token by ID. This feature was introduced in GitLab 14.9. " success Entities::DeployToken + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_tokens_tags end params do - requires :token_id, type: Integer, desc: 'The deploy token ID' + requires :token_id, type: Integer, desc: 'The ID of the deploy token' end get ':id/deploy_tokens/:token_id' do authorize!(:read_deploy_token, user_group) @@ -197,10 +246,15 @@ module API end desc 'Delete a group deploy token' do - detail 'This feature was introduced in GitLab 12.9' + detail 'Removes a deploy token from the group. This feature was introduced in GitLab 12.9.' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deploy_tokens_tags end params do - requires :token_id, type: Integer, desc: 'The deploy token ID' + requires :token_id, type: Integer, desc: 'The ID of the deploy token' end delete ':id/deploy_tokens/:token_id' do authorize!(:destroy_deploy_token, user_group) diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index ee0a026d7ac..141f089b5e1 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -5,25 +5,51 @@ module API class Deployments < ::API::Base include PaginationParams + deployments_tags = %w[deployments] + before { authenticate! } feature_category :continuous_delivery urgency :low params do - requires :id, type: String, desc: 'The project ID' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get all deployments of the project' do - detail 'This feature was introduced in GitLab 8.11.' + desc 'List project deployments' do + detail 'Get a list of deployments in a project. This feature was introduced in GitLab 8.11.' success Entities::Deployment + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags deployments_tags end params do use :pagination - optional :order_by, type: String, values: DeploymentsFinder::ALLOWED_SORT_VALUES, default: DeploymentsFinder::DEFAULT_SORT_VALUE, desc: 'Return deployments ordered by specified value' - optional :sort, type: String, values: DeploymentsFinder::ALLOWED_SORT_DIRECTIONS, default: DeploymentsFinder::DEFAULT_SORT_DIRECTION, desc: 'Sort by asc (ascending) or desc (descending)' - optional :updated_after, type: DateTime, desc: 'Return deployments updated after the specified date' - optional :updated_before, type: DateTime, desc: 'Return deployments updated before the specified date' + + optional :order_by, + type: String, + values: DeploymentsFinder::ALLOWED_SORT_VALUES, + default: DeploymentsFinder::DEFAULT_SORT_VALUE, + desc: 'Return deployments ordered by either one of `id`, `iid`, `created_at`, `updated_at` or `ref` fields. Default is `id`' + + optional :sort, + type: String, + values: DeploymentsFinder::ALLOWED_SORT_DIRECTIONS, + default: DeploymentsFinder::DEFAULT_SORT_DIRECTION, + desc: 'Return deployments sorted in `asc` or `desc` order. Default is `asc`' + + optional :updated_after, + type: DateTime, + desc: 'Return deployments updated after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`)' + + optional :updated_before, + type: DateTime, + desc: 'Return deployments updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`)' + optional :environment, type: String, desc: 'The name of the environment to filter deployments by' @@ -31,7 +57,7 @@ module API optional :status, type: String, values: Deployment.statuses.keys, - desc: 'The status to filter deployments by' + desc: 'The status to filter deployments by. One of `created`, `running`, `success`, `failed`, `canceled`, or `blocked`' end get ':id/deployments' do @@ -46,12 +72,17 @@ module API bad_request!(e.message) end - desc 'Gets a specific deployment' do + desc 'Get a specific deployment' do detail 'This feature was introduced in GitLab 8.11.' success Entities::DeploymentExtended + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deployments_tags end params do - requires :deployment_id, type: Integer, desc: 'The deployment ID' + requires :deployment_id, type: Integer, desc: 'The ID of the deployment' end get ':id/deployments/:deployment_id' do authorize! :read_deployment, user_project @@ -61,30 +92,36 @@ module API present deployment, with: Entities::DeploymentExtended end - desc 'Creates a new deployment' do - detail 'This feature was introduced in GitLab 12.4' + desc 'Create a deployment' do + detail 'This feature was introduced in GitLab 12.4.' success Entities::DeploymentExtended + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags deployments_tags end params do requires :environment, type: String, - desc: 'The name of the environment to deploy to' + desc: 'The name of the environment to create the deployment for' requires :sha, type: String, - desc: 'The SHA of the commit that was deployed' + desc: 'The SHA of the commit that is deployed' requires :ref, type: String, - desc: 'The name of the branch or tag that was deployed' + desc: 'The name of the branch or tag that is deployed' requires :tag, type: Boolean, - desc: 'A boolean indicating if the deployment ran for a tag' + desc: 'A boolean that indicates if the deployed ref is a tag (`true`) or not (`false`)' requires :status, type: String, - desc: 'The status of the deployment', + desc: 'The status to filter deployments by. One of `running`, `success`, `failed`, or `canceled`', values: %w[running success failed canceled] end post ':id/deployments' do @@ -96,7 +133,7 @@ module API .find_or_create_by_name(params[:environment]) unless environment.persisted? - render_validation_error!(deployment) + render_validation_error!(environment) end authorize!(:create_deployment, environment) @@ -113,14 +150,21 @@ module API end end - desc 'Updates an existing deployment' do - detail 'This feature was introduced in GitLab 12.4' + desc 'Update a deployment' do + detail 'This feature was introduced in GitLab 12.4.' success Entities::DeploymentExtended + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags deployments_tags end params do requires :status, type: String, - desc: 'The new status of the deployment', + desc: 'The new status of the deployment. One of `running`, `success`, `failed`, or `canceled`', values: %w[running success failed canceled] end put ':id/deployments/:deployment_id' do @@ -143,12 +187,17 @@ module API end end - desc 'Deletes an existing deployment' do - detail 'This feature was introduced in GitLab 15.3' - http_codes [[204, 'Deployment was deleted'], [403, 'Forbidden'], [400, 'Cannot destroy']] + desc 'Delete a specific deployment' do + detail 'Delete a specific deployment that is not currently the last deployment for an environment or in a running state. This feature was introduced in GitLab 15.3.' + http_codes [ + [204, 'Deployment destroyed'], + [403, 'Forbidden'], + [400, '"Cannot destroy running deployment" or "Deployment currently deployed to environment"'] + ] + tags deployments_tags end params do - requires :deployment_id, type: Integer, desc: 'The deployment ID' + requires :deployment_id, type: Integer, desc: 'The ID of the deployment' end delete ':id/deployments/:deployment_id' do deployment = user_project.deployments.find(params[:deployment_id]) @@ -166,13 +215,21 @@ module API helpers Helpers::MergeRequestsHelpers - desc 'Get all merge requests of a deployment' do - detail 'This feature was introduced in GitLab 12.7.' + desc 'List of merge requests associated with a deployment' do + detail 'Retrieves the list of merge requests shipped with a given deployment. This feature was introduced in GitLab 12.7.' success Entities::MergeRequestBasic + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags deployments_tags end params do use :pagination - requires :deployment_id, type: Integer, desc: 'The deployment ID' + + requires :deployment_id, type: Integer, desc: 'The ID of the deployment' + use :merge_requests_base_params end diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index f73e4b621ab..d3a25a076a0 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -18,17 +18,19 @@ module API Helpers::DiscussionsHelpers.feature_category_per_noteable_type.each do |noteable_type, feature_category| parent_type = noteable_type.parent_class.to_s.underscore noteables_str = noteable_type.to_s.underscore.pluralize + notable_name = noteable_type.to_s.underscore.humanize.downcase + notable_id_type = noteable_type == Commit ? String : Integer noteables_path = noteable_type == Commit ? "repository/#{noteables_str}" : noteables_str params do requires :id, type: String, desc: "The ID of a #{parent_type}" end resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc "Get a list of #{noteable_type.to_s.downcase} discussions" do + desc "Get a list of #{notable_name} discussions" do success Entities::Discussion end params do - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" use :pagination end @@ -41,12 +43,12 @@ module API present Discussion.build_collection(notes, noteable), with: Entities::Discussion end - desc "Get a single #{noteable_type.to_s.downcase} discussion" do + desc "Get a single #{notable_name} discussion" do success Entities::Discussion end params do requires :discussion_id, type: String, desc: 'The ID of a discussion' - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" end get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) @@ -61,39 +63,44 @@ module API present discussion, with: Entities::Discussion end - desc "Create a new #{noteable_type.to_s.downcase} discussion" do + desc "Create a new #{notable_name} discussion" do success Entities::Discussion end params do - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" requires :body, type: String, desc: 'The content of a note' optional :created_at, type: String, desc: 'The creation date of the note' - optional :position, type: Hash do - requires :base_sha, type: String, desc: 'Base commit SHA in the source branch' - requires :start_sha, type: String, desc: 'SHA referencing commit in target branch' - requires :head_sha, type: String, desc: 'SHA referencing HEAD of this merge request' - requires :position_type, type: String, desc: 'Type of the position reference', values: %w(text image) - optional :new_path, type: String, desc: 'File path after change' - optional :new_line, type: Integer, desc: 'Line number after change' - optional :old_path, type: String, desc: 'File path before change' - optional :old_line, type: Integer, desc: 'Line number before change' - optional :width, type: Integer, desc: 'Width of the image' - optional :height, type: Integer, desc: 'Height of the image' - optional :x, type: Integer, desc: 'X coordinate in the image' - optional :y, type: Integer, desc: 'Y coordinate in the image' - - optional :line_range, type: Hash, desc: 'Multi-line start and end' do - optional :start, type: Hash do - optional :line_code, type: String, desc: 'Start line code for multi-line note' - optional :type, type: String, desc: 'Start line type for multi-line note' - optional :old_line, type: String, desc: 'Start old_line line number' - optional :new_line, type: String, desc: 'Start new_line line number' - end - optional :end, type: Hash do - optional :line_code, type: String, desc: 'End line code for multi-line note' - optional :type, type: String, desc: 'End line type for multi-line note' - optional :old_line, type: String, desc: 'End old_line line number' - optional :new_line, type: String, desc: 'End new_line line number' + + if [Commit, MergeRequest].include?(noteable_type) + optional :position, type: Hash do + requires :base_sha, type: String, desc: 'Base commit SHA in the source branch' + requires :start_sha, type: String, desc: 'SHA referencing commit in target branch' + requires :head_sha, type: String, desc: 'SHA referencing HEAD of this merge request' + requires :position_type, type: String, desc: 'Type of the position reference', values: %w(text image) + optional :new_path, type: String, desc: 'File path after change' + optional :new_line, type: Integer, desc: 'Line number after change' + optional :old_path, type: String, desc: 'File path before change' + optional :old_line, type: Integer, desc: 'Line number before change' + optional :width, type: Integer, desc: 'Width of the image' + optional :height, type: Integer, desc: 'Height of the image' + optional :x, type: Integer, desc: 'X coordinate in the image' + optional :y, type: Integer, desc: 'Y coordinate in the image' + + if noteable_type == MergeRequest + optional :line_range, type: Hash, desc: 'Multi-line start and end' do + optional :start, type: Hash do + optional :line_code, type: String, desc: 'Start line code for multi-line note' + optional :type, type: String, desc: 'Start line type for multi-line note' + optional :old_line, type: String, desc: 'Start old_line line number' + optional :new_line, type: String, desc: 'Start new_line line number' + end + optional :end, type: Hash do + optional :line_code, type: String, desc: 'End line code for multi-line note' + optional :type, type: String, desc: 'End line type for multi-line note' + optional :old_line, type: String, desc: 'End old_line line number' + optional :new_line, type: String, desc: 'End new_line line number' + end + end end end end @@ -122,12 +129,12 @@ module API end end - desc "Get comments in a single #{noteable_type.to_s.downcase} discussion" do + desc "Get comments in a single #{notable_name} discussion" do success Entities::Discussion end params do requires :discussion_id, type: String, desc: 'The ID of a discussion' - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" end get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) @@ -140,11 +147,11 @@ module API present notes, with: Entities::Note end - desc "Add a comment to a #{noteable_type.to_s.downcase} discussion" do + desc "Add a comment to a #{notable_name} discussion" do success Entities::Note end params do - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :body, type: String, desc: 'The content of a note' optional :created_at, type: String, desc: 'The creation date of the note' @@ -175,11 +182,11 @@ module API end end - desc "Get a comment in a #{noteable_type.to_s.downcase} discussion" do + desc "Get a comment in a #{notable_name} discussion" do success Entities::Note end params do - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :note_id, type: Integer, desc: 'The ID of a note' end @@ -189,11 +196,11 @@ module API get_note(noteable, params[:note_id]) end - desc "Edit a comment in a #{noteable_type.to_s.downcase} discussion" do + desc "Edit a comment in a #{notable_name} discussion" do success Entities::Note end params do - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :note_id, type: Integer, desc: 'The ID of a note' optional :body, type: String, desc: 'The content of a note' @@ -210,11 +217,11 @@ module API end end - desc "Delete a comment in a #{noteable_type.to_s.downcase} discussion" do + desc "Delete a comment in a #{notable_name} discussion" do success Entities::Note end params do - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :note_id, type: Integer, desc: 'The ID of a note' end @@ -225,11 +232,11 @@ module API end if Noteable.resolvable_types.include?(noteable_type.to_s) - desc "Resolve/unresolve an existing #{noteable_type.to_s.downcase} discussion" do + desc "Resolve/unresolve an existing #{notable_name} discussion" do success Entities::Discussion end params do - requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' + requires :noteable_id, type: notable_id_type, desc: "The ID of the #{notable_name}" requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :resolved, type: Boolean, desc: 'Mark discussion resolved/unresolved' end diff --git a/lib/api/entities/application.rb b/lib/api/entities/application.rb index 33514200424..c3d8f9667c3 100644 --- a/lib/api/entities/application.rb +++ b/lib/api/entities/application.rb @@ -4,10 +4,12 @@ module API module Entities class Application < Grape::Entity expose :id - expose :uid, as: :application_id - expose :name, as: :application_name - expose :redirect_uri, as: :callback_url - expose :confidential + expose :uid, as: :application_id, + documentation: { type: 'string', + example: '5832fc6e14300a0d962240a8144466eef4ee93ef0d218477e55f11cf12fc3737' } + expose :name, as: :application_name, documentation: { type: 'string', example: 'MyApplication' } + expose :redirect_uri, as: :callback_url, documentation: { type: 'string', example: 'https://redirect.uri' } + expose :confidential, documentation: { type: 'boolean', example: true } end end end diff --git a/lib/api/entities/application_statistics.rb b/lib/api/entities/application_statistics.rb index 4bcba1da464..7e75ef23675 100644 --- a/lib/api/entities/application_statistics.rb +++ b/lib/api/entities/application_statistics.rb @@ -6,47 +6,57 @@ module API include ActionView::Helpers::NumberHelper include CountHelper - expose :forks do |counts| + expose :forks, + documentation: { type: 'integer', example: 6, desc: 'Approximate number of repo forks' } do |counts| approximate_fork_count_with_delimiters(counts) end - expose :issues do |counts| + expose :issues, + documentation: { type: 'integer', example: 121, desc: 'Approximate number of issues' } do |counts| approximate_count_with_delimiters(counts, ::Issue) end - expose :merge_requests do |counts| + expose :merge_requests, + documentation: { type: 'integer', example: 49, desc: 'Approximate number of merge requests' } do |counts| approximate_count_with_delimiters(counts, ::MergeRequest) end - expose :notes do |counts| + expose :notes, + documentation: { type: 'integer', example: 6, desc: 'Approximate number of notes' } do |counts| approximate_count_with_delimiters(counts, ::Note) end - expose :snippets do |counts| + expose :snippets, + documentation: { type: 'integer', example: 4, desc: 'Approximate number of snippets' } do |counts| approximate_count_with_delimiters(counts, ::Snippet) end - expose :ssh_keys do |counts| + expose :ssh_keys, + documentation: { type: 'integer', example: 11, desc: 'Approximate number of SSH keys' } do |counts| approximate_count_with_delimiters(counts, ::Key) end - expose :milestones do |counts| + expose :milestones, + documentation: { type: 'integer', example: 3, desc: 'Approximate number of milestones' } do |counts| approximate_count_with_delimiters(counts, ::Milestone) end - expose :users do |counts| + expose :users, documentation: { type: 'integer', example: 22, desc: 'Approximate number of users' } do |counts| approximate_count_with_delimiters(counts, ::User) end - expose :projects do |counts| + expose :projects, + documentation: { type: 'integer', example: 4, desc: 'Approximate number of projects' } do |counts| approximate_count_with_delimiters(counts, ::Project) end - expose :groups do |counts| + expose :groups, + documentation: { type: 'integer', example: 1, desc: 'Approximate number of projects' } do |counts| approximate_count_with_delimiters(counts, ::Group) end - expose :active_users do |_| + expose :active_users, + documentation: { type: 'integer', example: 21, desc: 'Number of active users' } do |_| number_with_delimiter(::User.active.count) end end diff --git a/lib/api/entities/application_with_secret.rb b/lib/api/entities/application_with_secret.rb index 3e540381d89..1d0acee8624 100644 --- a/lib/api/entities/application_with_secret.rb +++ b/lib/api/entities/application_with_secret.rb @@ -4,7 +4,8 @@ module API module Entities # Use with care, this exposes the secret class ApplicationWithSecret < Entities::Application - expose :secret + expose :secret, documentation: { type: 'string', + example: 'ee1dd64b6adc89cf7e2c23099301ccc2c61b441064e9324d963c46902a85ec34' } end end end diff --git a/lib/api/entities/basic_project_details.rb b/lib/api/entities/basic_project_details.rb index e96504db53e..2585b2d0b6d 100644 --- a/lib/api/entities/basic_project_details.rb +++ b/lib/api/entities/basic_project_details.rb @@ -6,15 +6,18 @@ module API include ::API::ProjectsRelationBuilder include Gitlab::Utils::StrongMemoize - expose :default_branch_or_main, as: :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } + expose :default_branch_or_main, documentation: { type: 'string', example: 'main' }, as: :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :read_code, project) } # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 - expose :topic_names, as: :tag_list - expose :topic_names, as: :topics + expose :topic_names, as: :tag_list, documentation: { type: 'string', is_array: true, example: 'tag' } + expose :topic_names, as: :topics, documentation: { type: 'string', is_array: true, example: 'topic' } - expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url + expose :ssh_url_to_repo, documentation: { type: 'string', example: 'git@gitlab.example.com:gitlab/gitlab.git' } + expose :http_url_to_repo, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab.git' } + expose :web_url, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab' } + expose :readme_url, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab/blob/master/README.md' } - expose :license_url, if: :license do |project| + expose :license_url, if: :license, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab/blob/master/LICENCE' } do |project| license = project.repository.license_blob if license @@ -26,13 +29,13 @@ module API project.repository.license end - expose :avatar_url do |project, options| + expose :avatar_url, documentation: { type: 'string', example: 'http://example.com/uploads/project/avatar/3/uploads/avatar.png' } do |project, options| project.avatar_url(only_path: false) end - expose :forks_count - expose :star_count - expose :last_activity_at + expose :forks_count, documentation: { type: 'integer', example: 1 } + expose :star_count, documentation: { type: 'integer', example: 1 } + expose :last_activity_at, documentation: { type: 'dateTime', example: '2013-09-30T13:46:02Z' } expose :namespace, using: 'API::Entities::NamespaceBasic' expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes @@ -57,6 +60,8 @@ module API super end + def self.postload_relation(projects_relation, options = {}) end + private alias_method :project, :object diff --git a/lib/api/entities/basic_ref.rb b/lib/api/entities/basic_ref.rb index 79c15075d99..1b821a5b0ec 100644 --- a/lib/api/entities/basic_ref.rb +++ b/lib/api/entities/basic_ref.rb @@ -3,7 +3,8 @@ module API module Entities class BasicRef < Grape::Entity - expose :type, :name + expose :type, documentation: { type: 'string', example: 'tag' } + expose :name, documentation: { type: 'string', example: 'v1.1.0' } end end end diff --git a/lib/api/entities/basic_release_details.rb b/lib/api/entities/basic_release_details.rb index d13080f32f4..dba19b3abd7 100644 --- a/lib/api/entities/basic_release_details.rb +++ b/lib/api/entities/basic_release_details.rb @@ -5,12 +5,12 @@ module API class BasicReleaseDetails < Grape::Entity include ::API::Helpers::Presentable - expose :name - expose :tag, as: :tag_name - expose :description - expose :created_at - expose :released_at - expose :upcoming_release?, as: :upcoming_release + expose :name, documentation: { type: 'string', example: 'Release v1.0' } + expose :tag, documentation: { type: 'string', example: 'v1.0' }, as: :tag_name + expose :description, documentation: { type: 'string', example: 'Finally released v1.0' } + expose :created_at, documentation: { type: 'dateTime', example: '2019-01-03T01:56:19.539Z' } + expose :released_at, documentation: { type: 'dateTime', example: '2019-01-03T01:56:19.539Z' } + expose :upcoming_release?, documentation: { type: 'boolean' }, as: :upcoming_release end end end diff --git a/lib/api/entities/basic_repository_storage_move.rb b/lib/api/entities/basic_repository_storage_move.rb index 3ee112fb9a2..83b4f428a56 100644 --- a/lib/api/entities/basic_repository_storage_move.rb +++ b/lib/api/entities/basic_repository_storage_move.rb @@ -3,11 +3,11 @@ module API module Entities class BasicRepositoryStorageMove < Grape::Entity - expose :id - expose :created_at - expose :human_state_name, as: :state - expose :source_storage_name - expose :destination_storage_name + expose :id, documentation: { type: 'integer', example: 1 } + expose :created_at, documentation: { type: 'dateTime', example: '2020-05-07T04:27:17.234Z' } + expose :human_state_name, as: :state, documentation: { type: 'string', example: 'scheduled' } + expose :source_storage_name, documentation: { type: 'string', example: 'default' } + expose :destination_storage_name, documentation: { type: 'string', example: 'storage1' } end end end diff --git a/lib/api/entities/basic_snippet.rb b/lib/api/entities/basic_snippet.rb index 26297514798..0e9977fd81b 100644 --- a/lib/api/entities/basic_snippet.rb +++ b/lib/api/entities/basic_snippet.rb @@ -3,16 +3,30 @@ module API module Entities class BasicSnippet < Grape::Entity - expose :id, :title, :description, :visibility - expose :updated_at, :created_at - expose :project_id - expose :web_url do |snippet| + expose :id, documentation: { type: 'integer', example: 1 } + expose :title, documentation: { type: 'string', example: 'test' } + expose :description, documentation: { type: 'string', example: 'Ruby test snippet' } + expose :visibility, documentation: { type: 'string', example: 'public' } + expose :author, using: Entities::UserBasic, documentation: { type: 'Entities::UserBasic' } + expose :created_at, documentation: { type: 'dateTime', example: '2012-06-28T10:52:04Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2012-06-28T10:52:04Z' } + expose :project_id, documentation: { type: 'integer', example: 1 } + expose :web_url, documentation: { + type: 'string', example: 'http://example.com/example/example/snippets/1' + } do |snippet| Gitlab::UrlBuilder.build(snippet) end - expose :raw_url do |snippet| + expose :raw_url, documentation: { + type: 'string', example: 'http://example.com/example/example/snippets/1/raw' + } do |snippet| Gitlab::UrlBuilder.build(snippet, raw: true) end - expose :ssh_url_to_repo, :http_url_to_repo, if: ->(snippet) { snippet.repository_exists? } + expose :ssh_url_to_repo, documentation: { + type: 'string', example: 'ssh://user@gitlab.example.com/snippets/65.git' + }, if: ->(snippet) { snippet.repository_exists? } + expose :http_url_to_repo, documentation: { + type: 'string', example: 'https://gitlab.example.com/snippets/65.git' + }, if: ->(snippet) { snippet.repository_exists? } end end end diff --git a/lib/api/entities/branch.rb b/lib/api/entities/branch.rb index 6a75dcddeda..01eaf5c8d31 100644 --- a/lib/api/entities/branch.rb +++ b/lib/api/entities/branch.rb @@ -5,13 +5,17 @@ module API class Branch < Grape::Entity include Gitlab::Routing - expose :name + expose :name, documentation: { type: 'string', example: 'master' } expose :commit, using: Entities::Commit do |repo_branch, options| options[:project].repository.commit(repo_branch.dereferenced_target) end - expose :merged do |repo_branch, options| + expose :merged, + documentation: { + type: 'boolean', + example: true + } do |repo_branch, options| if options[:merged_branch_names] options[:merged_branch_names].include?(repo_branch.name) else @@ -19,27 +23,51 @@ module API end end - expose :protected do |repo_branch, options| + expose :protected, + documentation: { + type: 'boolean', + example: true + } do |repo_branch, options| ::ProtectedBranch.protected?(options[:project], repo_branch.name) end - expose :developers_can_push do |repo_branch, options| + expose :developers_can_push, + documentation: { + type: 'boolean', + example: true + } do |repo_branch, options| ::ProtectedBranch.developers_can?(:push, repo_branch.name, protected_refs: options[:project].protected_branches) end - expose :developers_can_merge do |repo_branch, options| + expose :developers_can_merge, + documentation: { + type: 'boolean', + example: true + } do |repo_branch, options| ::ProtectedBranch.developers_can?(:merge, repo_branch.name, protected_refs: options[:project].protected_branches) end - expose :can_push do |repo_branch, options| + expose :can_push, + documentation: { + type: 'boolean', + example: true + } do |repo_branch, options| Gitlab::UserAccess.new(options[:current_user], container: options[:project]).can_push_to_branch?(repo_branch.name) end - expose :default do |repo_branch, options| + expose :default, + documentation: { + type: 'boolean', + example: true + } do |repo_branch, options| options[:project].default_branch == repo_branch.name end - expose :web_url do |repo_branch| + expose :web_url, + documentation: { + type: 'string', + example: 'https://gitlab.example.com/Commit921/the-dude/-/tree/master' + } do |repo_branch| project_tree_url(options[:project], repo_branch.name) end end diff --git a/lib/api/entities/bulk_import.rb b/lib/api/entities/bulk_import.rb index 373ae486dcf..75989cb4180 100644 --- a/lib/api/entities/bulk_import.rb +++ b/lib/api/entities/bulk_import.rb @@ -3,11 +3,13 @@ module API module Entities class BulkImport < Grape::Entity - expose :id - expose :status_name, as: :status - expose :source_type - expose :created_at - expose :updated_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :status_name, as: :status, documentation: { + type: 'string', example: 'finished', values: %w[created started finished timeout failed] + } + expose :source_type, documentation: { type: 'string', example: 'gitlab' } + expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } end end end diff --git a/lib/api/entities/bulk_imports/entity.rb b/lib/api/entities/bulk_imports/entity.rb index 142bfaf2149..8f9fbe57935 100644 --- a/lib/api/entities/bulk_imports/entity.rb +++ b/lib/api/entities/bulk_imports/entity.rb @@ -4,19 +4,21 @@ module API module Entities module BulkImports class Entity < Grape::Entity - expose :id - expose :bulk_import_id - expose :status_name, as: :status - expose :source_full_path - expose :destination_name # deprecated - expose :destination_slug - expose :destination_namespace - expose :parent_id - expose :namespace_id - expose :project_id - expose :created_at - expose :updated_at - expose :failures, using: EntityFailure + expose :id, documentation: { type: 'integer', example: 1 } + expose :bulk_import_id, documentation: { type: 'integer', example: 1 } + expose :status_name, as: :status, documentation: { + type: 'string', example: 'created', values: %w[created started finished timeout failed] + } + expose :source_full_path, documentation: { type: 'string', example: 'source_group' } + expose :destination_name, documentation: { type: 'string', example: 'destination_slug' } # deprecated + expose :destination_slug, documentation: { type: 'string', example: 'destination_slug' } + expose :destination_namespace, documentation: { type: 'string', example: 'destination_path' } + expose :parent_id, documentation: { type: 'integer', example: 1 } + expose :namespace_id, documentation: { type: 'integer', example: 1 } + expose :project_id, documentation: { type: 'integer', example: 1 } + expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :failures, using: EntityFailure, documentation: { is_array: true } end end end diff --git a/lib/api/entities/bulk_imports/entity_failure.rb b/lib/api/entities/bulk_imports/entity_failure.rb index 56312745868..3e69e7fa2aa 100644 --- a/lib/api/entities/bulk_imports/entity_failure.rb +++ b/lib/api/entities/bulk_imports/entity_failure.rb @@ -4,16 +4,18 @@ module API module Entities module BulkImports class EntityFailure < Grape::Entity - expose :relation - expose :pipeline_step, as: :step - expose :exception_message do |failure| + expose :relation, documentation: { type: 'string', example: 'group' } + expose :pipeline_step, as: :step, documentation: { type: 'string', example: 'extractor' } + expose :exception_message, documentation: { type: 'string', example: 'error message' } do |failure| ::Projects::ImportErrorFilter.filter_message(failure.exception_message.truncate(72)) end - expose :exception_class - expose :correlation_id_value - expose :created_at - expose :pipeline_class - expose :pipeline_step + expose :exception_class, documentation: { type: 'string', example: 'Exception' } + expose :correlation_id_value, documentation: { type: 'string', example: 'dfcf583058ed4508e4c7c617bd7f0edd' } + expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :pipeline_class, documentation: { + type: 'string', example: 'BulkImports::Groups::Pipelines::GroupPipeline' + } + expose :pipeline_step, documentation: { type: 'string', example: 'extractor' } end end end diff --git a/lib/api/entities/bulk_imports/export_status.rb b/lib/api/entities/bulk_imports/export_status.rb index c9c7f34a16a..fee983c6fd8 100644 --- a/lib/api/entities/bulk_imports/export_status.rb +++ b/lib/api/entities/bulk_imports/export_status.rb @@ -4,10 +4,10 @@ module API module Entities module BulkImports class ExportStatus < Grape::Entity - expose :relation - expose :status - expose :error - expose :updated_at + expose :relation, documentation: { type: 'string', example: 'issues' } + expose :status, documentation: { type: 'string', example: 'started', values: %w[started finished failed] } + expose :error, documentation: { type: 'string', example: 'Error message' } + expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } end end end diff --git a/lib/api/entities/ci/job.rb b/lib/api/entities/ci/job.rb index cf87684ce55..d9e6b7eed75 100644 --- a/lib/api/entities/ci/job.rb +++ b/lib/api/entities/ci/job.rb @@ -6,10 +6,17 @@ module API class Job < JobBasic # artifacts_file is included in job_artifacts, but kept for backward compatibility (remove in api/v5) expose :artifacts_file, using: ::API::Entities::Ci::JobArtifactFile, if: -> (job, opts) { job.artifacts? } - expose :job_artifacts, as: :artifacts, using: ::API::Entities::Ci::JobArtifact + expose :job_artifacts, as: :artifacts, + using: ::API::Entities::Ci::JobArtifact, + documentation: { is_array: true } expose :runner, with: ::API::Entities::Ci::Runner - expose :artifacts_expire_at - expose :tag_list do |job| + expose :artifacts_expire_at, + documentation: { type: 'dateTime', example: '2016-01-19T09:05:50.355Z' } + + expose( + :tag_list, + documentation: { type: 'string', is_array: true, example: ['ubuntu18', 'docker runner'] } + ) do |job| job.tags.map(&:name).sort end end diff --git a/lib/api/entities/ci/job_artifact.rb b/lib/api/entities/ci/job_artifact.rb index 9e504aee383..8276c0f4073 100644 --- a/lib/api/entities/ci/job_artifact.rb +++ b/lib/api/entities/ci/job_artifact.rb @@ -4,7 +4,12 @@ module API module Entities module Ci class JobArtifact < Grape::Entity - expose :file_type, :size, :filename, :file_format + expose :file_type, + documentation: { type: 'string', values: ::Ci::JobArtifact.file_types.keys, example: 'archive' } + expose :size, documentation: { type: 'integer', example: 1000 } + expose :filename, documentation: { type: 'string', example: 'artifacts.zip' } + expose :file_format, + documentation: { type: 'string', values: ::Ci::JobArtifact.file_formats.keys, example: 'zip' } end end end diff --git a/lib/api/entities/ci/job_artifact_file.rb b/lib/api/entities/ci/job_artifact_file.rb index 418eb408ab6..0266f99cd6d 100644 --- a/lib/api/entities/ci/job_artifact_file.rb +++ b/lib/api/entities/ci/job_artifact_file.rb @@ -4,8 +4,8 @@ module API module Entities module Ci class JobArtifactFile < Grape::Entity - expose :filename - expose :cached_size, as: :size + expose :filename, documentation: { type: 'string', example: 'artifacts.zip' } + expose :cached_size, as: :size, documentation: { type: 'integer', example: 1000 } end end end diff --git a/lib/api/entities/ci/job_basic.rb b/lib/api/entities/ci/job_basic.rb index fb975475cf5..3cbb8aad313 100644 --- a/lib/api/entities/ci/job_basic.rb +++ b/lib/api/entities/ci/job_basic.rb @@ -4,23 +4,36 @@ module API module Entities module Ci class JobBasic < Grape::Entity - expose :id, :status, :stage, :name, :ref, :tag, :coverage, :allow_failure - expose :created_at, :started_at, :finished_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :status, documentation: { type: 'string', example: 'waiting_for_resource' } + expose :stage, documentation: { type: 'string', example: 'deploy' } + expose :name, documentation: { type: 'string', example: 'deploy_to_production' } + expose :ref, documentation: { type: 'string', example: 'main' } + expose :tag, documentation: { type: 'boolean' } + expose :coverage, documentation: { type: 'number', format: 'float', example: 98.29 } + expose :allow_failure, documentation: { type: 'boolean' } + expose :created_at, documentation: { type: 'dateTime', example: '2015-12-24T15:51:21.880Z' } + expose :started_at, documentation: { type: 'dateTime', example: '2015-12-24T17:54:30.733Z' } + expose :finished_at, documentation: { type: 'dateTime', example: '2015-12-24T17:54:31.198Z' } expose :duration, - documentation: { type: 'Floating', desc: 'Time spent running' } + documentation: { type: 'number', format: 'float', desc: 'Time spent running', example: 0.465 } expose :queued_duration, - documentation: { type: 'Floating', desc: 'Time spent enqueued' } + documentation: { type: 'number', format: 'float', desc: 'Time spent enqueued', example: 0.123 } expose :user, with: ::API::Entities::User expose :commit, with: ::API::Entities::Commit expose :pipeline, with: ::API::Entities::Ci::PipelineBasic - expose :failure_reason, if: -> (job) { job.failed? } + expose :failure_reason, + documentation: { type: 'string', example: 'script_failure' }, if: -> (job) { job.failed? } - expose :web_url do |job, _options| + expose( + :web_url, + documentation: { type: 'string', example: 'https://example.com/foo/bar/-/jobs/1' } + ) do |job, _options| Gitlab::Routing.url_helpers.project_job_url(job.project, job) end expose :project do - expose :ci_job_token_scope_enabled do |job| + expose :ci_job_token_scope_enabled, documentation: { type: 'string', example: false } do |job| job.project.ci_outbound_job_token_scope_enabled? end end diff --git a/lib/api/entities/ci/lint/result.rb b/lib/api/entities/ci/lint/result.rb index b44a6e13463..698b02d3b4a 100644 --- a/lib/api/entities/ci/lint/result.rb +++ b/lib/api/entities/ci/lint/result.rb @@ -5,12 +5,17 @@ module API module Ci module Lint class Result < Grape::Entity - expose :valid?, as: :valid - expose :errors - expose :warnings - expose :merged_yaml - expose :includes - expose :jobs, if: -> (result, options) { options[:include_jobs] } + expose :valid?, as: :valid, documentation: { type: 'boolean' } + expose :errors, documentation: { is_array: true, type: 'string', + example: 'variables config should be a hash of key value pairs' } + expose :warnings, documentation: { is_array: true, type: 'string', + example: 'jobs:job may allow multiple pipelines ...' } + expose :merged_yaml, documentation: { type: 'string', example: '---\n:another_test:\n :stage: test\n + :script: echo 2\n:test:\n :stage: test\n :script: echo 1\n' } + expose :includes, documentation: { is_array: true, type: 'object', + example: '{ "blob": "https://gitlab.com/root/example-project/-/blob/...' } + expose :jobs, if: -> (result, options) { options[:include_jobs] }, + documentation: { is_array: true, type: 'object', example: '{ "name": "test: .... }' } end end end diff --git a/lib/api/entities/ci/pipeline.rb b/lib/api/entities/ci/pipeline.rb index a8033a21044..7631cf60dbd 100644 --- a/lib/api/entities/ci/pipeline.rb +++ b/lib/api/entities/ci/pipeline.rb @@ -4,13 +4,21 @@ module API module Entities module Ci class Pipeline < PipelineBasic - expose :before_sha, :tag, :yaml_errors + expose :before_sha, documentation: { type: 'string', example: 'a91957a858320c0e17f3a0eca7cfacbff50ea29a' } + expose :tag, documentation: { type: 'boolean', example: false } + expose :yaml_errors, documentation: { type: 'string', example: "widgets:build: needs 'widgets:test'" } expose :user, with: Entities::UserBasic - expose :created_at, :updated_at, :started_at, :finished_at, :committed_at - expose :duration - expose :queued_duration - expose :coverage do |pipeline| + expose :created_at, documentation: { type: 'dateTime', example: '2015-12-24T15:51:21.880Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2015-12-24T17:54:31.198Z' } + expose :started_at, documentation: { type: 'dateTime', example: '2015-12-24T17:54:30.733Z' } + expose :finished_at, documentation: { type: 'dateTime', example: '2015-12-24T17:54:31.198Z' } + expose :committed_at, documentation: { type: 'dateTime', example: '2015-12-24T15:51:21.880Z' } + expose :duration, + documentation: { type: 'integer', desc: 'Time spent running in seconds', example: 127 } + expose :queued_duration, + documentation: { type: 'integer', desc: 'Time spent enqueued in seconds', example: 63 } + expose :coverage, documentation: { type: 'number', format: 'float', example: 98.29 } do |pipeline| pipeline.present.coverage end expose :detailed_status, using: DetailedStatusEntity do |pipeline, options| diff --git a/lib/api/entities/ci/pipeline_basic.rb b/lib/api/entities/ci/pipeline_basic.rb index a2a5a98920a..6d82cca1bf1 100644 --- a/lib/api/entities/ci/pipeline_basic.rb +++ b/lib/api/entities/ci/pipeline_basic.rb @@ -4,10 +4,21 @@ module API module Entities module Ci class PipelineBasic < Grape::Entity - expose :id, :iid, :project_id, :sha, :ref, :status, :source - expose :created_at, :updated_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :iid, documentation: { type: 'integer', example: 2 } + expose :project_id, documentation: { type: 'integer', example: 3 } + expose :sha, documentation: { type: 'string', example: '0ec9e58fdfca6cdd6652c083c9edb53abc0bad52' } + expose :ref, documentation: { type: 'string', example: 'feature-branch' } + expose :status, documentation: { type: 'string', example: 'success' } + expose :source, documentation: { type: 'string', example: 'push' } + expose :created_at, documentation: { type: 'dateTime', example: '2022-10-21T16:49:48.000+02:00' } + expose :updated_at, documentation: { type: 'dateTime', example: '2022-10-21T16:49:48.000+02:00' } - expose :web_url do |pipeline, _options| + expose :web_url, + documentation: { + type: 'string', + example: 'https://gitlab.example.com/gitlab-org/gitlab-foss/-/pipelines/61' + } do |pipeline, _options| Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline) end end diff --git a/lib/api/entities/ci/pipeline_schedule.rb b/lib/api/entities/ci/pipeline_schedule.rb index f1596b7d285..58496bded03 100644 --- a/lib/api/entities/ci/pipeline_schedule.rb +++ b/lib/api/entities/ci/pipeline_schedule.rb @@ -4,9 +4,15 @@ module API module Entities module Ci class PipelineSchedule < Grape::Entity - expose :id - expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active - expose :created_at, :updated_at + expose :id, documentation: { type: 'integer', example: 13 } + expose :description, documentation: { type: 'string', example: 'Test schedule pipeline' } + expose :ref, documentation: { type: 'string', example: 'develop' } + expose :cron, documentation: { type: 'string', example: '* * * * *' } + expose :cron_timezone, documentation: { type: 'string', example: 'Asia/Tokyo' } + expose :next_run_at, documentation: { type: 'dateTime', example: '2017-05-19T13:41:00.000Z' } + expose :active, documentation: { type: 'boolean', example: true } + expose :created_at, documentation: { type: 'dateTime', example: '2017-05-19T13:31:08.849Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2017-05-19T13:40:17.727Z' } expose :owner, using: ::API::Entities::UserBasic end end diff --git a/lib/api/entities/ci/resource_group.rb b/lib/api/entities/ci/resource_group.rb index 0afadfa9e2a..c14e32d32b1 100644 --- a/lib/api/entities/ci/resource_group.rb +++ b/lib/api/entities/ci/resource_group.rb @@ -4,7 +4,11 @@ module API module Entities module Ci class ResourceGroup < Grape::Entity - expose :id, :key, :process_mode, :created_at, :updated_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :key, documentation: { type: 'string', example: 'production' } + expose :process_mode, documentation: { type: 'string', example: 'unordered' } + expose :created_at, documentation: { type: 'dateTime', example: '2021-09-01T08:04:59.650Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2021-09-01T08:04:59.650Z' } end end end diff --git a/lib/api/entities/ci/runner.rb b/lib/api/entities/ci/runner.rb index f034eb5c94c..9361709b6ed 100644 --- a/lib/api/entities/ci/runner.rb +++ b/lib/api/entities/ci/runner.rb @@ -4,20 +4,22 @@ module API module Entities module Ci class Runner < Grape::Entity - expose :id - expose :description - expose :ip_address - expose :active # TODO Remove in v5 in favor of `paused` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/375709 - expose :paused do |runner| + expose :id, documentation: { type: 'integer', example: 8 } + expose :description, documentation: { type: 'string', example: 'test-1-20150125' } + expose :ip_address, documentation: { type: 'string', example: '127.0.0.1' } + # TODO Remove in v5 in favor of `paused` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/375709 + expose :active, documentation: { type: 'boolean', example: true } + expose :paused, documentation: { type: 'boolean', example: false } do |runner| !runner.active end - expose :instance_type?, as: :is_shared - expose :runner_type - expose :name - expose :online?, as: :online + expose :instance_type?, as: :is_shared, documentation: { type: 'boolean', example: true } + expose :runner_type, + documentation: { type: 'string', values: ::Ci::Runner.runner_types.keys, example: 'instance_type' } + expose :name, documentation: { type: 'string', example: 'test' } + expose :online?, as: :online, documentation: { type: 'boolean', example: true } # DEPRECATED # TODO Remove in v5 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/375709 - expose :deprecated_rest_status, as: :status + expose :deprecated_rest_status, as: :status, documentation: { type: 'string', example: 'online' } end end end diff --git a/lib/api/entities/ci/secure_file.rb b/lib/api/entities/ci/secure_file.rb index 639615e5779..d957e4488fd 100644 --- a/lib/api/entities/ci/secure_file.rb +++ b/lib/api/entities/ci/secure_file.rb @@ -9,6 +9,8 @@ module API expose :checksum expose :checksum_algorithm expose :created_at + expose :expires_at + expose :metadata end end end diff --git a/lib/api/entities/ci/variable.rb b/lib/api/entities/ci/variable.rb index f4d5248245a..47597cb77be 100644 --- a/lib/api/entities/ci/variable.rb +++ b/lib/api/entities/ci/variable.rb @@ -4,10 +4,16 @@ module API module Entities module Ci class Variable < Grape::Entity - expose :variable_type, :key, :value - expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } - expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) } - expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) } + expose :variable_type, documentation: { type: 'string', example: 'env_var' } + expose :key, documentation: { type: 'string', example: 'TEST_VARIABLE_1' } + expose :value, documentation: { type: 'string', example: 'TEST_1' } + expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) }, + documentation: { type: 'boolean' } + expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) }, + documentation: { type: 'boolean' } + expose :raw?, as: :raw, if: -> (entity, _) { entity.respond_to?(:raw?) }, documentation: { type: 'boolean' } + expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) }, + documentation: { type: 'string', example: '*' } end end end diff --git a/lib/api/entities/commit.rb b/lib/api/entities/commit.rb index 6cd180cd584..ab1f51289d7 100644 --- a/lib/api/entities/commit.rb +++ b/lib/api/entities/commit.rb @@ -3,15 +3,26 @@ module API module Entities class Commit < Grape::Entity - expose :id, :short_id, :created_at - expose :parent_ids - expose :full_title, as: :title - expose :safe_message, as: :message - expose :author_name, :author_email, :authored_date - expose :committer_name, :committer_email, :committed_date - expose :trailers + expose :id, documentation: { type: 'string', example: '2695effb5807a22ff3d138d593fd856244e155e7' } + expose :short_id, documentation: { type: 'string', example: '2695effb' } + expose :created_at, documentation: { type: 'dateTime', example: '2017-07-26T11:08:53.000+02:00' } + expose :parent_ids, + documentation: { type: 'string', is_array: true, example: '2a4b78934375d7f53875269ffd4f45fd83a84ebe' } + expose :full_title, as: :title, documentation: { type: 'string', example: 'Initial commit' } + expose :safe_message, as: :message, documentation: { type: 'string', example: 'Initial commit' } + expose :author_name, documentation: { type: 'string', example: 'John Smith' } + expose :author_email, documentation: { type: 'string', example: 'john@example.com' } + expose :authored_date, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :committer_name, documentation: { type: 'string', example: 'Jack Smith' } + expose :committer_email, documentation: { type: 'string', example: 'jack@example.com' } + expose :committed_date, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :trailers, documentation: { type: 'object', example: '{ "Merged-By": "Jane Doe janedoe@gitlab.com" }' } - expose :web_url do |commit, _options| + expose :web_url, + documentation: { + type: 'string', + example: 'https://gitlab.example.com/janedoe/gitlab-foss/-/commit/ed899a2f4b50b4370feeea94676502b42383c746' + } do |commit, _options| c = commit c = c.__subject__ if c.is_a?(Gitlab::View::Presenter::Base) Gitlab::UrlBuilder.build(c) diff --git a/lib/api/entities/commit_detail.rb b/lib/api/entities/commit_detail.rb index cc529639359..428c53f7fe3 100644 --- a/lib/api/entities/commit_detail.rb +++ b/lib/api/entities/commit_detail.rb @@ -6,10 +6,10 @@ module API include ::API::Helpers::Presentable expose :stats, using: Entities::CommitStats, if: :include_stats - expose :status_for, as: :status - expose :project_id + expose :status_for, as: :status, documentation: { type: 'string', example: 'success' } + expose :project_id, documentation: { type: 'integer', example: 1 } - expose :last_pipeline do |commit, options| + expose :last_pipeline, documentation: { type: ::API::Entities::Ci::PipelineBasic.to_s } do |commit, options| pipeline = commit.last_pipeline if can_read_pipeline? ::API::Entities::Ci::PipelineBasic.represent(pipeline, options) end diff --git a/lib/api/entities/commit_note.rb b/lib/api/entities/commit_note.rb index fe91712b48d..0632dc467b8 100644 --- a/lib/api/entities/commit_note.rb +++ b/lib/api/entities/commit_note.rb @@ -3,12 +3,22 @@ module API module Entities class CommitNote < Grape::Entity - expose :note - expose(:path) { |note| note.diff_file.try(:file_path) if note.diff_note? } - expose(:line) { |note| note.diff_line.try(:line) if note.diff_note? } - expose(:line_type) { |note| note.diff_line.try(:type) if note.diff_note? } + expose :note, documentation: { type: 'string', example: 'this doc is really nice' } + + expose :path, documentation: { type: 'string', example: 'README.md' } do |note| + note.diff_file.try(:file_path) if note.diff_note? + end + + expose :line, documentation: { type: 'integer', example: 11 } do |note| + note.diff_line.try(:line) if note.diff_note? + end + + expose :line_type, documentation: { type: 'string', example: 'new' } do |note| + note.diff_line.try(:type) if note.diff_note? + end + expose :author, using: Entities::UserBasic - expose :created_at + expose :created_at, documentation: { type: 'dateTime', example: '2016-01-19T09:44:55.600Z' } end end end diff --git a/lib/api/entities/commit_signature.rb b/lib/api/entities/commit_signature.rb index 0d8e977a9f5..9430dd5e2a2 100644 --- a/lib/api/entities/commit_signature.rb +++ b/lib/api/entities/commit_signature.rb @@ -3,7 +3,7 @@ module API module Entities class CommitSignature < Grape::Entity - expose :signature_type + expose :signature_type, documentation: { type: 'string', example: 'PGP' } expose :signature, merge: true do |commit, options| if commit.signature.is_a?(::CommitSignatures::GpgSignature) || commit.raw_commit_from_rugged? @@ -13,7 +13,7 @@ module API end end - expose :commit_source do |commit, _| + expose :commit_source, documentation: { type: 'string', example: 'gitaly' } do |commit, _| commit.raw_commit_from_rugged? ? "rugged" : "gitaly" end diff --git a/lib/api/entities/commit_stats.rb b/lib/api/entities/commit_stats.rb index d9ba99c8eb0..e07483e5d97 100644 --- a/lib/api/entities/commit_stats.rb +++ b/lib/api/entities/commit_stats.rb @@ -3,7 +3,9 @@ module API module Entities class CommitStats < Grape::Entity - expose :additions, :deletions, :total + expose :additions, documentation: { type: 'integer', example: 1 } + expose :deletions, documentation: { type: 'integer', example: 0 } + expose :total, documentation: { type: 'integer', example: 1 } end end end diff --git a/lib/api/entities/commit_status.rb b/lib/api/entities/commit_status.rb index 61b8bf89cfe..df6a41895ff 100644 --- a/lib/api/entities/commit_status.rb +++ b/lib/api/entities/commit_status.rb @@ -3,8 +3,22 @@ module API module Entities class CommitStatus < Grape::Entity - expose :id, :sha, :ref, :status, :name, :target_url, :description, - :created_at, :started_at, :finished_at, :allow_failure, :coverage + expose :id, documentation: { type: 'integer', example: 93 } + expose :sha, documentation: { type: 'string', example: '18f3e63d05582537db6d183d9d557be09e1f90c8' } + expose :ref, documentation: { type: 'string', example: 'develop' } + expose :status, documentation: { type: 'string', example: 'success' } + expose :name, documentation: { type: 'string', example: 'default' } + expose :target_url, documentation: { + type: 'string', + example: 'https://gitlab.example.com/janedoe/gitlab-foss/builds/91' + } + expose :description, documentation: { type: 'string' } + expose :created_at, documentation: { type: 'dateTime', example: '2016-01-19T09:05:50.355Z' } + expose :started_at, documentation: { type: 'dateTime', example: '2016-01-20T08:40:25.832Z' } + expose :finished_at, documentation: { type: 'dateTime', example: '2016-01-21T08:40:25.832Z' } + expose :allow_failure, documentation: { type: 'boolean', example: false } + expose :coverage, documentation: { type: 'number', format: 'float', example: 98.29 } + expose :author, using: Entities::UserBasic end end diff --git a/lib/api/entities/compare.rb b/lib/api/entities/compare.rb index 75a36d9bb01..92066868d3c 100644 --- a/lib/api/entities/compare.rb +++ b/lib/api/entities/compare.rb @@ -7,21 +7,24 @@ module API compare.commits.last end - expose :commits, using: Entities::Commit do |compare, _| + expose :commits, documentation: { is_array: true }, using: Entities::Commit do |compare, _| compare.commits end - expose :diffs, using: Entities::Diff do |compare, _| + expose :diffs, documentation: { is_array: true }, using: Entities::Diff do |compare, _| compare.diffs.diffs.to_a end - expose :compare_timeout do |compare, _| + expose :compare_timeout, documentation: { type: 'boolean' } do |compare, _| compare.diffs.diffs.overflow? end - expose :same, as: :compare_same_ref + expose :same, as: :compare_same_ref, documentation: { type: 'boolean' } - expose :web_url do |compare, _| + expose :web_url, + documentation: { + example: "https://gitlab.example.com/gitlab/gitlab-foss/-/compare/main...feature" + } do |compare, _| Gitlab::UrlBuilder.build(compare) end end diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index 2fdfac40c32..d12c8142e69 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -12,13 +12,13 @@ module API class Repository < Grape::Entity include ::API::Helpers::RelatedResourcesHelpers - expose :id - expose :name - expose :path - expose :project_id - expose :location - expose :created_at - expose :expiration_policy_started_at, as: :cleanup_policy_started_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :name, documentation: { type: 'string', example: 'releases' } + expose :path, documentation: { type: 'string', example: 'group/project/releases' } + expose :project_id, documentation: { type: 'integer', example: 9 } + expose :location, documentation: { type: 'string', example: 'gitlab.example.com/group/project/releases' } + expose :created_at, documentation: { type: 'dateTime', example: '2019-01-10T13:39:08.229Z' } + expose :expiration_policy_started_at, as: :cleanup_policy_started_at, documentation: { type: 'dateTime', example: '2020-08-17T03:12:35.489Z' } expose :tags_count, if: -> (_, options) { options[:tags_count] } expose :tags, using: Tag, if: -> (_, options) { options[:tags] } expose :delete_api_path, if: ->(object, options) { Ability.allowed?(options[:user], :admin_container_image, object) } diff --git a/lib/api/entities/contributor.rb b/lib/api/entities/contributor.rb index 8763822b674..4fab953f0f6 100644 --- a/lib/api/entities/contributor.rb +++ b/lib/api/entities/contributor.rb @@ -3,7 +3,11 @@ module API module Entities class Contributor < Grape::Entity - expose :name, :email, :commits, :additions, :deletions + expose :name, documentation: { example: 'John Doe' } + expose :email, documentation: { example: 'johndoe@example.com' } + expose :commits, documentation: { type: 'integer', example: 117 } + expose :additions, documentation: { type: 'integer', example: 3 } + expose :deletions, documentation: { type: 'integer', example: 5 } end end end diff --git a/lib/api/entities/custom_attribute.rb b/lib/api/entities/custom_attribute.rb index f949b709517..883b572ac75 100644 --- a/lib/api/entities/custom_attribute.rb +++ b/lib/api/entities/custom_attribute.rb @@ -3,8 +3,8 @@ module API module Entities class CustomAttribute < Grape::Entity - expose :key - expose :value + expose :key, documentation: { type: 'string', example: 'foo' } + expose :value, documentation: { type: 'string', example: 'bar' } end end end diff --git a/lib/api/entities/deploy_key.rb b/lib/api/entities/deploy_key.rb index 2c9c33549a1..1bcd06f2c88 100644 --- a/lib/api/entities/deploy_key.rb +++ b/lib/api/entities/deploy_key.rb @@ -3,9 +3,15 @@ module API module Entities class DeployKey < Entities::SSHKey - expose :key - expose :fingerprint, if: ->(key, _) { key.fingerprint.present? } - expose :fingerprint_sha256 + expose :key, + documentation: { type: 'string', example: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDNJAkI3Wdf0r13c8a5pEExB2YowPWCSVzfZV22pNBc1CuEbyYLHpUyaD0GwpGvFdx2aP7lMEk35k6Rz3ccBF6jRaVJyhsn5VNnW92PMpBJ/P1UebhXwsFHdQf5rTt082cSxWuk61kGWRQtk4ozt/J2DF/dIUVaLvc+z4HomT41fQ==' } + + expose :fingerprint, + documentation: { type: 'string', example: '4a:9d:64:15:ed:3a:e6:07:6e:89:36:b3:3b:03:05:d9' }, + if: ->(key, _) { key.fingerprint.present? } + + expose :fingerprint_sha256, + documentation: { type: 'string', example: 'SHA256:Jrs3LD1Ji30xNLtTVf9NDCj7kkBgPBb2pjvTZ3HfIgU' } expose :projects_with_write_access, using: Entities::ProjectIdentity, if: -> (_, options) { options[:include_projects_with_write_access] } end diff --git a/lib/api/entities/deploy_keys_project.rb b/lib/api/entities/deploy_keys_project.rb index 12a86fbdf8e..4501af88067 100644 --- a/lib/api/entities/deploy_keys_project.rb +++ b/lib/api/entities/deploy_keys_project.rb @@ -4,7 +4,7 @@ module API module Entities class DeployKeysProject < Grape::Entity expose :deploy_key, merge: true, using: Entities::DeployKey - expose :can_push + expose :can_push, documentation: { type: 'boolean' } end end end diff --git a/lib/api/entities/deploy_token.rb b/lib/api/entities/deploy_token.rb index daee104ba6b..9861467e35d 100644 --- a/lib/api/entities/deploy_token.rb +++ b/lib/api/entities/deploy_token.rb @@ -4,8 +4,13 @@ module API module Entities class DeployToken < Grape::Entity # exposing :token is a security risk and should be avoided - expose :id, :name, :username, :expires_at, :scopes, :revoked - expose :expired?, as: :expired + expose :id, documentation: { type: 'integer', example: 1 } + expose :name, documentation: { type: 'string', example: 'MyToken' } + expose :username, documentation: { type: 'string', example: 'gitlab+deploy-token-1' } + expose :expires_at, documentation: { type: 'dateTime', example: '2020-02-14T00:00:00.000Z' } + expose :scopes, documentation: { type: 'array', example: ['read_repository'] } + expose :revoked, documentation: { type: 'boolean' } + expose :expired?, documentation: { type: 'boolean' }, as: :expired end end end diff --git a/lib/api/entities/deploy_token_with_token.rb b/lib/api/entities/deploy_token_with_token.rb index 11efe3720fa..a96051e1403 100644 --- a/lib/api/entities/deploy_token_with_token.rb +++ b/lib/api/entities/deploy_token_with_token.rb @@ -3,7 +3,7 @@ module API module Entities class DeployTokenWithToken < Entities::DeployToken - expose :token + expose :token, documentation: { type: 'string', example: 'jMRvtPNxrn3crTAGukpZ' } end end end diff --git a/lib/api/entities/deployment.rb b/lib/api/entities/deployment.rb index 4e3a4c289d9..426e92e7723 100644 --- a/lib/api/entities/deployment.rb +++ b/lib/api/entities/deployment.rb @@ -3,11 +3,16 @@ module API module Entities class Deployment < Grape::Entity - expose :id, :iid, :ref, :sha, :created_at, :updated_at + expose :id, documentation: { type: 'integer', example: 41 } + expose :iid, documentation: { type: 'integer', example: 1 } + expose :ref, documentation: { type: 'string', example: 'main' } + expose :sha, documentation: { type: 'string', example: '99d03678b90d914dbb1b109132516d71a4a03ea8' } + expose :created_at, documentation: { type: 'dateTime', example: '2016-08-11T11:32:35.444Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2016-08-11T11:32:35.444Z' } expose :user, using: Entities::UserBasic expose :environment, using: Entities::EnvironmentBasic expose :deployable, using: Entities::Ci::Job - expose :status + expose :status, documentation: { type: 'string', example: 'created' } end end end diff --git a/lib/api/entities/diff.rb b/lib/api/entities/diff.rb index e92bc5d6b68..e9650f07f00 100644 --- a/lib/api/entities/diff.rb +++ b/lib/api/entities/diff.rb @@ -3,11 +3,17 @@ module API module Entities class Diff < Grape::Entity - expose :old_path, :new_path, :a_mode, :b_mode - expose :new_file?, as: :new_file - expose :renamed_file?, as: :renamed_file - expose :deleted_file?, as: :deleted_file - expose :json_safe_diff, as: :diff + expose :json_safe_diff, as: :diff, documentation: { + type: 'string', + example: '--- a/doc/update/5.4-to-6.0.md\n+++ b/doc/update/5.4-to-6.0.md\n@@ -71,6 +71,8 @@\n...' + } + expose :new_path, documentation: { type: 'string', example: 'doc/update/5.4-to-6.0.md' } + expose :old_path, documentation: { type: 'string', example: 'doc/update/5.4-to-6.0.md' } + expose :a_mode, documentation: { type: 'string', example: '100755' } + expose :b_mode, documentation: { type: 'string', example: '100644' } + expose :new_file?, as: :new_file, documentation: { type: 'boolean' } + expose :renamed_file?, as: :renamed_file, documentation: { type: 'boolean' } + expose :deleted_file?, as: :deleted_file, documentation: { type: 'boolean' } end end end diff --git a/lib/api/entities/entity_helpers.rb b/lib/api/entities/entity_helpers.rb index 3a68044ad35..6fb04bb8ad6 100644 --- a/lib/api/entities/entity_helpers.rb +++ b/lib/api/entities/entity_helpers.rb @@ -11,8 +11,8 @@ module API ->(obj, opts) { Ability.allowed?(opts[:user], "destroy_#{attr}".to_sym, yield(obj)) } end - def expose_restricted(attr, &block) - expose attr, if: can_read(attr, &block) + def expose_restricted(attr, documentation: {}, &block) + expose attr, documentation: documentation, if: can_read(attr, &block) end end end diff --git a/lib/api/entities/environment.rb b/lib/api/entities/environment.rb index 3b6ed94c3f1..dc9911d5acb 100644 --- a/lib/api/entities/environment.rb +++ b/lib/api/entities/environment.rb @@ -5,10 +5,10 @@ module API class Environment < Entities::EnvironmentBasic include RequestAwareEntity - expose :tier + expose :tier, documentation: { type: 'string', example: 'development' } expose :project, using: Entities::BasicProjectDetails expose :last_deployment, using: Entities::Deployment, if: { last_deployment: true } - expose :state + expose :state, documentation: { type: 'string', example: 'available' } end end end diff --git a/lib/api/entities/environment_basic.rb b/lib/api/entities/environment_basic.rb index d9894eac147..1b4a9371820 100644 --- a/lib/api/entities/environment_basic.rb +++ b/lib/api/entities/environment_basic.rb @@ -3,7 +3,12 @@ module API module Entities class EnvironmentBasic < Grape::Entity - expose :id, :name, :slug, :external_url, :created_at, :updated_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :name, documentation: { type: 'string', example: 'deploy' } + expose :slug, documentation: { type: 'string', example: 'deploy' } + expose :external_url, documentation: { type: 'string', example: 'https://deploy.gitlab.example.com' } + expose :created_at, documentation: { type: 'dateTime', example: '2019-05-25T18:55:13.252Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2019-05-25T18:55:13.252Z' } end end end diff --git a/lib/api/entities/error_tracking.rb b/lib/api/entities/error_tracking.rb index 163bda92680..5e3b983c58c 100644 --- a/lib/api/entities/error_tracking.rb +++ b/lib/api/entities/error_tracking.rb @@ -4,11 +4,11 @@ module API module Entities module ErrorTracking class ProjectSetting < Grape::Entity - expose :enabled, as: :active - expose :project_name - expose :sentry_external_url - expose :api_url - expose :integrated + expose :enabled, as: :active, documentation: { type: 'boolean' } + expose :project_name, documentation: { type: 'string', example: 'sample sentry project' } + expose :sentry_external_url, documentation: { type: 'string', example: 'https://sentry.io/myawesomeproject/project' } + expose :api_url, documentation: { type: 'string', example: 'https://sentry.io/api/0/projects/myawesomeproject/project' } + expose :integrated, documentation: { type: 'boolean' } def integrated return false unless ::Feature.enabled?(:integrated_error_tracking, object.project) @@ -18,10 +18,10 @@ module API end class ClientKey < Grape::Entity - expose :id - expose :active - expose :public_key - expose :sentry_dsn + expose :id, documentation: { type: 'integer', example: 1 } + expose :active, documentation: { type: 'boolean' } + expose :public_key, documentation: { type: 'string', example: 'glet_aa77551d849c083f76d0bc545ed053a3' } + expose :sentry_dsn, documentation: { type: 'string', example: 'https://glet_aa77551d849c083f76d0bc545ed053a3@gitlab.example.com/api/v4/error_tracking/collector/5' } end end end diff --git a/lib/api/entities/feature.rb b/lib/api/entities/feature.rb index d1151849cd7..48dd5a22a7e 100644 --- a/lib/api/entities/feature.rb +++ b/lib/api/entities/feature.rb @@ -3,8 +3,8 @@ module API module Entities class Feature < Grape::Entity - expose :name - expose :state + expose :name, documentation: { type: 'string', example: 'experimental_feature' } + expose :state, documentation: { type: 'string', example: 'off' } expose :gates, using: Entities::FeatureGate do |model| model.gates.map do |gate| value = model.gate_values[gate.key] diff --git a/lib/api/entities/feature_flag.rb b/lib/api/entities/feature_flag.rb index 9dec3873504..273307357a2 100644 --- a/lib/api/entities/feature_flag.rb +++ b/lib/api/entities/feature_flag.rb @@ -3,12 +3,12 @@ module API module Entities class FeatureFlag < Grape::Entity - expose :name - expose :description - expose :active - expose :version - expose :created_at - expose :updated_at + expose :name, documentation: { type: 'string', example: 'merge_train' } + expose :description, documentation: { type: 'string', example: 'merge train feature flag' } + expose :active, documentation: { type: 'boolean' } + expose :version, documentation: { type: 'string', example: 'new_version_flag' } + expose :created_at, documentation: { type: 'dateTime', example: '2019-11-04T08:13:51.423Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2019-11-04T08:13:51.423Z' } expose :scopes do |_ff| [] end diff --git a/lib/api/entities/feature_flag/scope.rb b/lib/api/entities/feature_flag/scope.rb index 906fe718257..e29793c250a 100644 --- a/lib/api/entities/feature_flag/scope.rb +++ b/lib/api/entities/feature_flag/scope.rb @@ -4,8 +4,8 @@ module API module Entities class FeatureFlag < Grape::Entity class Scope < Grape::Entity - expose :id - expose :environment_scope + expose :id, documentation: { type: 'integer', example: 1 } + expose :environment_scope, documentation: { type: 'string', example: 'production' } end end end diff --git a/lib/api/entities/feature_flag/strategy.rb b/lib/api/entities/feature_flag/strategy.rb index 32699be0ee3..62178420370 100644 --- a/lib/api/entities/feature_flag/strategy.rb +++ b/lib/api/entities/feature_flag/strategy.rb @@ -4,9 +4,9 @@ module API module Entities class FeatureFlag < Grape::Entity class Strategy < Grape::Entity - expose :id - expose :name - expose :parameters + expose :id, documentation: { type: 'integer', example: 1 } + expose :name, documentation: { type: 'string', example: 'userWithId' } + expose :parameters, documentation: { type: 'string', example: '{"userIds": "user1"}' } expose :scopes, using: FeatureFlag::Scope end end diff --git a/lib/api/entities/feature_flag/user_list.rb b/lib/api/entities/feature_flag/user_list.rb index bc8b12ea22e..efb3261658a 100644 --- a/lib/api/entities/feature_flag/user_list.rb +++ b/lib/api/entities/feature_flag/user_list.rb @@ -6,13 +6,13 @@ module API class UserList < Grape::Entity include RequestAwareEntity - expose :id - expose :iid - expose :project_id - expose :created_at - expose :updated_at - expose :name - expose :user_xids + expose :id, documentation: { type: 'integer', example: 1 } + expose :iid, documentation: { type: 'integer', example: 1 } + expose :project_id, documentation: { type: 'integer', example: 2 } + expose :created_at, documentation: { type: 'dateTime', example: '2020-02-04T08:13:10.507Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2020-02-04T08:13:10.507Z' } + expose :name, documentation: { type: 'string', example: 'user_list' } + expose :user_xids, documentation: { type: 'string', example: 'user1,user2' } expose :path do |list| project_feature_flags_user_list_path(list.project, list) diff --git a/lib/api/entities/feature_gate.rb b/lib/api/entities/feature_gate.rb index bea9c9474b3..ad4702bf210 100644 --- a/lib/api/entities/feature_gate.rb +++ b/lib/api/entities/feature_gate.rb @@ -3,8 +3,8 @@ module API module Entities class FeatureGate < Grape::Entity - expose :key - expose :value + expose :key, documentation: { type: 'string', example: 'percentage_of_actors' } + expose :value, documentation: { type: 'integer', example: 34 } end end end diff --git a/lib/api/entities/freeze_period.rb b/lib/api/entities/freeze_period.rb index 9b5f08925db..d6853c544a5 100644 --- a/lib/api/entities/freeze_period.rb +++ b/lib/api/entities/freeze_period.rb @@ -3,9 +3,11 @@ module API module Entities class FreezePeriod < Grape::Entity - expose :id - expose :freeze_start, :freeze_end, :cron_timezone - expose :created_at, :updated_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :freeze_start, documentation: { type: 'string', example: '0 23 * * 5' } + expose :freeze_end, documentation: { type: 'string', example: '0 8 * * 1' } + expose :cron_timezone, documentation: { type: 'string', example: 'UTC' } + expose :created_at, :updated_at, documentation: { type: 'dateTime', example: '2020-05-15T17:03:35.702Z' } end end end diff --git a/lib/api/entities/go_module_version.rb b/lib/api/entities/go_module_version.rb index 643e25df9e0..b9dd88982dd 100644 --- a/lib/api/entities/go_module_version.rb +++ b/lib/api/entities/go_module_version.rb @@ -3,8 +3,8 @@ module API module Entities class GoModuleVersion < Grape::Entity - expose :name, as: 'Version' - expose :time, as: 'Time' + expose :name, as: 'Version', documentation: { type: 'string', example: 'v1.0.0' } + expose :time, as: 'Time', documentation: { type: 'string', example: '1617822312 -0600' } end end end diff --git a/lib/api/entities/hook.rb b/lib/api/entities/hook.rb index 95924321221..e24e201ac57 100644 --- a/lib/api/entities/hook.rb +++ b/lib/api/entities/hook.rb @@ -3,12 +3,18 @@ module API module Entities class Hook < Grape::Entity - expose :id, :url, :created_at, :push_events, :tag_push_events, :merge_requests_events, :repository_update_events - expose :enable_ssl_verification + expose :id, documentation: { type: 'string', example: 1 } + expose :url, documentation: { type: 'string', example: 'https://webhook.site' } + expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :push_events, documentation: { type: 'boolean' } + expose :tag_push_events, documentation: { type: 'boolean' } + expose :merge_requests_events, documentation: { type: 'boolean' } + expose :repository_update_events, documentation: { type: 'boolean' } + expose :enable_ssl_verification, documentation: { type: 'boolean' } - expose :alert_status - expose :disabled_until - expose :url_variables + expose :alert_status, documentation: { type: 'symbol', example: :executable } + expose :disabled_until, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :url_variables, documentation: { type: 'Hash', example: { "token" => "secr3t" }, is_array: true } def url_variables object.url_variables.keys.map { { key: _1 } } diff --git a/lib/api/entities/issuable_entity.rb b/lib/api/entities/issuable_entity.rb index e2c674c0b8b..4e70f945a48 100644 --- a/lib/api/entities/issuable_entity.rb +++ b/lib/api/entities/issuable_entity.rb @@ -3,10 +3,16 @@ module API module Entities class IssuableEntity < Grape::Entity - expose :id, :iid - expose(:project_id) { |entity| entity&.project.try(:id) } - expose :title, :description - expose :state, :created_at, :updated_at + expose :id, documentation: { type: 'integer', example: 84 } + expose :iid, documentation: { type: 'integer', example: 14 } + expose :project_id, documentation: { type: 'integer', example: 4 } do |entity| + entity&.project.try(:id) + end + expose :title, documentation: { type: 'string', example: 'Impedit et ut et dolores vero provident ullam est' } + expose :description, documentation: { type: 'string', example: 'Repellendus impedit et vel velit dignissimos.' } + expose :state, documentation: { type: 'string', example: 'closed' } + expose :created_at, documentation: { type: 'dateTime', example: '2022-08-17T12:46:35.053Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2022-11-14T17:22:01.470Z' } def presented lazy_issuable_metadata diff --git a/lib/api/entities/issue_basic.rb b/lib/api/entities/issue_basic.rb index 20f66c026e6..89fb8bbe1c0 100644 --- a/lib/api/entities/issue_basic.rb +++ b/lib/api/entities/issue_basic.rb @@ -7,10 +7,10 @@ module API item.upcase if item.respond_to?(:upcase) end - expose :closed_at + expose :closed_at, documentation: { type: 'dateTime', example: '2022-11-15T08:30:55.232Z' } expose :closed_by, using: Entities::UserBasic - expose :labels do |issue, options| + expose :labels, documentation: { type: 'string', is_array: true, example: 'bug' } do |issue, options| if options[:with_labels_details] ::API::Entities::LabelBasic.represent(issue.labels.sort_by(&:title)) else @@ -23,7 +23,7 @@ module API expose :issue_type, as: :type, format_with: :upcase, - documentation: { type: "String", desc: "One of #{::WorkItems::Type.allowed_types_for_issues.map(&:upcase)}" } + documentation: { type: 'String', example: 'ISSUE', desc: "One of #{::WorkItems::Type.allowed_types_for_issues.map(&:upcase)}" } expose :assignee, using: ::API::Entities::UserBasic do |issue| issue.assignees.first @@ -33,12 +33,12 @@ module API expose(:merge_requests_count) { |issue, options| issuable_metadata.merge_requests_count } expose(:upvotes) { |issue, options| issuable_metadata.upvotes } expose(:downvotes) { |issue, options| issuable_metadata.downvotes } - expose :due_date - expose :confidential - expose :discussion_locked - expose :issue_type + expose :due_date, documentation: { type: 'date', example: '2022-11-20' } + expose :confidential, documentation: { type: 'boolean' } + expose :discussion_locked, documentation: { type: 'boolean' } + expose :issue_type, documentation: { type: 'string', example: 'issue' } - expose :web_url do |issue| + expose :web_url, documentation: { type: 'string', example: 'http://example.com/example/example/issues/14' } do |issue| Gitlab::UrlBuilder.build(issue) end diff --git a/lib/api/entities/license.rb b/lib/api/entities/license.rb index 8ecf8a430fe..6318fec6774 100644 --- a/lib/api/entities/license.rb +++ b/lib/api/entities/license.rb @@ -4,12 +4,25 @@ module API module Entities # Serializes a Licensee::License class License < Entities::LicenseBasic - expose :popular?, as: :popular - expose(:description) { |license| license.meta['description'] } - expose(:conditions) { |license| license.meta['conditions'] } - expose(:permissions) { |license| license.meta['permissions'] } - expose(:limitations) { |license| license.meta['limitations'] } - expose :content + expose :popular?, as: :popular, documentation: { type: 'boolean' } + + expose :description, documentation: { type: 'string', example: 'A simple license' } do |license| + license.meta['description'] + end + + expose :conditions, documentation: { type: 'string', is_array: true, example: 'include-copyright' } do |license| + license.meta['conditions'] + end + + expose :permissions, documentation: { type: 'string', is_array: true, example: 'commercial-use' } do |license| + license.meta['permissions'] + end + + expose :limitations, documentation: { type: 'string', is_array: true, example: 'liability' } do |license| + license.meta['limitations'] + end + + expose :content, documentation: { type: 'string', example: 'GNU GENERAL PUBLIC LICENSE' } end end end diff --git a/lib/api/entities/license_basic.rb b/lib/api/entities/license_basic.rb index 0916738d21d..e3bb55d4104 100644 --- a/lib/api/entities/license_basic.rb +++ b/lib/api/entities/license_basic.rb @@ -4,8 +4,10 @@ module API module Entities # Serializes a Gitlab::Git::DeclaredLicense class LicenseBasic < Grape::Entity - expose :key, :name, :nickname - expose :url, as: :html_url + expose :key, documentation: { type: 'string', example: 'gpl-3.0' } + expose :name, documentation: { type: 'string', example: 'GNU General Public License v3.0' } + expose :nickname, documentation: { type: 'string', example: 'GNU GPLv3' } + expose :url, as: :html_url, documentation: { example: 'http://choosealicense.com/licenses/gpl-3.0' } # This was dropped: # https://github.com/github/choosealicense.com/commit/325806b42aa3d5b78e84120327ec877bc936dbdd#diff-66df8f1997786f7052d29010f2cbb4c66391d60d24ca624c356acc0ab986f139 diff --git a/lib/api/entities/markdown.rb b/lib/api/entities/markdown.rb new file mode 100644 index 00000000000..0fbaec4375e --- /dev/null +++ b/lib/api/entities/markdown.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class Markdown < Grape::Entity + expose :html, documentation: { type: 'string', example: '<p dir=\"auto\">Hello world!</p>"' } + end + end +end diff --git a/lib/api/entities/merge_request_approvals.rb b/lib/api/entities/merge_request_approvals.rb index 6810952b2fc..54f196e0d74 100644 --- a/lib/api/entities/merge_request_approvals.rb +++ b/lib/api/entities/merge_request_approvals.rb @@ -3,15 +3,15 @@ module API module Entities class MergeRequestApprovals < Grape::Entity - expose :user_has_approved do |merge_request, options| + expose :user_has_approved, documentation: { type: 'boolean' } do |merge_request, options| merge_request.approved_by?(options[:current_user]) end - expose :user_can_approve do |merge_request, options| + expose :user_can_approve, documentation: { type: 'boolean' } do |merge_request, options| merge_request.eligible_for_approval_by?(options[:current_user]) end - expose :approved do |merge_request| + expose :approved, documentation: { type: 'boolean' } do |merge_request| merge_request.approvals.present? end diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb index 55d58166590..27f6e6ade06 100644 --- a/lib/api/entities/merge_request_basic.rb +++ b/lib/api/entities/merge_request_basic.rb @@ -58,6 +58,7 @@ module API merge_request.check_mergeability(async: true) unless options[:skip_merge_status_recheck] merge_request.public_merge_status end + expose :detailed_merge_status expose :diff_head_sha, as: :sha expose :merge_commit_sha expose :squash_commit_sha @@ -93,6 +94,12 @@ module API expose :task_completion_status expose :cannot_be_merged?, as: :has_conflicts expose :mergeable_discussions_state?, as: :blocking_discussions_resolved + + private + + def detailed_merge_status + ::MergeRequests::Mergeability::DetailedMergeStatusService.new(merge_request: object).execute + end end end end diff --git a/lib/api/entities/merge_request_simple.rb b/lib/api/entities/merge_request_simple.rb index f3ff4cc18a8..d5c511ad9a4 100644 --- a/lib/api/entities/merge_request_simple.rb +++ b/lib/api/entities/merge_request_simple.rb @@ -3,8 +3,11 @@ module API module Entities class MergeRequestSimple < IssuableEntity - expose :title - expose :web_url do |merge_request, options| + expose :title, documentation: { type: 'string', example: 'Test MR 1580978354' } + expose :web_url, + documentation: { + type: 'string', example: 'http://local.gitlab.test:8181/root/merge-train-race-condition/-/merge_requests/59' + } do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) end end diff --git a/lib/api/entities/metadata.rb b/lib/api/entities/metadata.rb index daa491ec42a..7dfcad2ccab 100644 --- a/lib/api/entities/metadata.rb +++ b/lib/api/entities/metadata.rb @@ -3,13 +3,14 @@ module API module Entities class Metadata < Grape::Entity - expose :version - expose :revision + expose :version, documentation: { type: 'string', example: '15.2-pre' } + expose :revision, documentation: { type: 'string', example: 'c401a659d0c' } expose :kas do expose :enabled, documentation: { type: 'boolean' } - expose :externalUrl - expose :version + expose :externalUrl, documentation: { type: 'string', example: 'grpc://gitlab.example.com:8150' } + expose :version, documentation: { type: 'string', example: '15.0.0' } end + expose :enterprise, documentation: { type: 'boolean' } end end end diff --git a/lib/api/entities/metrics/dashboard/annotation.rb b/lib/api/entities/metrics/dashboard/annotation.rb index 66bd09d84f9..08d1a333259 100644 --- a/lib/api/entities/metrics/dashboard/annotation.rb +++ b/lib/api/entities/metrics/dashboard/annotation.rb @@ -5,13 +5,13 @@ module API module Metrics module Dashboard class Annotation < Grape::Entity - expose :id - expose :starting_at - expose :ending_at - expose :dashboard_path - expose :description - expose :environment_id - expose :cluster_id + expose :id, documentation: { type: 'integer', example: 4 } + expose :starting_at, documentation: { type: 'dateTime', example: '2016-04-08T03:45:40.000Z' } + expose :ending_at, documentation: { type: 'dateTime', example: '2016-08-08T09:00:00.000Z' } + expose :dashboard_path, documentation: { type: 'string', example: '.gitlab/dashboards/custom_metrics.yml' } + expose :description, documentation: { type: 'string', example: 'annotation description' } + expose :environment_id, documentation: { type: 'integer', example: 1 } + expose :cluster_id, documentation: { type: 'integer', example: 2 } end end end diff --git a/lib/api/entities/metrics/user_starred_dashboard.rb b/lib/api/entities/metrics/user_starred_dashboard.rb index d774160e3ea..1d2a8a39547 100644 --- a/lib/api/entities/metrics/user_starred_dashboard.rb +++ b/lib/api/entities/metrics/user_starred_dashboard.rb @@ -4,7 +4,10 @@ module API module Entities module Metrics class UserStarredDashboard < Grape::Entity - expose :id, :dashboard_path, :user_id, :project_id + expose :id, documentation: { type: 'integer', example: 5 } + expose :dashboard_path, documentation: { type: 'string', example: 'config/prometheus/common_metrics.yml' } + expose :user_id, documentation: { type: 'integer', example: 1 } + expose :project_id, documentation: { type: 'integer', example: 20 } end end end diff --git a/lib/api/entities/ml/mlflow/run.rb b/lib/api/entities/ml/mlflow/run.rb index a8e1cfe08dd..8b16c67611f 100644 --- a/lib/api/entities/ml/mlflow/run.rb +++ b/lib/api/entities/ml/mlflow/run.rb @@ -6,7 +6,7 @@ module API module Mlflow class Run < Grape::Entity expose :run do - expose(:info) { |candidate| RunInfo.represent(candidate) } + expose :itself, using: RunInfo, as: :info expose :data do expose :metrics, using: Metric expose :params, using: RunParam diff --git a/lib/api/entities/ml/mlflow/run_info.rb b/lib/api/entities/ml/mlflow/run_info.rb index 096950e349d..d3934545ba4 100644 --- a/lib/api/entities/ml/mlflow/run_info.rb +++ b/lib/api/entities/ml/mlflow/run_info.rb @@ -11,7 +11,7 @@ module API expose(:start_time) { |candidate| candidate.start_time || 0 } expose :end_time, expose_nil: false expose(:status) { |candidate| candidate.status.to_s.upcase } - expose(:artifact_uri) { |candidate| 'not_implemented' } + expose(:artifact_uri) { |candidate, options| "#{options[:packages_url]}#{candidate.artifact_root}" } expose(:lifecycle_stage) { |candidate| 'active' } expose(:user_id) { |candidate| candidate.user_id.to_s } diff --git a/lib/api/entities/ml/mlflow/update_run.rb b/lib/api/entities/ml/mlflow/update_run.rb index 090d69b8895..55def810ef5 100644 --- a/lib/api/entities/ml/mlflow/update_run.rb +++ b/lib/api/entities/ml/mlflow/update_run.rb @@ -5,13 +5,7 @@ module API module Ml module Mlflow class UpdateRun < Grape::Entity - expose :run_info - - private - - def run_info - RunInfo.represent object - end + expose :itself, using: RunInfo, as: :run_info end end end diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index 18fc0576dd4..c92a4677220 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -4,11 +4,12 @@ module API module Entities class Package < Grape::Entity include ::API::Helpers::RelatedResourcesHelpers + include ::Routing::PackagesHelper extend ::API::Entities::EntityHelpers - expose :id + expose :id, documentation: { type: 'integer', example: 1 } - expose :name do |package| + expose :name, documentation: { type: 'string', example: '@foo/bar' } do |package| if package.conan? package.conan_recipe else @@ -20,17 +21,13 @@ module API package.name end - expose :version - expose :package_type - expose :status + expose :version, documentation: { type: 'string', example: '1.0.3' } + expose :package_type, documentation: { type: 'string', example: 'npm' } + expose :status, documentation: { type: 'string', example: 'default' } expose :_links do - expose :web_path do |package, opts| - if package.infrastructure_package? - ::Gitlab::Routing.url_helpers.namespace_project_infrastructure_registry_path(opts[:namespace], package.project, package) - else - ::Gitlab::Routing.url_helpers.project_package_path(package.project, package) - end + expose :web_path do |package| + package_path(package) end expose :delete_api_path, if: can_destroy(:package, &:project) do |package| @@ -38,10 +35,12 @@ module API end end - expose :created_at - expose :last_downloaded_at - expose :project_id, if: ->(_, opts) { opts[:group] } - expose :project_path, if: ->(obj, opts) { opts[:group] && Ability.allowed?(opts[:user], :read_project, obj.project) } + expose :created_at, documentation: { type: 'dateTime', example: '2022-09-16T12:47:31.949Z' } + expose :last_downloaded_at, documentation: { type: 'dateTime', example: '2022-09-19T11:32:35.169Z' } + expose :project_id, documentation: { type: 'integer', example: 2 }, if: ->(_, opts) { opts[:group] } + expose :project_path, documentation: { type: 'string', example: 'gitlab/foo/bar' }, if: ->(obj, opts) do + opts[:group] && Ability.allowed?(opts[:user], :read_project, obj.project) + end expose :tags expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline diff --git a/lib/api/entities/package_file.rb b/lib/api/entities/package_file.rb index e34a6a7aa1d..19372b75012 100644 --- a/lib/api/entities/package_file.rb +++ b/lib/api/entities/package_file.rb @@ -3,9 +3,14 @@ module API module Entities class PackageFile < Grape::Entity - expose :id, :package_id, :created_at - expose :file_name, :size - expose :file_md5, :file_sha1, :file_sha256 + expose :id, documentation: { type: 'integer', example: 225 } + expose :package_id, documentation: { type: 'integer', example: 4 } + expose :created_at, documentation: { type: 'dateTime', example: '2018-11-07T15:25:52.199Z' } + expose :file_name, documentation: { type: 'string', example: 'my-app-1.5-20181107.152550-1.jar' } + expose :size, documentation: { type: 'integer', example: '2421' } + expose :file_md5, documentation: { type: 'string', example: '58e6a45a629910c6ff99145a688971ac' } + expose :file_sha1, documentation: { type: 'string', example: 'ebd193463d3915d7e22219f52740056dfd26cbfe' } + expose :file_sha256, documentation: { type: 'string', example: 'a903393463d3915d7e22219f52740056dfd26cbfeff321b' } expose :pipelines, if: ->(package_file) { package_file.pipelines.present? }, using: Package::Pipeline end end diff --git a/lib/api/entities/personal_access_token.rb b/lib/api/entities/personal_access_token.rb index 55764daef9d..3ec91ca5fc9 100644 --- a/lib/api/entities/personal_access_token.rb +++ b/lib/api/entities/personal_access_token.rb @@ -3,9 +3,16 @@ module API module Entities class PersonalAccessToken < Grape::Entity - expose :id, :name, :revoked, :created_at, :scopes, :user_id, :last_used_at - expose :active?, as: :active - expose :expires_at do |personal_access_token| + expose :id, documentation: { type: 'integer', example: 2 } + expose :name, documentation: { type: 'string', example: 'John Doe' } + expose :revoked, documentation: { type: 'boolean' } + expose :created_at, documentation: { type: 'dateTime' } + expose :scopes, documentation: { type: 'array', example: ['api'] } + expose :user_id, documentation: { type: 'integer', example: 3 } + expose :last_used_at, documentation: { type: 'dateTime', example: '2020-08-31T15:53:00.073Z' } + expose :active?, as: :active, documentation: { type: 'boolean' } + expose :expires_at, documentation: + { type: 'dateTime', example: '2020-08-31T15:53:00.073Z' } do |personal_access_token| personal_access_token.expires_at ? personal_access_token.expires_at.strftime("%Y-%m-%d") : nil end end diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb index 94e50f19b35..34018f03eb1 100644 --- a/lib/api/entities/plan_limit.rb +++ b/lib/api/entities/plan_limit.rb @@ -3,23 +3,23 @@ module API module Entities class PlanLimit < Grape::Entity - expose :ci_pipeline_size - expose :ci_active_jobs - expose :ci_active_pipelines - expose :ci_project_subscriptions - expose :ci_pipeline_schedules - expose :ci_needs_size_limit - expose :ci_registered_group_runners - expose :ci_registered_project_runners - expose :conan_max_file_size - expose :generic_packages_max_file_size - expose :helm_max_file_size - expose :maven_max_file_size - expose :npm_max_file_size - expose :nuget_max_file_size - expose :pypi_max_file_size - expose :terraform_module_max_file_size - expose :storage_size_limit + expose :ci_pipeline_size, documentation: { type: 'integer', example: 0 } + expose :ci_active_jobs, documentation: { type: 'integer', example: 0 } + expose :ci_active_pipelines, documentation: { type: 'integer', example: 0 } + expose :ci_project_subscriptions, documentation: { type: 'integer', example: 2 } + expose :ci_pipeline_schedules, documentation: { type: 'integer', example: 10 } + expose :ci_needs_size_limit, documentation: { type: 'integer', example: 50 } + expose :ci_registered_group_runners, documentation: { type: 'integer', example: 1000 } + expose :ci_registered_project_runners, documentation: { type: 'integer', example: 1000 } + expose :conan_max_file_size, documentation: { type: 'integer', example: 3221225472 } + expose :generic_packages_max_file_size, documentation: { type: 'integer', example: 5368709120 } + expose :helm_max_file_size, documentation: { type: 'integer', example: 5242880 } + expose :maven_max_file_size, documentation: { type: 'integer', example: 3221225472 } + expose :npm_max_file_size, documentation: { type: 'integer', example: 524288000 } + expose :nuget_max_file_size, documentation: { type: 'integer', example: 524288000 } + expose :pypi_max_file_size, documentation: { type: 'integer', example: 3221225472 } + expose :terraform_module_max_file_size, documentation: { type: 'integer', example: 1073741824 } + expose :storage_size_limit, documentation: { type: 'integer', example: 15000 } end end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index f158695f605..1c1bafbf161 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -5,147 +5,148 @@ module API class Project < BasicProjectDetails include ::API::Helpers::RelatedResourcesHelpers - expose :container_registry_url, as: :container_registry_image_prefix, if: -> (_, _) { Gitlab.config.registry.enabled } + expose :container_registry_url, as: :container_registry_image_prefix, documentation: { type: 'string', example: 'registry.gitlab.example.com/gitlab/gitlab-client' }, if: -> (_, _) { Gitlab.config.registry.enabled } expose :_links do - expose :self do |project| + expose :self, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/4' } do |project| expose_url(api_v4_projects_path(id: project.id)) end - expose :issues, if: -> (project, options) { issues_available?(project, options) } do |project| + expose :issues, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/4/issues' }, if: -> (project, options) { issues_available?(project, options) } do |project| expose_url(api_v4_projects_issues_path(id: project.id)) end - expose :merge_requests, if: -> (project, options) { mrs_available?(project, options) } do |project| + expose :merge_requests, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/4/merge_requests' }, if: -> (project, options) { mrs_available?(project, options) } do |project| expose_url(api_v4_projects_merge_requests_path(id: project.id)) end - expose :repo_branches do |project| + expose :repo_branches, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/4/repository/branches' } do |project| expose_url(api_v4_projects_repository_branches_path(id: project.id)) end - expose :labels do |project| + expose :labels, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/4/labels' } do |project| expose_url(api_v4_projects_labels_path(id: project.id)) end - expose :events do |project| + expose :events, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/4/events' } do |project| expose_url(api_v4_projects_events_path(id: project.id)) end - expose :members do |project| + expose :members, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/4/members' } do |project| expose_url(api_v4_projects_members_path(id: project.id)) end - expose :cluster_agents do |project| + expose :cluster_agents, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/4/cluster_agents' } do |project| expose_url(api_v4_projects_cluster_agents_path(id: project.id)) end end - expose :packages_enabled - expose :empty_repo?, as: :empty_repo - expose :archived?, as: :archived - expose :visibility + expose :packages_enabled, documentation: { type: 'boolean' } + expose :empty_repo?, as: :empty_repo, documentation: { type: 'boolean' } + expose :archived?, as: :archived, documentation: { type: 'boolean' } + expose :visibility, documentation: { type: 'string', example: 'public' } expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } - expose :resolve_outdated_diff_discussions + expose :resolve_outdated_diff_discussions, documentation: { type: 'boolean' } expose :container_expiration_policy, using: Entities::ContainerExpirationPolicy, if: -> (project, _) { project.container_expiration_policy } # Expose old field names with the new permissions methods to keep API compatible # TODO: remove in API v5, replaced by *_access_level - expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } - expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } - expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } - expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } - expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } - expose(:container_registry_enabled) { |project, options| project.feature_available?(:container_registry, options[:current_user]) } - expose :service_desk_enabled - expose :service_desk_address, if: -> (project, options) do + expose(:issues_enabled, documentation: { type: 'boolean' }) { |project, options| project.feature_available?(:issues, options[:current_user]) } + expose(:merge_requests_enabled, documentation: { type: 'boolean' }) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } + expose(:wiki_enabled, documentation: { type: 'boolean' }) { |project, options| project.feature_available?(:wiki, options[:current_user]) } + expose(:jobs_enabled, documentation: { type: 'boolean' }) { |project, options| project.feature_available?(:builds, options[:current_user]) } + expose(:snippets_enabled, documentation: { type: 'boolean' }) { |project, options| project.feature_available?(:snippets, options[:current_user]) } + expose(:container_registry_enabled, documentation: { type: 'boolean' }) { |project, options| project.feature_available?(:container_registry, options[:current_user]) } + expose :service_desk_enabled, documentation: { type: 'boolean' } + expose :service_desk_address, documentation: { type: 'string', example: 'address@example.com' }, if: -> (project, options) do Ability.allowed?(options[:current_user], :admin_issue, project) end - expose(:can_create_merge_request_in) do |project, options| + expose(:can_create_merge_request_in, documentation: { type: 'boolean' }) do |project, options| Ability.allowed?(options[:current_user], :create_merge_request_in, project) end - expose(:issues_access_level) { |project, options| project_feature_string_access_level(project, :issues) } - expose(:repository_access_level) { |project, options| project_feature_string_access_level(project, :repository) } - expose(:merge_requests_access_level) { |project, options| project_feature_string_access_level(project, :merge_requests) } - expose(:forking_access_level) { |project, options| project_feature_string_access_level(project, :forking) } - expose(:wiki_access_level) { |project, options| project_feature_string_access_level(project, :wiki) } - expose(:builds_access_level) { |project, options| project_feature_string_access_level(project, :builds) } - expose(:snippets_access_level) { |project, options| project_feature_string_access_level(project, :snippets) } - expose(:pages_access_level) { |project, options| project_feature_string_access_level(project, :pages) } - expose(:operations_access_level) { |project, options| project_feature_string_access_level(project, :operations) } - expose(:analytics_access_level) { |project, options| project_feature_string_access_level(project, :analytics) } - expose(:container_registry_access_level) { |project, options| project_feature_string_access_level(project, :container_registry) } - expose(:security_and_compliance_access_level) { |project, options| project_feature_string_access_level(project, :security_and_compliance) } - expose(:releases_access_level) { |project, options| project_feature_string_access_level(project, :releases) } - - expose :emails_disabled - expose :shared_runners_enabled - expose :lfs_enabled?, as: :lfs_enabled - expose :creator_id + expose(:issues_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :issues) } + expose(:repository_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :repository) } + expose(:merge_requests_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :merge_requests) } + expose(:forking_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :forking) } + expose(:wiki_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :wiki) } + expose(:builds_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :builds) } + expose(:snippets_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :snippets) } + expose(:pages_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :pages) } + expose(:operations_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :operations) } + expose(:analytics_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :analytics) } + expose(:container_registry_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :container_registry) } + expose(:security_and_compliance_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :security_and_compliance) } + expose(:releases_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :releases) } + + expose :emails_disabled, documentation: { type: 'boolean' } + expose :shared_runners_enabled, documentation: { type: 'boolean' } + expose :lfs_enabled?, as: :lfs_enabled, documentation: { type: 'boolean' } + expose :creator_id, documentation: { type: 'integer', example: 1 } expose :forked_from_project, using: Entities::BasicProjectDetails, if: ->(project, options) do project.forked? && Ability.allowed?(options[:current_user], :read_project, project.forked_from_project) end - expose :mr_default_target_self, if: -> (project) { project.forked? } + expose :mr_default_target_self, if: -> (project) { project.forked? }, documentation: { type: 'boolean' } - expose :import_url, if: -> (project, options) { Ability.allowed?(options[:current_user], :admin_project, project) } do |project| + expose :import_url, documentation: { type: 'string', example: 'https://gitlab.com/gitlab/gitlab.git' }, if: -> (project, options) { Ability.allowed?(options[:current_user], :admin_project, project) } do |project| project[:import_url] end - expose :import_type, if: -> (project, options) { Ability.allowed?(options[:current_user], :admin_project, project) } - expose :import_status - expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } do |project| + expose :import_type, documentation: { type: 'string', example: 'git' }, if: -> (project, options) { Ability.allowed?(options[:current_user], :admin_project, project) } + expose :import_status, documentation: { type: 'string', example: 'none' } + expose :import_error, documentation: { type: 'string', example: 'Import error' }, if: lambda { |_project, options| options[:user_can_admin_project] } do |project| project.import_state&.last_error end - expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } - expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } - expose :ci_default_git_depth - expose :ci_forward_deployment_enabled - expose(:ci_job_token_scope_enabled) { |p, _| p.ci_outbound_job_token_scope_enabled? } - expose :ci_separated_caches - expose :ci_opt_in_jwt - expose :ci_allow_fork_pipelines_to_run_in_parent_project - expose :public_builds, as: :public_jobs - expose :build_git_strategy, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options| + expose :open_issues_count, documentation: { type: 'integer', example: 1 }, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } + expose :runners_token, documentation: { type: 'string', example: 'b8547b1dc37721d05889db52fa2f02' }, if: lambda { |_project, options| options[:user_can_admin_project] } + expose :ci_default_git_depth, documentation: { type: 'integer', example: 20 } + expose :ci_forward_deployment_enabled, documentation: { type: 'boolean' } + expose(:ci_job_token_scope_enabled, documentation: { type: 'boolean' }) { |p, _| p.ci_outbound_job_token_scope_enabled? } + expose :ci_separated_caches, documentation: { type: 'boolean' } + expose :ci_opt_in_jwt, documentation: { type: 'boolean' } + expose :ci_allow_fork_pipelines_to_run_in_parent_project, documentation: { type: 'boolean' } + expose :public_builds, as: :public_jobs, documentation: { type: 'boolean' } + expose :build_git_strategy, documentation: { type: 'string', example: 'fetch' }, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options| project.build_allow_git_fetch ? 'fetch' : 'clone' end - expose :build_timeout - expose :auto_cancel_pending_pipelines - expose :ci_config_path, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } - expose :shared_with_groups do |project, options| + expose :build_timeout, documentation: { type: 'integer', example: 3600 } + expose :auto_cancel_pending_pipelines, documentation: { type: 'string', example: 'enabled' } + expose :ci_config_path, documentation: { type: 'string', example: '' }, if: -> (project, options) { Ability.allowed?(options[:current_user], :read_code, project) } + expose :shared_with_groups, documentation: { is_array: true } do |project, options| user = options[:current_user] SharedGroupWithProject.represent(project.visible_group_links(for_user: user), options) end - expose :only_allow_merge_if_pipeline_succeeds - expose :allow_merge_on_skipped_pipeline - expose :restrict_user_defined_variables - expose :request_access_enabled - expose :only_allow_merge_if_all_discussions_are_resolved - expose :remove_source_branch_after_merge - expose :printing_merge_request_link_enabled - expose :merge_method - expose :squash_option - expose :enforce_auth_checks_on_uploads - expose :suggestion_commit_message - expose :merge_commit_template - expose :squash_commit_template + expose :only_allow_merge_if_pipeline_succeeds, documentation: { type: 'boolean' } + expose :allow_merge_on_skipped_pipeline, documentation: { type: 'boolean' } + expose :restrict_user_defined_variables, documentation: { type: 'boolean' } + expose :request_access_enabled, documentation: { type: 'boolean' } + expose :only_allow_merge_if_all_discussions_are_resolved, documentation: { type: 'boolean' } + expose :remove_source_branch_after_merge, documentation: { type: 'boolean' } + expose :printing_merge_request_link_enabled, documentation: { type: 'boolean' } + expose :merge_method, documentation: { type: 'string', example: 'merge' } + expose :squash_option, documentation: { type: 'string', example: 'default_off' } + expose :enforce_auth_checks_on_uploads, documentation: { type: 'boolean' } + expose :suggestion_commit_message, documentation: { type: 'string', example: 'Suggestion message' } + expose :merge_commit_template, documentation: { type: 'string', example: '%(title)' } + expose :squash_commit_template, documentation: { type: 'string', example: '%(source_branch)' } + expose :issue_branch_template, documentation: { type: 'string', example: '%(title)' } expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project) } - expose :auto_devops_enabled?, as: :auto_devops_enabled - expose :auto_devops_deploy_strategy do |project, options| + expose :auto_devops_enabled?, as: :auto_devops_enabled, documentation: { type: 'boolean' } + expose :auto_devops_deploy_strategy, documentation: { type: 'string', example: 'continuous' } do |project, options| project.auto_devops.nil? ? 'continuous' : project.auto_devops.deploy_strategy end - expose :autoclose_referenced_issues - expose :repository_storage, if: ->(project, options) { + expose :autoclose_referenced_issues, documentation: { type: 'boolean' } + expose :repository_storage, documentation: { type: 'string', example: 'default' }, if: ->(project, options) { Ability.allowed?(options[:current_user], :change_repository_storage, project) } - expose :keep_latest_artifacts_available?, as: :keep_latest_artifact - expose :runner_token_expiration_interval + expose :keep_latest_artifacts_available?, as: :keep_latest_artifact, documentation: { type: 'boolean' } + expose :runner_token_expiration_interval, documentation: { type: 'integer', example: 3600 } # rubocop: disable CodeReuse/ActiveRecord def self.preload_resource(project) diff --git a/lib/api/entities/project_daily_fetches.rb b/lib/api/entities/project_daily_fetches.rb index 036b5dc99b8..8797aeb9521 100644 --- a/lib/api/entities/project_daily_fetches.rb +++ b/lib/api/entities/project_daily_fetches.rb @@ -3,8 +3,8 @@ module API module Entities class ProjectDailyFetches < Grape::Entity - expose :fetch_count, as: :count - expose :date + expose :fetch_count, as: :count, documentation: { type: 'integer', example: 3 } + expose :date, documentation: { type: 'date', example: '2022-01-01' } end end end diff --git a/lib/api/entities/project_daily_statistics.rb b/lib/api/entities/project_daily_statistics.rb index 803ee445851..555ecc6be39 100644 --- a/lib/api/entities/project_daily_statistics.rb +++ b/lib/api/entities/project_daily_statistics.rb @@ -4,8 +4,8 @@ module API module Entities class ProjectDailyStatistics < Grape::Entity expose :fetches do - expose :total_fetch_count, as: :total - expose :fetches, as: :days, using: ProjectDailyFetches + expose :total_fetch_count, as: :total, documentation: { type: 'integer', example: 3 } + expose :fetches, as: :days, using: ProjectDailyFetches, documentation: { is_array: true } end end end diff --git a/lib/api/entities/project_export_status.rb b/lib/api/entities/project_export_status.rb index ad84a45996a..9a2aeb7a6bb 100644 --- a/lib/api/entities/project_export_status.rb +++ b/lib/api/entities/project_export_status.rb @@ -5,13 +5,21 @@ module API class ProjectExportStatus < ProjectIdentity include ::API::Helpers::RelatedResourcesHelpers - expose :export_status + expose :export_status, documentation: { + type: 'string', example: 'finished', values: %w[queued started finished failed] + } expose :_links, if: lambda { |project, _options| project.export_status == :finished } do - expose :api_url do |project| + expose :api_url, documentation: { + type: 'string', + example: 'https://gitlab.example.com/api/v4/projects/1/export/download' + } do |project| expose_url(api_v4_projects_export_download_path(id: project.id)) end - expose :web_url do |project| + expose :web_url, documentation: { + type: 'string', + example: 'https://gitlab.example.com/gitlab-org/gitlab-test/download_export' + } do |project| Gitlab::Routing.url_helpers.download_export_project_url(project) end end diff --git a/lib/api/entities/project_group_link.rb b/lib/api/entities/project_group_link.rb index 89138854e67..b5d5991ec7f 100644 --- a/lib/api/entities/project_group_link.rb +++ b/lib/api/entities/project_group_link.rb @@ -3,7 +3,11 @@ module API module Entities class ProjectGroupLink < Grape::Entity - expose :id, :project_id, :group_id, :group_access, :expires_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :project_id, documentation: { type: 'integer', example: 1 } + expose :group_id, documentation: { type: 'integer', example: 1 } + expose :group_access, documentation: { type: 'integer', example: 10 } + expose :expires_at, documentation: { type: 'date', example: '2016-09-26' } end end end diff --git a/lib/api/entities/project_hook.rb b/lib/api/entities/project_hook.rb index 6c71e5d317c..bffb057abed 100644 --- a/lib/api/entities/project_hook.rb +++ b/lib/api/entities/project_hook.rb @@ -3,10 +3,17 @@ module API module Entities class ProjectHook < Hook - expose :project_id, :issues_events, :confidential_issues_events - expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events, :deployment_events - expose :job_events, :releases_events - expose :push_events_branch_filter + expose :project_id, documentation: { type: 'string', example: 1 } + expose :issues_events, documentation: { type: 'boolean' } + expose :confidential_issues_events, documentation: { type: 'boolean' } + expose :note_events, documentation: { type: 'boolean' } + expose :confidential_note_events, documentation: { type: 'boolean' } + expose :pipeline_events, documentation: { type: 'boolean' } + expose :wiki_page_events, documentation: { type: 'boolean' } + expose :deployment_events, documentation: { type: 'boolean' } + expose :job_events, documentation: { type: 'boolean' } + expose :releases_events, documentation: { type: 'boolean' } + expose :push_events_branch_filter, documentation: { type: 'string', example: 'my-branch-*' } end end end diff --git a/lib/api/entities/project_identity.rb b/lib/api/entities/project_identity.rb index 2055195eea0..14aef05b95e 100644 --- a/lib/api/entities/project_identity.rb +++ b/lib/api/entities/project_identity.rb @@ -3,10 +3,13 @@ module API module Entities class ProjectIdentity < Grape::Entity - expose :id, :description - expose :name, :name_with_namespace - expose :path, :path_with_namespace - expose :created_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :description, documentation: { type: 'string', example: 'desc' } + expose :name, documentation: { type: 'string', example: 'project1' } + expose :name_with_namespace, documentation: { type: 'string', example: 'John Doe / project1' } + expose :path, documentation: { type: 'string', example: 'project1' } + expose :path_with_namespace, documentation: { type: 'string', example: 'namespace1/project1' } + expose :created_at, documentation: { type: 'dateTime', example: '2020-05-07T04:27:17.016Z' } end end end diff --git a/lib/api/entities/project_import_failed_relation.rb b/lib/api/entities/project_import_failed_relation.rb index 26cfae7260c..543cc62f364 100644 --- a/lib/api/entities/project_import_failed_relation.rb +++ b/lib/api/entities/project_import_failed_relation.rb @@ -3,14 +3,17 @@ module API module Entities class ProjectImportFailedRelation < Grape::Entity - expose :id, :created_at, :exception_class, :source + expose :id, documentation: { type: 'string', example: 1 } + expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :exception_class, documentation: { type: 'string', example: 'StandardError' } + expose :source, documentation: { type: 'string', example: 'ImportRepositoryWorker' } - expose :exception_message do |_| + expose :exception_message, documentation: { type: 'string' } do |_| nil end - expose :relation_key, as: :relation_name - expose :relation_index, as: :line_number + expose :relation_key, as: :relation_name, documentation: { type: 'string', example: 'issues' } + expose :relation_index, as: :line_number, documentation: { type: 'integer', example: 1 } end end end diff --git a/lib/api/entities/project_import_status.rb b/lib/api/entities/project_import_status.rb index 5daae4a70f2..59388aacafd 100644 --- a/lib/api/entities/project_import_status.rb +++ b/lib/api/entities/project_import_status.rb @@ -3,21 +3,25 @@ module API module Entities class ProjectImportStatus < ProjectIdentity - expose :import_status - expose :import_type - expose :correlation_id do |project, _options| + expose :import_status, documentation: { type: 'string', example: 'scheduled' } + expose :import_type, documentation: { type: 'string', example: 'gitlab_project' } + expose :correlation_id, documentation: { + type: 'string', example: 'dfcf583058ed4508e4c7c617bd7f0edd' + } do |project, _options| project.import_state&.correlation_id end - expose :failed_relations, using: Entities::ProjectImportFailedRelation do |project, _options| + expose :failed_relations, using: Entities::ProjectImportFailedRelation, documentation: { + is_array: true + } do |project, _options| project.import_state&.relation_hard_failures(limit: 100) || [] end - expose :import_error do |project, _options| + expose :import_error, documentation: { type: 'string', example: 'Error message' } do |project, _options| project.import_state&.last_error end - expose :stats do |project, _options| + expose :stats, documentation: { type: 'object' } do |project, _options| if project.github_import? ::Gitlab::GithubImport::ObjectCounter.summary(project) end diff --git a/lib/api/entities/project_integration.rb b/lib/api/entities/project_integration.rb index 155136d2f80..29bb60a19e5 100644 --- a/lib/api/entities/project_integration.rb +++ b/lib/api/entities/project_integration.rb @@ -4,7 +4,7 @@ module API module Entities class ProjectIntegration < Entities::ProjectIntegrationBasic # Expose serialized properties - expose :properties do |integration, options| + expose :properties, documentation: { type: 'Hash', example: { "token" => "secr3t" } } do |integration, options| integration.api_field_names.to_h do |name| [name, integration.public_send(name)] # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/api/entities/project_integration_basic.rb b/lib/api/entities/project_integration_basic.rb index 2870123b83d..aa0ad158b83 100644 --- a/lib/api/entities/project_integration_basic.rb +++ b/lib/api/entities/project_integration_basic.rb @@ -3,15 +3,26 @@ module API module Entities class ProjectIntegrationBasic < Grape::Entity - expose :id, :title - expose :slug do |integration| + expose :id, documentation: { type: 'integer', example: 75 } + expose :title, documentation: { type: 'string', example: 'Jenkins CI' } + expose :slug, documentation: { type: 'integer', example: 'jenkins' } do |integration| integration.to_param.dasherize end - expose :created_at, :updated_at, :active - expose :commit_events, :push_events, :issues_events, :confidential_issues_events - expose :merge_requests_events, :tag_push_events, :note_events - expose :confidential_note_events, :pipeline_events, :wiki_page_events - expose :job_events, :comment_on_event_enabled + expose :created_at, documentation: { type: 'dateTime', example: '2019-11-20T11:20:25.297Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2019-11-20T12:24:37.498Z' } + expose :active, documentation: { type: 'boolean' } + expose :commit_events, documentation: { type: 'boolean' } + expose :push_events, documentation: { type: 'boolean' } + expose :issues_events, documentation: { type: 'boolean' } + expose :confidential_issues_events, documentation: { type: 'boolean' } + expose :merge_requests_events, documentation: { type: 'boolean' } + expose :tag_push_events, documentation: { type: 'boolean' } + expose :note_events, documentation: { type: 'boolean' } + expose :confidential_note_events, documentation: { type: 'boolean' } + expose :pipeline_events, documentation: { type: 'boolean' } + expose :wiki_page_events, documentation: { type: 'boolean' } + expose :job_events, documentation: { type: 'boolean' } + expose :comment_on_event_enabled, documentation: { type: 'boolean' } end end end diff --git a/lib/api/entities/project_repository_storage.rb b/lib/api/entities/project_repository_storage.rb index 0816bebde2c..ae5039601d7 100644 --- a/lib/api/entities/project_repository_storage.rb +++ b/lib/api/entities/project_repository_storage.rb @@ -5,12 +5,16 @@ module API class ProjectRepositoryStorage < Grape::Entity include Gitlab::Routing - expose :disk_path do |project| + expose :disk_path, documentation: { + type: 'string', + example: '@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' + } do |project| project.repository.disk_path end - expose :id, as: :project_id - expose :repository_storage, :created_at + expose :id, as: :project_id, documentation: { type: 'integer', example: 1 } + expose :repository_storage, documentation: { type: 'string', example: 'default' } + expose :created_at, documentation: { type: 'dateTime', example: '2012-10-12T17:04:47Z' } end end end diff --git a/lib/api/entities/project_with_access.rb b/lib/api/entities/project_with_access.rb index b541ccbadcf..9722b8806d4 100644 --- a/lib/api/entities/project_with_access.rb +++ b/lib/api/entities/project_with_access.rb @@ -25,23 +25,41 @@ module API # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) - relation = super(projects_relation, options) - # use reselect to override the existing select and - # prevent an error `subquery has too many columns` - project_ids = relation.reselect('projects.id') - namespace_ids = relation.reselect(:namespace_id) + if ::Feature.enabled?(:projects_preloader_fix) + super(projects_relation, options) + else + relation = super(projects_relation, options) + # use reselect to override the existing select and + # prevent an error `subquery has too many columns` + project_ids = relation.reselect('projects.id') + namespace_ids = relation.reselect(:namespace_id) + + options[:project_members] = options[:current_user] + .project_members + .where(source_id: project_ids) + .preload(:source, user: [notification_settings: :source]) + + options[:group_members] = options[:current_user] + .group_members + .where(source_id: namespace_ids) + .preload(:source, user: [notification_settings: :source]) + + relation + end + end + + def self.postload_relation(projects_relation, options = {}) + return unless ::Feature.enabled?(:projects_preloader_fix) options[:project_members] = options[:current_user] .project_members - .where(source_id: project_ids) + .where(source_id: projects_relation.subquery(:id)) .preload(:source, user: [notification_settings: :source]) options[:group_members] = options[:current_user] .group_members - .where(source_id: namespace_ids) + .where(source_id: projects_relation.subquery(:namespace_id)) .preload(:source, user: [notification_settings: :source]) - - relation end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/entities/protected_branch.rb b/lib/api/entities/protected_branch.rb index ac44d06e69c..42f721b40a6 100644 --- a/lib/api/entities/protected_branch.rb +++ b/lib/api/entities/protected_branch.rb @@ -3,11 +3,11 @@ module API module Entities class ProtectedBranch < Grape::Entity - expose :id - expose :name - expose :push_access_levels, using: Entities::ProtectedRefAccess - expose :merge_access_levels, using: Entities::ProtectedRefAccess - expose :allow_force_push + expose :id, documentation: { type: 'integer', example: 1 } + expose :name, documentation: { type: 'string', example: 'main' } + expose :push_access_levels, using: Entities::ProtectedRefAccess, documentation: { is_array: true } + expose :merge_access_levels, using: Entities::ProtectedRefAccess, documentation: { is_array: true } + expose :allow_force_push, documentation: { type: 'boolean' } end end end diff --git a/lib/api/entities/protected_ref_access.rb b/lib/api/entities/protected_ref_access.rb index 443277e23cf..ba28c724448 100644 --- a/lib/api/entities/protected_ref_access.rb +++ b/lib/api/entities/protected_ref_access.rb @@ -3,10 +3,12 @@ module API module Entities class ProtectedRefAccess < Grape::Entity - expose :access_level - expose :access_level_description do |protected_ref_access| - protected_ref_access.humanize - end + expose :id, documentation: { type: 'integer', example: 1 } + expose :access_level, documentation: { type: 'integer', example: 40 } + expose :access_level_description, + documentation: { type: 'string', example: 'Maintainers' } do |protected_ref_access| + protected_ref_access.humanize + end end end end diff --git a/lib/api/entities/protected_tag.rb b/lib/api/entities/protected_tag.rb index dc397f01af6..ba984ae79b8 100644 --- a/lib/api/entities/protected_tag.rb +++ b/lib/api/entities/protected_tag.rb @@ -3,7 +3,7 @@ module API module Entities class ProtectedTag < Grape::Entity - expose :name + expose :name, documentation: { type: 'string', example: 'release-1-0' } expose :create_access_levels, using: Entities::ProtectedRefAccess end end diff --git a/lib/api/entities/pull_mirror.rb b/lib/api/entities/pull_mirror.rb new file mode 100644 index 00000000000..72a5220987e --- /dev/null +++ b/lib/api/entities/pull_mirror.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module API + module Entities + class PullMirror < Grape::Entity + expose :id, documentation: { type: 'integer', example: 101486 } + expose :status, as: :update_status, documentation: { type: 'string', example: 'finished' } + expose :url, +documentation: { type: 'string', + example: 'https://*****:*****@gitlab.com/gitlab-org/security/gitlab.git' } do |import_state| + import_state.project.safe_import_url + end + expose :last_error, documentation: { type: 'string', example: nil } + expose :last_update_at, documentation: { type: 'dateTime', example: '2020-01-06T17:32:02.823Z' } + expose :last_update_started_at, documentation: { type: 'dateTime', example: '2020-01-06T17:32:02.823Z' } + expose :last_successful_update_at, documentation: { type: 'dateTime', example: '2020-01-06T17:32:02.823Z' } + end + end +end diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb index 2403c907f7f..c1a48a46d64 100644 --- a/lib/api/entities/release.rb +++ b/lib/api/entities/release.rb @@ -9,22 +9,24 @@ module API MarkupHelper.markdown_field(entity, :description, current_user: options[:current_user]) end expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } - expose :commit, using: Entities::Commit, if: ->(_, _) { can_download_code? } + expose :commit, using: Entities::Commit, if: ->(_, _) { can_read_code? } expose :milestones, using: Entities::MilestoneWithStats, if: -> (release, _) { release.milestones.present? && can_read_milestone? } do |release, _| release.milestones.order_by_dates_and_title end - expose :commit_path, expose_nil: false - expose :tag_path, expose_nil: false + expose :commit_path, + documentation: { type: 'string', example: '/root/app/commit/588440f66559714280628a4f9799f0c4eb880a4a' }, + expose_nil: false + expose :tag_path, documentation: { type: 'string', example: '/root/app/-/tags/v1.0' }, expose_nil: false expose :assets do - expose :assets_count, as: :count - expose :sources, using: Entities::Releases::Source, if: ->(_, _) { can_download_code? } + expose :assets_count, documentation: { type: 'integer', example: 2 }, as: :count + expose :sources, using: Entities::Releases::Source, if: ->(_, _) { can_read_code? } expose :sorted_links, as: :links, using: Entities::Releases::Link end - expose :evidences, using: Entities::Releases::Evidence, expose_nil: false, if: ->(_, _) { can_download_code? } + expose :evidences, using: Entities::Releases::Evidence, expose_nil: false, if: ->(_, _) { can_read_code? } expose :_links do expose :self_url, as: :self, expose_nil: false expose :edit_url, expose_nil: false @@ -32,8 +34,8 @@ module API private - def can_download_code? - Ability.allowed?(options[:current_user], :download_code, object.project) + def can_read_code? + Ability.allowed?(options[:current_user], :read_code, object.project) end def can_read_milestone? diff --git a/lib/api/entities/releases/evidence.rb b/lib/api/entities/releases/evidence.rb index 01603a71dbf..9d324309213 100644 --- a/lib/api/entities/releases/evidence.rb +++ b/lib/api/entities/releases/evidence.rb @@ -6,9 +6,9 @@ module API class Evidence < Grape::Entity include ::API::Helpers::Presentable - expose :sha - expose :filepath - expose :collected_at + expose :sha, documentation: { type: 'string', example: '760d6cdfb0879c3ffedec13af470e0f71cf52c6cde4d' } + expose :filepath, documentation: { type: 'string', example: 'https://gitlab.example.com/root/app/-/releases/v1.0/evidence.json' } + expose :collected_at, documentation: { type: 'dateTime', example: '2019-01-03T01:56:19.539Z' } end end end diff --git a/lib/api/entities/releases/link.rb b/lib/api/entities/releases/link.rb index 5157645af69..abf380e11d5 100644 --- a/lib/api/entities/releases/link.rb +++ b/lib/api/entities/releases/link.rb @@ -4,14 +4,22 @@ module API module Entities module Releases class Link < Grape::Entity - expose :id - expose :name - expose :url - expose :direct_asset_url do |link| + expose :id, documentation: { type: 'integer', example: 1 } + expose :name, documentation: { type: 'string', example: 'app-v1.0.dmg' } + expose :url, documentation: + { + type: 'string', + example: 'https://gitlab.example.com/root/app/-/jobs/688/artifacts/raw/bin/app-v1.0.dmg' + } + expose :direct_asset_url, documentation: + { + type: 'string', + example: 'https://gitlab.example.com/root/app/-/releases/v1.0/downloads/app-v1.0.dmg' + } do |link| ::Releases::LinkPresenter.new(link).direct_asset_url end - expose :external?, as: :external - expose :link_type + expose :external?, documentation: { type: 'boolean' }, as: :external + expose :link_type, documentation: { type: 'string', example: 'other' } end end end diff --git a/lib/api/entities/releases/source.rb b/lib/api/entities/releases/source.rb index 2b0c8038ddf..8c6750d6142 100644 --- a/lib/api/entities/releases/source.rb +++ b/lib/api/entities/releases/source.rb @@ -4,8 +4,8 @@ module API module Entities module Releases class Source < Grape::Entity - expose :format - expose :url + expose :format, documentation: { type: 'string', example: 'zip' } + expose :url, documentation: { type: 'string', example: 'https://gitlab.example.com/root/app/-/archive/v1.0/app-v1.0.zip' } end end end diff --git a/lib/api/entities/remote_mirror.rb b/lib/api/entities/remote_mirror.rb index 87daef9a05c..9fb5b2697bc 100644 --- a/lib/api/entities/remote_mirror.rb +++ b/lib/api/entities/remote_mirror.rb @@ -3,16 +3,16 @@ module API module Entities class RemoteMirror < Grape::Entity - expose :id - expose :enabled - expose :safe_url, as: :url - expose :update_status - expose :last_update_at - expose :last_update_started_at - expose :last_successful_update_at - expose :last_error - expose :only_protected_branches - expose :keep_divergent_refs + expose :id, documentation: { type: 'integer', example: 101486 } + expose :enabled, documentation: { type: 'boolean', example: true } + expose :safe_url, as: :url, documentation: { type: 'string', example: 'https://*****:*****@example.com/gitlab/example.git' } + expose :update_status, documentation: { type: 'string', example: 'finished' } + expose :last_update_at, documentation: { type: 'dateTime', example: '2020-01-06T17:32:02.823Z' } + expose :last_update_started_at, documentation: { type: 'dateTime', example: '2020-01-06T17:32:02.823Z' } + expose :last_successful_update_at, documentation: { type: 'dateTime', example: '2020-01-06T17:31:55.864Z' } + expose :last_error, documentation: { type: 'integer', example: 'The remote mirror URL is invalid.' } + expose :only_protected_branches, documentation: { type: 'boolean' } + expose :keep_divergent_refs, documentation: { type: 'boolean' } end end end diff --git a/lib/api/entities/resource_access_token.rb b/lib/api/entities/resource_access_token.rb index 569fd16f488..e4f140d3fc0 100644 --- a/lib/api/entities/resource_access_token.rb +++ b/lib/api/entities/resource_access_token.rb @@ -3,7 +3,12 @@ module API module Entities class ResourceAccessToken < Entities::PersonalAccessToken - expose :access_level do |token, options| + expose :access_level, + documentation: { type: 'integer', + example: 40, + description: 'Access level. Valid values are 10 (Guest), 20 (Reporter), 30 (Developer) \ + , 40 (Maintainer), and 50 (Owner). Defaults to 40.', + values: [10, 20, 30, 40, 50] } do |token, options| options[:resource].member(token.user).access_level end end diff --git a/lib/api/entities/resource_milestone_event.rb b/lib/api/entities/resource_milestone_event.rb index 26dc6620cbe..b301f5b7d0a 100644 --- a/lib/api/entities/resource_milestone_event.rb +++ b/lib/api/entities/resource_milestone_event.rb @@ -3,18 +3,18 @@ module API module Entities class ResourceMilestoneEvent < Grape::Entity - expose :id + expose :id, documentation: { type: 'integer', example: 142 } expose :user, using: Entities::UserBasic - expose :created_at - expose :resource_type do |event, _options| + expose :created_at, documentation: { type: 'dateTime', example: '2018-08-20T13:38:20.077Z' } + expose :resource_type, documentation: { type: 'string', example: 'Issue' } do |event, _options| event.issuable.class.name end - expose :resource_id do |event, _options| + expose :resource_id, documentation: { type: 'integer', example: 253 } do |event, _options| event.issuable.id end expose :milestone, using: Entities::Milestone - expose :action - expose :state + expose :action, documentation: { type: 'string', example: 'add' } + expose :state, documentation: { type: 'string', example: 'active' } end end end diff --git a/lib/api/entities/snippet.rb b/lib/api/entities/snippet.rb index af885aaf0eb..709566944ed 100644 --- a/lib/api/entities/snippet.rb +++ b/lib/api/entities/snippet.rb @@ -3,11 +3,13 @@ module API module Entities class Snippet < BasicSnippet - expose :author, using: Entities::UserBasic - expose :file_name do |snippet| + expose :author, using: Entities::UserBasic, documentation: { type: 'Entities::UserBasic' } + expose :file_name, documentation: { type: 'string', example: 'add.rb' } do |snippet| snippet_files.first || snippet.file_name end - expose :files do |snippet, options| + expose :files, documentation: { + is_array: true, example: 'e0d123e5f316bef78bfdf5a008837577' + } do |snippet, options| snippet_files.map do |file| { path: file, diff --git a/lib/api/entities/snippets/repository_storage_move.rb b/lib/api/entities/snippets/repository_storage_move.rb index 4e14d1dfba2..711d07545fb 100644 --- a/lib/api/entities/snippets/repository_storage_move.rb +++ b/lib/api/entities/snippets/repository_storage_move.rb @@ -4,7 +4,7 @@ module API module Entities module Snippets class RepositoryStorageMove < BasicRepositoryStorageMove - expose :snippet, using: Entities::BasicSnippet + expose :snippet, using: Entities::BasicSnippet, documentation: { type: 'Entities::BasicSnippet' } end end end diff --git a/lib/api/entities/ssh_key.rb b/lib/api/entities/ssh_key.rb index e1554730cb6..3db10bb8ec2 100644 --- a/lib/api/entities/ssh_key.rb +++ b/lib/api/entities/ssh_key.rb @@ -3,8 +3,15 @@ module API module Entities class SSHKey < Grape::Entity - expose :id, :title, :created_at, :expires_at - expose :publishable_key, as: :key + expose :id, documentation: { type: 'integer', example: 1 } + expose :title, documentation: { type: 'string', example: 'Sample key 25' } + expose :created_at, documentation: { type: 'dateTime', example: '2015-09-03T07:24:44.627Z' } + expose :expires_at, documentation: { type: 'dateTime', example: '2020-09-03T07:24:44.627Z' } + expose :publishable_key, as: :key, documentation: + { type: 'string', + example: 'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt1256k6Yjz\ + GGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCdd\ + NaP0L+hM7zhFNzjFvpaMgJw0=' } end end end diff --git a/lib/api/entities/tag.rb b/lib/api/entities/tag.rb index 2d3569bb9bb..713bae64d5c 100644 --- a/lib/api/entities/tag.rb +++ b/lib/api/entities/tag.rb @@ -3,7 +3,9 @@ module API module Entities class Tag < Grape::Entity - expose :name, :message, :target + expose :name, documentation: { type: 'string', example: 'v1.0.0' } + expose :message, documentation: { type: 'string', example: 'Release v1.0.0' } + expose :target, documentation: { type: 'string', example: '2695effb5807a22ff3d138d593fd856244e155e7' } expose :commit, using: Entities::Commit do |repo_tag, options| options[:project].repository.commit(repo_tag.dereferenced_target) @@ -15,7 +17,7 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - expose :protected do |repo_tag, options| + expose :protected, documentation: { type: 'boolean', example: true } do |repo_tag, options| ::ProtectedTag.protected?(options[:project], repo_tag.name) end end diff --git a/lib/api/entities/tag_release.rb b/lib/api/entities/tag_release.rb index d5f73d60332..66d1eeeab4a 100644 --- a/lib/api/entities/tag_release.rb +++ b/lib/api/entities/tag_release.rb @@ -4,8 +4,8 @@ module API module Entities # deprecated old Release representation class TagRelease < Grape::Entity - expose :tag, as: :tag_name - expose :description + expose :tag, as: :tag_name, documentation: { type: 'string', example: '1.0.0' } + expose :description, documentation: { type: 'string', example: 'Amazing release. Wow' } end end end diff --git a/lib/api/entities/templates_list.rb b/lib/api/entities/templates_list.rb index 8e8aa1bd285..eba80bd04d3 100644 --- a/lib/api/entities/templates_list.rb +++ b/lib/api/entities/templates_list.rb @@ -3,8 +3,8 @@ module API module Entities class TemplatesList < Grape::Entity - expose :key - expose :name + expose :key, documentation: { type: 'string', example: 'mit' } + expose :name, documentation: { type: 'string', example: 'MIT License' } end end end diff --git a/lib/api/entities/tree_object.rb b/lib/api/entities/tree_object.rb index e4e840ebe43..1f542885169 100644 --- a/lib/api/entities/tree_object.rb +++ b/lib/api/entities/tree_object.rb @@ -3,9 +3,12 @@ module API module Entities class TreeObject < Grape::Entity - expose :id, :name, :type, :path + expose :id, documentation: { example: 'a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba' } + expose :name, documentation: { example: 'html' } + expose :type, documentation: { example: 'tree' } + expose :path, documentation: { example: 'files/html' } - expose :mode do |obj, options| + expose :mode, documentation: { example: '040000' } do |obj, options| filemode = obj.mode filemode = "0" + filemode if filemode.length < 6 filemode diff --git a/lib/api/entities/trigger.rb b/lib/api/entities/trigger.rb index 6a9f772fc6b..ccdaeb6a07a 100644 --- a/lib/api/entities/trigger.rb +++ b/lib/api/entities/trigger.rb @@ -5,10 +5,12 @@ module API class Trigger < Grape::Entity include ::API::Helpers::Presentable - expose :id - expose :token - expose :description - expose :created_at, :updated_at, :last_used + expose :id, documentation: { type: 'integer', example: 10 } + expose :token, documentation: { type: 'string', example: '6d056f63e50fe6f8c5f8f4aa10edb7' } + expose :description, documentation: { type: 'string', example: 'test' } + expose :created_at, documentation: { type: 'dateTime', example: '2015-12-24T15:51:21.880Z' } + expose :updated_at, documentation: { type: 'dateTime', example: '2015-12-24T17:54:31.198Z' } + expose :last_used, documentation: { type: 'dateTime', example: '2015-12-24T17:54:31.198Z' } expose :owner, using: Entities::UserBasic end end diff --git a/lib/api/entities/user_agent_detail.rb b/lib/api/entities/user_agent_detail.rb index a2d02c16589..eb6d909794e 100644 --- a/lib/api/entities/user_agent_detail.rb +++ b/lib/api/entities/user_agent_detail.rb @@ -3,9 +3,9 @@ module API module Entities class UserAgentDetail < Grape::Entity - expose :user_agent - expose :ip_address - expose :submitted, as: :akismet_submitted + expose :user_agent, documentation: { type: 'string', example: 'AppleWebKit/537.36' } + expose :ip_address, documentation: { type: 'string', example: '127.0.0.1' } + expose :submitted, as: :akismet_submitted, documentation: { type: 'boolean', example: false } end end end diff --git a/lib/api/entities/user_associations_count.rb b/lib/api/entities/user_associations_count.rb new file mode 100644 index 00000000000..af744d2d49a --- /dev/null +++ b/lib/api/entities/user_associations_count.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module API + module Entities + class UserAssociationsCount < Grape::Entity + expose :groups_count do |user| + user.groups.size + end + + expose :projects_count do |user| + user.projects.size + end + + expose :issues_count do |user| + user.issues.size + end + + expose :merge_requests_count do |user| + user.merge_requests.size + end + end + end +end diff --git a/lib/api/entities/user_basic.rb b/lib/api/entities/user_basic.rb index b8ee4e5a6e0..32e066b9f7e 100644 --- a/lib/api/entities/user_basic.rb +++ b/lib/api/entities/user_basic.rb @@ -3,16 +3,25 @@ module API module Entities class UserBasic < UserSafe - expose :state + expose :state, documentation: { type: 'string', example: 'active' } - expose :avatar_url do |user, options| + expose :avatar_url, documentation: { type: 'string', example: 'https://gravatar.com/avatar/1' } do |user, options| user.avatar_url(only_path: false) end - expose :avatar_path, if: ->(user, options) { options.fetch(:only_path, false) && user.avatar_path } - expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes + expose( + :avatar_path, + documentation: { + type: 'string', + example: '/user/avatar/28/The-Big-Lebowski-400-400.png' + }, + if: ->(user, options) { options.fetch(:only_path, false) && user.avatar_path } + ) - expose :web_url do |user, options| + expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes, + documentation: { is_array: true } + + expose :web_url, documentation: { type: 'string', example: 'https://gitlab.example.com/root' } do |user, options| Gitlab::Routing.url_helpers.user_url(user) end end diff --git a/lib/api/entities/user_counts.rb b/lib/api/entities/user_counts.rb new file mode 100644 index 00000000000..e86454c249b --- /dev/null +++ b/lib/api/entities/user_counts.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module API + module Entities + class UserCounts < Grape::Entity + expose( + :assigned_open_merge_requests_count, # @deprecated + as: :merge_requests, + documentation: { type: 'integer', example: 10 } + ) + expose :assigned_open_issues_count, as: :assigned_issues, documentation: { type: 'integer', example: 10 } + expose( + :assigned_open_merge_requests_count, + as: :assigned_merge_requests, + documentation: { type: 'integer', example: 10 } + ) + expose( + :review_requested_open_merge_requests_count, + as: :review_requested_merge_requests, + documentation: { type: 'integer', example: 10 } + ) + expose :todos_pending_count, as: :todos, documentation: { type: 'integer', example: 10 } + end + end +end diff --git a/lib/api/entities/user_public.rb b/lib/api/entities/user_public.rb index 5d0e464abe1..eda72d2cfc6 100644 --- a/lib/api/entities/user_public.rb +++ b/lib/api/entities/user_public.rb @@ -3,17 +3,23 @@ module API module Entities class UserPublic < Entities::User - expose :last_sign_in_at - expose :confirmed_at - expose :last_activity_on - expose :email - expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at + expose :last_sign_in_at, documentation: { type: 'dateTime', example: '2015-09-03T07:24:01.670Z' } + expose :confirmed_at, documentation: { type: 'dateTime', example: '2015-09-03T07:24:01.670Z' } + expose :last_activity_on, documentation: { type: 'dateTime', example: '2015-09-03T07:24:01.670Z' } + expose :email, documentation: { type: 'string', example: 'john@example.com' } + expose :theme_id, documentation: { type: 'integer', example: 2 } + expose :color_scheme_id, documentation: { type: 'integer', example: 1 } + expose :projects_limit, documentation: { type: 'integer', example: 10 } + expose :current_sign_in_at, documentation: { type: 'dateTime', example: '2015-09-03T07:24:01.670Z' } expose :identities, using: Entities::Identity - expose :can_create_group?, as: :can_create_group - expose :can_create_project?, as: :can_create_project - expose :two_factor_enabled?, as: :two_factor_enabled + expose :can_create_group?, as: :can_create_group, documentation: { type: 'boolean', example: true } + expose :can_create_project?, as: :can_create_project, documentation: { type: 'boolean', example: true } + + expose :two_factor_enabled?, as: :two_factor_enabled, documentation: { type: 'boolean', example: true } + expose :external - expose :private_profile + + expose :private_profile, documentation: { type: 'boolean', example: :null } expose :commit_email_or_default, as: :commit_email end end diff --git a/lib/api/entities/user_safe.rb b/lib/api/entities/user_safe.rb index 127a8ef2160..0fbb10307cf 100644 --- a/lib/api/entities/user_safe.rb +++ b/lib/api/entities/user_safe.rb @@ -5,8 +5,9 @@ module API class UserSafe < Grape::Entity include RequestAwareEntity - expose :id, :username - expose :name do |user| + expose :id, documentation: { type: 'integer', example: 1 } + expose :username, documentation: { type: 'string', example: 'admin' } + expose :name, documentation: { type: 'string', example: 'Administrator' } do |user| current_user = request.respond_to?(:current_user) ? request.current_user : options.fetch(:current_user, nil) user.redacted_name(current_user) diff --git a/lib/api/entities/wiki_attachment.rb b/lib/api/entities/wiki_attachment.rb index 03a6cc8d644..8629261cfc6 100644 --- a/lib/api/entities/wiki_attachment.rb +++ b/lib/api/entities/wiki_attachment.rb @@ -5,12 +5,16 @@ module API class WikiAttachment < Grape::Entity include Gitlab::FileMarkdownLinkBuilder - expose :file_name - expose :file_path - expose :branch + expose :file_name, documentation: { type: 'string', example: 'dk.png' } + expose :file_path, documentation: { type: 'string', example: 'uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png' } + expose :branch, documentation: { type: 'string', example: 'main' } expose :link do - expose :file_path, as: :url - expose :markdown do |_entity| + expose :file_path, as: :url, documentation: { + type: 'string', example: 'uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png' + } + expose :markdown, documentation: { + type: 'string', example: '' + } do |_entity| self.markdown_link end end diff --git a/lib/api/entities/wiki_page.rb b/lib/api/entities/wiki_page.rb index 5bba4271396..07ef4a4a156 100644 --- a/lib/api/entities/wiki_page.rb +++ b/lib/api/entities/wiki_page.rb @@ -5,7 +5,9 @@ module API class WikiPage < WikiPageBasic include ::MarkupHelper - expose :content do |wiki_page, options| + expose :content, documentation: { + type: 'string', example: 'Here is an instruction how to deploy this project.' + } do |wiki_page, options| if options[:render_html] render_wiki_content( wiki_page, @@ -17,7 +19,7 @@ module API end end - expose :encoding do |wiki_page| + expose :encoding, documentation: { type: 'string', example: 'UTF-8' } do |wiki_page| wiki_page.content.encoding.name end end diff --git a/lib/api/entities/wiki_page_basic.rb b/lib/api/entities/wiki_page_basic.rb index e10c0e6d553..088a0d1bf55 100644 --- a/lib/api/entities/wiki_page_basic.rb +++ b/lib/api/entities/wiki_page_basic.rb @@ -3,9 +3,9 @@ module API module Entities class WikiPageBasic < Grape::Entity - expose :format - expose :slug - expose :title + expose :format, documentation: { type: 'string', example: 'markdown' } + expose :slug, documentation: { type: 'string', example: 'deploy' } + expose :title, documentation: { type: 'string', example: 'deploy' } end end end diff --git a/lib/api/entities/x509_certificate.rb b/lib/api/entities/x509_certificate.rb index aad11339148..95d4948906e 100644 --- a/lib/api/entities/x509_certificate.rb +++ b/lib/api/entities/x509_certificate.rb @@ -3,13 +3,16 @@ module API module Entities class X509Certificate < Grape::Entity - expose :id - expose :subject - expose :subject_key_identifier - expose :email - expose :serial_number - expose :certificate_status - expose :x509_issuer, using: 'API::Entities::X509Issuer' + expose :id, documentation: { type: 'integer', example: 1 } + expose :subject, documentation: { type: 'string', example: 'CN=gitlab@example.org,OU=Example,O=World' } + expose :subject_key_identifier, documentation: { + type: 'string', + example: 'BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC' + } + expose :email, documentation: { type: 'string', example: 'gitlab@example.org' } + expose :serial_number, documentation: { type: 'integer', example: 278969561018901340486471282831158785578 } + expose :certificate_status, documentation: { type: 'string', example: 'good' } + expose :x509_issuer, using: 'API::Entities::X509Issuer', documentation: { type: 'string', example: '100755' } end end end diff --git a/lib/api/entities/x509_issuer.rb b/lib/api/entities/x509_issuer.rb index b480bc107bc..22429560c72 100644 --- a/lib/api/entities/x509_issuer.rb +++ b/lib/api/entities/x509_issuer.rb @@ -3,10 +3,13 @@ module API module Entities class X509Issuer < Grape::Entity - expose :id - expose :subject - expose :subject_key_identifier - expose :crl_url + expose :id, documentation: { type: 'integer', example: 1 } + expose :subject, documentation: { type: 'string', example: 'CN=PKI,OU=Example,O=World' } + expose :subject_key_identifier, documentation: { + type: 'string', + example: 'AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB' + } + expose :crl_url, documentation: { type: 'string', example: 'http://example.com/pki.crl' } end end end diff --git a/lib/api/entities/x509_signature.rb b/lib/api/entities/x509_signature.rb index 909b630288c..c3f0cb3659d 100644 --- a/lib/api/entities/x509_signature.rb +++ b/lib/api/entities/x509_signature.rb @@ -3,7 +3,7 @@ module API module Entities class X509Signature < Grape::Entity - expose :verification_status + expose :verification_status, documentation: { type: 'string', example: 'unverified' } expose :x509_certificate, using: 'API::Entities::X509Certificate' end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 42d5e6a73b3..01d46ee7bfb 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -5,24 +5,35 @@ module API class Environments < ::API::Base include PaginationParams + environments_tags = %w[environments] + before { authenticate! } feature_category :continuous_delivery urgency :low params do - requires :id, type: String, desc: 'The project ID' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get all environments of the project' do - detail 'This feature was introduced in GitLab 8.11.' + desc 'List environments' do + detail 'Get all environments for a given project. This feature was introduced in GitLab 8.11.' success Entities::Environment + is_array true + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags environments_tags end params do use :pagination - optional :name, type: String, desc: 'Returns the environment with this name' - optional :search, type: String, desc: 'Returns list of environments matching the search criteria' - optional :states, type: String, values: Environment.valid_states.map(&:to_s), desc: 'List all environments that match a specific state' + optional :name, type: String, desc: 'Return the environment with this name. Mutually exclusive with search' + optional :search, type: String, desc: 'Return list of environments matching the search criteria. Mutually exclusive with name' + optional :states, + type: String, + values: Environment.valid_states.map(&:to_s), + desc: 'List all environments that match a specific state. Accepted values: `available`, `stopping`, or `stopped`. If no state value given, returns all environments' mutually_exclusive :name, :search, message: 'cannot be used together' end get ':id/environments' do @@ -33,15 +44,21 @@ module API present paginate(environments), with: Entities::Environment, current_user: current_user end - desc 'Creates a new environment' do - detail 'This feature was introduced in GitLab 8.11.' + desc 'Create a new environment' do + detail 'Creates a new environment with the given name and `external_url`. It returns `201` if the environment was successfully created, `400` for wrong parameters. This feature was introduced in GitLab 8.11.' success Entities::Environment + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags environments_tags end params do - requires :name, type: String, desc: 'The name of the environment to be created' - optional :external_url, type: String, desc: 'URL on which this deployment is viewable' + requires :name, type: String, desc: 'The name of the environment' + optional :external_url, type: String, desc: 'Place to link to for this environment' optional :slug, absence: { message: "is automatically generated and cannot be changed" }, documentation: { hidden: true } - optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the environment to be created' + optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the new environment. Allowed values are `production`, `staging`, `testing`, `development`, and `other`' end post ':id/environments' do authorize! :create_environment, user_project @@ -55,17 +72,23 @@ module API end end - desc 'Updates an existing environment' do - detail 'This feature was introduced in GitLab 8.11.' + desc 'Update an existing environment' do + detail 'Updates an existing environment name and/or `external_url`. It returns `200` if the environment was successfully updated. In case of an error, a status code `400` is returned. This feature was introduced in GitLab 8.11.' success Entities::Environment + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags environments_tags end params do - requires :environment_id, type: Integer, desc: 'The environment ID' + requires :environment_id, type: Integer, desc: 'The ID of the environment' # TODO: disallow renaming via the API https://gitlab.com/gitlab-org/gitlab/-/issues/338897 optional :name, type: String, desc: 'DEPRECATED: Renaming environment can lead to errors, this will be removed in 15.0' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' optional :slug, absence: { message: "is automatically generated and cannot be changed" }, documentation: { hidden: true } - optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the environment to be created' + optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the new environment. Allowed values are `production`, `staging`, `testing`, `development`, and `other`' end put ':id/environments/:environment_id' do authorize! :update_environment, user_project @@ -80,14 +103,21 @@ module API end end - desc "Delete multiple stopped review apps" do - detail "Remove multiple stopped review environments older than a specific age" + desc 'Delete multiple stopped review apps' do + detail 'It schedules for deletion multiple environments that have already been stopped and are in the review app folder. The actual deletion is performed after 1 week from the time of execution. By default, it only deletes environments 30 days or older. You can change this default using the `before` parameter.' success Entities::EnvironmentBasic + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' } + ] + tags environments_tags end params do - optional :before, type: Time, desc: "The timestamp before which environments can be deleted. Defaults to 30 days ago.", default: -> { 30.days.ago } - optional :limit, type: Integer, desc: "Maximum number of environments to delete. Defaults to 100.", default: 100, values: 1..1000 - optional :dry_run, type: Boolean, desc: "If set, perform a dry run where no actual deletions will be performed. Defaults to true.", default: true + optional :before, type: Time, desc: "The date before which environments can be deleted. Defaults to 30 days ago. Expected in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`)", default: -> { 30.days.ago } + optional :limit, type: Integer, desc: "Maximum number of environments to delete. Defaults to 100", default: 100, values: 1..1000 + optional :dry_run, type: Boolean, desc: "Defaults to true for safety reasons. It performs a dry run where no actual deletion will be performed. Set to false to actually delete the environment", default: true end delete ":id/environments/review_apps" do authorize! :read_environment, user_project @@ -107,12 +137,17 @@ module API end end - desc 'Deletes an existing environment' do - detail 'This feature was introduced in GitLab 8.11.' + desc 'Delete an environment' do + detail 'It returns 204 if the environment was successfully deleted, and 404 if the environment does not exist. This feature was introduced in GitLab 8.11.' success Entities::Environment + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[environments] end params do - requires :environment_id, type: Integer, desc: 'The environment ID' + requires :environment_id, type: Integer, desc: 'The ID of the environment' end delete ':id/environments/:environment_id' do authorize! :read_environment, user_project @@ -123,12 +158,18 @@ module API destroy_conditionally!(environment) end - desc 'Stops an existing environment' do + desc 'Stop an environment' do + detail 'It returns 200 if the environment was successfully stopped, and 404 if the environment does not exist.' success Entities::Environment + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[environments] end params do - requires :environment_id, type: Integer, desc: 'The environment ID' - optional :force, type: Boolean, default: false + requires :environment_id, type: Integer, desc: 'The ID of the environment' + optional :force, type: Boolean, default: false, desc: 'Force environment to stop without executing `on_stop` actions' end post ':id/environments/:environment_id/stop' do authorize! :read_environment, user_project @@ -141,11 +182,16 @@ module API present environment, with: Entities::Environment, current_user: current_user end - desc 'Get a single environment' do + desc 'Get a specific environment' do success Entities::Environment + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[environments] end params do - requires :environment_id, type: Integer, desc: 'The environment ID' + requires :environment_id, type: Integer, desc: 'The ID of the environment' end get ':id/environments/:environment_id' do authorize! :read_environment, user_project diff --git a/lib/api/error_tracking/client_keys.rb b/lib/api/error_tracking/client_keys.rb index c1c378111a7..8a0a5e2a9b7 100644 --- a/lib/api/error_tracking/client_keys.rb +++ b/lib/api/error_tracking/client_keys.rb @@ -4,11 +4,14 @@ module API class ErrorTracking::ClientKeys < ::API::Base before { authenticate! } + ERROR_TRACKING_CLIENT_KEYS_TAGS = %w[error_tracking_client_keys].freeze + feature_category :error_tracking urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], + desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -17,9 +20,11 @@ module API authorize! :admin_operations, user_project end - desc 'List all client keys' do - detail 'This feature was introduced in GitLab 14.3.' + desc 'List project client keys' do + detail 'List all client keys. This feature was introduced in GitLab 14.3.' success Entities::ErrorTracking::ClientKey + is_array true + tags ERROR_TRACKING_CLIENT_KEYS_TAGS end get '/client_keys' do collection = user_project.error_tracking_client_keys @@ -28,8 +33,10 @@ module API end desc 'Create a client key' do - detail 'This feature was introduced in GitLab 14.3.' + detail 'Creates a new client key for a project. The public key attribute is generated automatically.'\ + 'This feature was introduced in GitLab 14.3.' success Entities::ErrorTracking::ClientKey + tags ERROR_TRACKING_CLIENT_KEYS_TAGS end post '/client_keys' do key = user_project.error_tracking_client_keys.create! @@ -38,8 +45,14 @@ module API end desc 'Delete a client key' do - detail 'This feature was introduced in GitLab 14.3.' + detail 'Removes a client key from the project. This feature was introduced in GitLab 14.3.' success Entities::ErrorTracking::ClientKey + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags ERROR_TRACKING_CLIENT_KEYS_TAGS end delete '/client_keys/:key_id' do key = user_project.error_tracking_client_keys.find(params[:key_id]) diff --git a/lib/api/error_tracking/collector.rb b/lib/api/error_tracking/collector.rb index eea0fd2bce9..e10125e02c6 100644 --- a/lib/api/error_tracking/collector.rb +++ b/lib/api/error_tracking/collector.rb @@ -67,7 +67,7 @@ module API detail 'This feature was introduced in GitLab 14.1.' end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end post 'error_tracking/collector/api/:id/envelope' do # There is a reason why we have such uncommon path. @@ -119,7 +119,7 @@ module API detail 'This feature was introduced in GitLab 14.1.' end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end post 'error_tracking/collector/api/:id/store' do # There is a reason why we have such uncommon path. diff --git a/lib/api/error_tracking/project_settings.rb b/lib/api/error_tracking/project_settings.rb index fefc2098137..ec1d6a8b87f 100644 --- a/lib/api/error_tracking/project_settings.rb +++ b/lib/api/error_tracking/project_settings.rb @@ -4,6 +4,8 @@ module API class ErrorTracking::ProjectSettings < ::API::Base before { authenticate! } + ERROR_TRACKING_PROJECT_SETTINGS_TAGS = %w[error_tracking_project_settings].freeze + feature_category :error_tracking urgency :low @@ -14,7 +16,8 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], + desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -24,22 +27,35 @@ module API not_found!('Error Tracking Setting') unless project_setting end - desc 'Get error tracking settings for the project' do - detail 'This feature was introduced in GitLab 12.7.' + desc 'Get Error Tracking settings' do + detail 'Get error tracking settings for the project. This feature was introduced in GitLab 12.7.' success Entities::ErrorTracking::ProjectSetting + tags ERROR_TRACKING_PROJECT_SETTINGS_TAGS end get ':id/error_tracking/settings' do present project_setting, with: Entities::ErrorTracking::ProjectSetting end - desc 'Enable or disable error tracking settings for the project' do - detail 'This feature was introduced in GitLab 12.8.' + desc 'Enable or disable the Error Tracking project settings' do + detail 'The API allows you to enable or disable the Error Tracking settings for a project.'\ + 'Only for users with the Maintainer role for the project.' success Entities::ErrorTracking::ProjectSetting + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags ERROR_TRACKING_PROJECT_SETTINGS_TAGS end params do - requires :active, type: Boolean, desc: 'Specifying whether to enable or disable error tracking settings', allow_blank: false - optional :integrated, type: Boolean, desc: 'Specifying whether to enable or disable integrated error tracking' + requires :active, + type: Boolean, + desc: 'Pass true to enable the already configured Error Tracking settings or false to disable it.', + allow_blank: false + optional :integrated, + type: Boolean, + desc: 'Pass true to enable the integrated Error Tracking backend. Available in GitLab 14.2 and later.' end patch ':id/error_tracking/settings/' do diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb index 67e96284449..1846ddf6833 100644 --- a/lib/api/feature_flags.rb +++ b/lib/api/feature_flags.rb @@ -4,6 +4,8 @@ module API class FeatureFlags < ::API::Base include PaginationParams + feature_flags_tags = %w[feature_flags] + FEATURE_FLAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS .merge(name: API::NO_SLASH_URL_PART_REGEX) @@ -15,18 +17,24 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do resource :feature_flags do - desc 'Get all feature flags of a project' do - detail 'This feature was introduced in GitLab 12.5' + desc 'List feature flags for a project' do + detail 'Gets all feature flags of the requested project. This feature was introduced in GitLab 12.5.' success ::API::Entities::FeatureFlag + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags feature_flags_tags end params do optional :scope, type: String, - desc: 'The scope of feature flags', + desc: 'The scope of feature flags, one of: `enabled`, `disabled`', values: %w[enabled disabled] use :pagination end @@ -39,22 +47,23 @@ module API end desc 'Create a new feature flag' do - detail 'This feature was introduced in GitLab 12.5' + detail 'Creates a new feature flag. This feature was introduced in GitLab 12.5.' success ::API::Entities::FeatureFlag + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' } + ] + tags feature_flags_tags end params do - requires :name, type: String, desc: 'The name of feature flag' + requires :name, type: String, desc: 'The name of the feature flag' optional :description, type: String, desc: 'The description of the feature flag' - optional :active, type: Boolean, desc: 'Active/inactive value of the flag' - optional :version, type: String, desc: 'The version of the feature flag' - optional :scopes, type: Array do - requires :environment_scope, type: String, desc: 'The environment scope of the scope' - requires :active, type: Boolean, desc: 'Active/inactive of the scope' - requires :strategies, type: JSON, desc: 'The strategies of the scope' - end + optional :active, type: Boolean, desc: 'The active state of the flag. Defaults to `true`. Supported in GitLab 13.3 and later' + optional :version, type: String, desc: 'The version of the feature flag. Must be `new_version_flag`. Omit to create a Legacy feature flag.' optional :strategies, type: Array do - requires :name, type: String, desc: 'The strategy name' - requires :parameters, type: JSON, desc: 'The strategy parameters' + requires :name, type: String, desc: 'The strategy name. Can be `default`, `gradualRolloutUserId`, `userWithId`, or `gitlabUserList`. In GitLab 13.5 and later, can be `flexibleRollout`' + requires :parameters, type: JSON, desc: 'The strategy parameters as a JSON-formatted string e.g. `{"userIds":"user1"}`', documentation: { type: 'String' } optional :scopes, type: Array do requires :environment_scope, type: String, desc: 'The environment scope of the scope' end @@ -87,9 +96,14 @@ module API requires :feature_flag_name, type: String, desc: 'The name of the feature flag' end resource 'feature_flags/:feature_flag_name', requirements: FEATURE_FLAG_ENDPOINT_REQUIREMENTS do - desc 'Get a feature flag of a project' do - detail 'This feature was introduced in GitLab 12.5' + desc 'Get a single feature flag' do + detail 'Gets a single feature flag. This feature was introduced in GitLab 12.5.' success ::API::Entities::FeatureFlag + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags feature_flags_tags end get do authorize_read_feature_flag! @@ -99,20 +113,27 @@ module API end desc 'Update a feature flag' do - detail 'This feature was introduced in GitLab 13.2' + detail 'Updates a feature flag. This feature was introduced in GitLab 13.2.' success ::API::Entities::FeatureFlag + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags feature_flags_tags end params do - optional :name, type: String, desc: 'The name of the feature flag' + optional :name, type: String, desc: 'The new name of the feature flag. Supported in GitLab 13.3 and later' optional :description, type: String, desc: 'The description of the feature flag' - optional :active, type: Boolean, desc: 'Active/inactive value of the flag' + optional :active, type: Boolean, desc: 'The active state of the flag. Supported in GitLab 13.3 and later' optional :strategies, type: Array do - optional :id, type: Integer, desc: 'The strategy id' - optional :name, type: String, desc: 'The strategy type' - optional :parameters, type: JSON, desc: 'The strategy parameters' + optional :id, type: Integer, desc: 'The feature flag strategy ID' + optional :name, type: String, desc: 'The strategy name' + optional :parameters, type: JSON, desc: 'The strategy parameters as a JSON-formatted string e.g. `{"userIds":"user1"}`', documentation: { type: 'String' } optional :_destroy, type: Boolean, desc: 'Delete the strategy when true' optional :scopes, type: Array do - optional :id, type: Integer, desc: 'The environment scope id' + optional :id, type: Integer, desc: 'The scope id' optional :environment_scope, type: String, desc: 'The environment scope of the scope' optional :_destroy, type: Boolean, desc: 'Delete the scope when true' end @@ -142,8 +163,14 @@ module API end desc 'Delete a feature flag' do - detail 'This feature was introduced in GitLab 12.5' + detail 'Deletes a feature flag. This feature was introduced in GitLab 12.5.' success ::API::Entities::FeatureFlag + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags feature_flags_tags end delete do authorize_destroy_feature_flag! diff --git a/lib/api/feature_flags_user_lists.rb b/lib/api/feature_flags_user_lists.rb index f4771c07260..aed277d28a2 100644 --- a/lib/api/feature_flags_user_lists.rb +++ b/lib/api/feature_flags_user_lists.rb @@ -4,6 +4,8 @@ module API class FeatureFlagsUserLists < ::API::Base include PaginationParams + feature_flags_user_lists_tags = %w[feature_flags_user_lists] + error_formatter :json, -> (message, _backtrace, _options, _env, _original_exception) { message.is_a?(String) ? { message: message }.to_json : message.to_json } @@ -16,16 +18,23 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do resource :feature_flags_user_lists do - desc 'Get all feature flags user lists of a project' do - detail 'This feature was introduced in GitLab 12.10' + desc 'List all feature flag user lists for a project' do + detail 'Gets all feature flag user lists for the requested project. ' \ + 'This feature was introduced in GitLab 12.10.' success ::API::Entities::FeatureFlag::UserList + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags feature_flags_user_lists_tags end params do - optional :search, type: String, desc: 'Returns the list of user lists matching the search critiera' + optional :search, type: String, desc: 'Return user lists matching the search criteria' use :pagination end @@ -35,9 +44,15 @@ module API with: ::API::Entities::FeatureFlag::UserList end - desc 'Create a feature flags user list for a project' do - detail 'This feature was introduced in GitLab 12.10' + desc 'Create a feature flag user list' do + detail 'Creates a feature flag user list. This feature was introduced in GitLab 12.10.' success ::API::Entities::FeatureFlag::UserList + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags feature_flags_user_lists_tags end params do requires :name, type: String, desc: 'The name of the list' @@ -59,12 +74,17 @@ module API end params do - requires :iid, type: String, desc: 'The internal ID of the user list' + requires :iid, types: [String, Integer], desc: "The internal ID of the project's feature flag user list" end resource 'feature_flags_user_lists/:iid' do - desc 'Get a single feature flag user list belonging to a project' do - detail 'This feature was introduced in GitLab 12.10' + desc 'Get a feature flag user list' do + detail 'Gets a feature flag user list. This feature was introduced in GitLab 12.10.' success ::API::Entities::FeatureFlag::UserList + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags feature_flags_user_lists_tags end get do present user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid]), @@ -72,8 +92,14 @@ module API end desc 'Update a feature flag user list' do - detail 'This feature was introduced in GitLab 12.10' + detail 'Updates a feature flag user list. This feature was introduced in GitLab 12.10.' success ::API::Entities::FeatureFlag::UserList + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags feature_flags_user_lists_tags end params do optional :name, type: String, desc: 'The name of the list' @@ -93,8 +119,14 @@ module API end end - desc 'Delete a feature flag user list' do - detail 'This feature was introduced in GitLab 12.10' + desc 'Delete feature flag user list' do + detail 'Deletes a feature flag user list. This feature was introduced in GitLab 12.10.' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' } + ] + tags feature_flags_user_lists_tags end delete do # TODO: Move the business logic to a service class in app/services/feature_flags. diff --git a/lib/api/features.rb b/lib/api/features.rb index 9d4e6eee82c..6b6f5cbfb3f 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -4,6 +4,8 @@ module API class Features < ::API::Base before { authenticated_as_admin! } + features_tags = %w[features] + feature_category :feature_flags urgency :low @@ -44,8 +46,11 @@ module API end resource :features do - desc 'Get a list of all features' do + desc 'List all features' do + detail 'Get a list of all persisted features, with its gate values.' success Entities::Feature + is_array true + tags features_tags end get do features = Feature.all @@ -53,8 +58,11 @@ module API present features, with: Entities::Feature, current_user: current_user end - desc 'Get a list of all feature definitions' do + desc 'List all feature definitions' do + detail 'Get a list of all feature definitions.' success Entities::Feature::Definition + is_array true + tags features_tags end get :definitions do definitions = ::Feature::Definition.definitions.values.map(&:to_h) @@ -62,30 +70,44 @@ module API present definitions, with: Entities::Feature::Definition, current_user: current_user end - desc 'Set the gate value for the given feature' do + desc 'Set or create a feature' do + detail "Set a feature's gate value. If a feature with the given name doesn't exist yet, it's created. " \ + "The value can be a boolean, or an integer to indicate percentage of time." success Entities::Feature + failure [ + { code: 400, message: 'Bad request' } + ] + tags features_tags end params do - requires :value, type: String, desc: '`true` or `false` to enable/disable, a float for percentage of time' - optional :key, type: String, desc: '`percentage_of_actors` or the default `percentage_of_time`' + requires :value, + types: [String, Integer], + desc: '`true` or `false` to enable/disable, or an integer for percentage of time' + optional :key, type: String, desc: '`percentage_of_actors` or `percentage_of_time` (default)' optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username or comma-separated multiple usernames' optional :group, type: String, - desc: "A GitLab group's path, such as 'gitlab-org', or comma-separated multiple group paths" + desc: "A GitLab group's path, for example `gitlab-org`, or comma-separated multiple group paths" optional :namespace, type: String, - desc: "A GitLab group or user namespace path, such as 'john-doe', or comma-separated multiple namespace paths" + desc: "A GitLab group or user namespace's path, for example `john-doe`, or comma-separated " \ + "multiple namespace paths. Introduced in GitLab 15.0." optional :project, type: String, - desc: "A projects path, such as `gitlab-org/gitlab-ce`, or comma-separated multiple project paths" - optional :force, type: Boolean, desc: 'Skip feature flag validation checks, ie. YAML definition' + desc: "A projects path, for example `gitlab-org/gitlab-foss`, or comma-separated multiple project paths" + optional :repository, + type: String, + desc: "A repository path, for example `gitlab-org/gitlab-test.git`, `gitlab-org/gitlab-test.wiki.git`, " \ + "`snippets/21.git`, to name a few. Use comma to separate multiple repository paths" + optional :force, type: Boolean, desc: 'Skip feature flag validation checks, such as a YAML definition' mutually_exclusive :key, :feature_group mutually_exclusive :key, :user mutually_exclusive :key, :group mutually_exclusive :key, :namespace mutually_exclusive :key, :project + mutually_exclusive :key, :repository end post ':name' do if Feature.enabled?(:set_feature_flag_service) @@ -135,7 +157,10 @@ module API bad_request!(e.message) end - desc 'Remove the gate value for the given feature' + desc 'Delete a feature' do + detail "Removes a feature gate. Response is equal when the gate exists, or doesn't." + tags features_tags + end delete ':name' do Feature.remove(params[:name]) diff --git a/lib/api/files.rb b/lib/api/files.rb index fd574ca865b..fa749299b9a 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -30,7 +30,7 @@ module API end def assign_file_vars! - authorize! :download_code, user_project + authorize! :read_code, user_project @commit = user_project.commit(params[:ref]) not_found!('Commit') unless @commit @@ -82,33 +82,44 @@ module API end params :simple_file_params do - requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false - requires :commit_message, type: String, allow_blank: false, desc: 'Commit message' - optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' - optional :author_email, type: String, desc: 'The email of the author' - optional :author_name, type: String, desc: 'The name of the author' + requires :file_path, type: String, file_path: true, + desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } + requires :branch, type: String, + desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false, + documentation: { example: 'main' } + requires :commit_message, type: String, + allow_blank: false, desc: 'Commit message', documentation: { example: 'Initial commit' } + optional :start_branch, type: String, + desc: 'Name of the branch to start the new commit from', documentation: { example: 'main' } + optional :author_email, type: String, + desc: 'The email of the author', documentation: { example: 'johndoe@example.com' } + optional :author_name, type: String, + desc: 'The name of the author', documentation: { example: 'John Doe' } end params :extended_file_params do use :simple_file_params - requires :content, type: String, desc: 'File content' - optional :encoding, type: String, values: %w[base64], desc: 'File encoding' - optional :last_commit_id, type: String, desc: 'Last known commit id for this file' + requires :content, type: String, desc: 'File content', documentation: { example: 'file content' } + optional :encoding, type: String, values: %w[base64 text], default: 'text', desc: 'File encoding' + optional :last_commit_id, type: String, + desc: 'Last known commit id for this file', + documentation: { example: '2695effb5807a22ff3d138d593fd856244e155e7' } optional :execute_filemode, type: Boolean, desc: 'Enable / Disable the executable flag on the file path' end end params do - requires :id, type: String, desc: 'The project ID' + requires :id, type: String, desc: 'The project ID', documentation: { example: 'gitlab-org/gitlab' } end resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do allow_access_with_scope :read_repository, if: -> (request) { request.get? || request.head? } desc 'Get blame file metadata from repository' params do - requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + requires :file_path, type: String, file_path: true, + desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } + requires :ref, type: String, + desc: 'The name of branch, tag or commit', allow_blank: false, documentation: { example: 'main' } end head ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! @@ -118,11 +129,15 @@ module API desc 'Get blame file from the repository' params do - requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + requires :file_path, type: String, file_path: true, + desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } + requires :ref, type: String, + desc: 'The name of branch, tag or commit', allow_blank: false, documentation: { example: 'main' } optional :range, type: Hash do - requires :start, type: Integer, desc: 'The first line of the range to blame', allow_blank: false, values: ->(v) { v > 0 } - requires :end, type: Integer, desc: 'The last line of the range to blame', allow_blank: false, values: ->(v) { v > 0 } + requires :start, type: Integer, + desc: 'The first line of the range to blame', allow_blank: false, values: ->(v) { v > 0 } + requires :end, type: Integer, + desc: 'The last line of the range to blame', allow_blank: false, values: ->(v) { v > 0 } end end get ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do @@ -138,8 +153,10 @@ module API desc 'Get raw file metadata from repository' params do - requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + requires :file_path, type: String, file_path: true, + desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } + optional :ref, type: String, + desc: 'The name of branch, tag or commit', allow_blank: false, documentation: { example: 'main' } end head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do assign_file_vars! @@ -147,10 +164,14 @@ module API set_http_headers(blob_data) end - desc 'Get raw file contents from the repository' + desc 'Get raw file contents from the repository' do + success File + end params do - requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + requires :file_path, type: String, file_path: true, + desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } + optional :ref, type: String, + desc: 'The name of branch, tag or commit', allow_blank: false, documentation: { example: 'main' } end get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do assign_file_vars! @@ -163,8 +184,10 @@ module API desc 'Get file metadata from repository' params do - requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + requires :file_path, type: String, file_path: true, + desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } + requires :ref, type: String, + desc: 'The name of branch, tag or commit', allow_blank: false, documentation: { example: 'main' } end head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do assign_file_vars! @@ -174,8 +197,10 @@ module API desc 'Get a file from the repository' params do - requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + requires :file_path, type: String, file_path: true, + desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } + requires :ref, type: String, + desc: 'The name of branch, tag or commit', allow_blank: false, documentation: { example: 'main' } end get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! diff --git a/lib/api/freeze_periods.rb b/lib/api/freeze_periods.rb index e69baeee97f..40f1be83028 100644 --- a/lib/api/freeze_periods.rb +++ b/lib/api/freeze_periods.rb @@ -4,19 +4,28 @@ module API class FreezePeriods < ::API::Base include PaginationParams + freeze_periods_tags = %w[freeze_periods] + before { authenticate! } feature_category :continuous_delivery urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get project freeze periods' do - detail 'This feature was introduced in GitLab 13.0.' + desc 'List freeze periods' do + detail 'Paginated list of Freeze Periods, sorted by created_at in ascending order. ' \ + 'This feature was introduced in GitLab 13.0.' success Entities::FreezePeriod + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags freeze_periods_tags end params do use :pagination @@ -30,12 +39,17 @@ module API present paginate(freeze_periods), with: Entities::FreezePeriod, current_user: current_user end - desc 'Get a single freeze period' do - detail 'This feature was introduced in GitLab 13.0.' + desc 'Get a freeze period' do + detail 'Get a freeze period for the given `freeze_period_id`. This feature was introduced in GitLab 13.0.' success Entities::FreezePeriod + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags freeze_periods_tags end params do - requires :freeze_period_id, type: Integer, desc: 'The ID of a project freeze period' + requires :freeze_period_id, type: Integer, desc: 'The ID of the freeze period' end get ":id/freeze_periods/:freeze_period_id" do authorize! :read_freeze_period, user_project @@ -43,14 +57,21 @@ module API present freeze_period, with: Entities::FreezePeriod, current_user: current_user end - desc 'Create a new freeze period' do - detail 'This feature was introduced in GitLab 13.0.' + desc 'Create a freeze period' do + detail 'Creates a freeze period. This feature was introduced in GitLab 13.0.' success Entities::FreezePeriod + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' } + ] + tags freeze_periods_tags end params do - requires :freeze_start, type: String, desc: 'Freeze Period start' - requires :freeze_end, type: String, desc: 'Freeze Period end' - optional :cron_timezone, type: String, desc: 'Timezone' + requires :freeze_start, type: String, desc: 'Start of the freeze period in cron format.' + requires :freeze_end, type: String, desc: 'End of the freeze period in cron format' + optional :cron_timezone, + type: String, + desc: 'The time zone for the cron fields, defaults to UTC if not provided' end post ':id/freeze_periods' do authorize! :create_freeze_period, user_project @@ -67,13 +88,18 @@ module API end desc 'Update a freeze period' do - detail 'This feature was introduced in GitLab 13.0.' + detail 'Updates a freeze period for the given `freeze_period_id`. This feature was introduced in GitLab 13.0.' success Entities::FreezePeriod + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' } + ] + tags freeze_periods_tags end params do - optional :freeze_start, type: String, desc: 'Freeze Period start' - optional :freeze_end, type: String, desc: 'Freeze Period end' - optional :cron_timezone, type: String, desc: 'Freeze Period Timezone' + optional :freeze_start, type: String, desc: 'Start of the freeze period in cron format' + optional :freeze_end, type: String, desc: 'End of the freeze period in cron format' + optional :cron_timezone, type: String, desc: 'The time zone for the cron fields' end put ':id/freeze_periods/:freeze_period_id' do authorize! :update_freeze_period, user_project @@ -88,11 +114,15 @@ module API end desc 'Delete a freeze period' do - detail 'This feature was introduced in GitLab 13.0.' + detail 'Deletes a freeze period for the given `freeze_period_id`. This feature was introduced in GitLab 13.0.' success Entities::FreezePeriod + failure [ + { code: 401, message: 'Unauthorized' } + ] + tags freeze_periods_tags end params do - requires :freeze_period_id, type: Integer, desc: 'Freeze Period ID' + requires :freeze_period_id, type: Integer, desc: 'The ID of the freeze period' end delete ':id/freeze_periods/:freeze_period_id' do authorize! :destroy_freeze_period, user_project diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 0098b074f05..3584f8d025a 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -18,7 +18,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -54,7 +54,7 @@ module API requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true optional :status, type: String, values: ALLOWED_STATUSES, desc: 'Package status' - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } optional :select, type: String, values: %w[package_file] end diff --git a/lib/api/geo.rb b/lib/api/geo.rb index cb04d2a4e1e..8798b76b52b 100644 --- a/lib/api/geo.rb +++ b/lib/api/geo.rb @@ -13,6 +13,13 @@ module API end resource :geo do + desc 'Returns a Geo proxy response' do + summary "Determine if a Geo site should proxy requests" + success code: 200 + failure [{ code: 403, message: 'Forbidden' }] + tags %w[geo] + end + # Workhorse calls this to determine if it is a Geo site that should proxy # requests. Workhorse doesn't know if it's in a FOSS/EE context. get '/proxy' do diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb index 2d9c0cd6ce1..8fde40a4713 100755 --- a/lib/api/go_proxy.rb +++ b/lib/api/go_proxy.rb @@ -4,6 +4,8 @@ module API helpers Gitlab::Golang helpers ::API::Helpers::PackagesHelpers + GO_PROXY_TAGS = %w[go_proxy].freeze + feature_category :package_registry urgency :low @@ -17,6 +19,10 @@ module API before { require_packages_enabled! } helpers do + def project + user_project(action: :read_package) + end + def case_decode(str) # Converts "github.com/!azure" to "github.com/Azure" # @@ -32,12 +38,12 @@ module API end def find_module - not_found! unless Feature.enabled?(:go_proxy, user_project) + not_found! unless Feature.enabled?(:go_proxy, project) module_name = case_decode params[:module_name] bad_request_missing_attribute!('Module Name') if module_name.blank? - mod = ::Packages::Go::ModuleFinder.new(user_project, module_name).execute + mod = ::Packages::Go::ModuleFinder.new(project, module_name).execute not_found! unless mod @@ -58,18 +64,21 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' - requires :module_name, type: String, desc: 'Module name', coerce_with: ->(val) { CGI.unescape(val) } + requires :id, types: [String, Integer], desc: 'The project ID or full path of a project' + requires :module_name, type: String, desc: 'The name of the Go module', coerce_with: ->(val) { CGI.unescape(val) } end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, authenticate_non_public: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, + authenticate_non_public: true resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do - authorize_read_package! + authorize_read_package!(project) end namespace ':id/packages/go/*module_name/@v' do - desc 'Get all tagged versions for a given Go module' do - detail 'See `go help goproxy`, GET $GOPROXY/<module>/@v/list. This feature was introduced in GitLab 13.1.' + desc 'List' do + detail 'Get all tagged versions for a given Go module.'\ + 'See `go help goproxy`, GET $GOPROXY/<module>/@v/list. This feature was introduced in GitLab 13.1.' + tags GO_PROXY_TAGS end get 'list' do mod = find_module @@ -78,12 +87,14 @@ module API mod.versions.map { |t| t.name }.join("\n") end - desc 'Get information about the given module version' do - detail 'See `go help goproxy`, GET $GOPROXY/<module>/@v/<version>.info. This feature was introduced in GitLab 13.1.' + desc 'Version metadata' do + detail 'Get all tagged versions for a given Go module.'\ + 'See `go help goproxy`, GET $GOPROXY/<module>/@v/<version>.info. This feature was introduced in GitLab 13.1' success ::API::Entities::GoModuleVersion + tags GO_PROXY_TAGS end params do - requires :module_version, type: String, desc: 'Module version' + requires :module_version, type: String, desc: 'The version of the Go module' end get ':module_version.info', requirements: MODULE_VERSION_REQUIREMENTS do ver = find_version @@ -91,11 +102,13 @@ module API present ::Packages::Go::ModuleVersionPresenter.new(ver), with: ::API::Entities::GoModuleVersion end - desc 'Get the module file of the given module version' do - detail 'See `go help goproxy`, GET $GOPROXY/<module>/@v/<version>.mod. This feature was introduced in GitLab 13.1.' + desc 'Download module file' do + detail 'Get the module file of a given module version.'\ + 'See `go help goproxy`, GET $GOPROXY/<module>/@v/<version>.mod. This feature was introduced in GitLab 13.1.' + tags GO_PROXY_TAGS end params do - requires :module_version, type: String, desc: 'Module version' + requires :module_version, type: String, desc: 'The version of the Go module' end get ':module_version.mod', requirements: MODULE_VERSION_REQUIREMENTS do ver = find_version @@ -104,18 +117,21 @@ module API ver.gomod end - desc 'Get a zip of the source of the given module version' do - detail 'See `go help goproxy`, GET $GOPROXY/<module>/@v/<version>.zip. This feature was introduced in GitLab 13.1.' + desc 'Download module source' do + detail 'Get a zip of the source of the given module version.'\ + 'See `go help goproxy`, GET $GOPROXY/<module>/@v/<version>.zip. This feature was introduced in GitLab 13.1.' + tags GO_PROXY_TAGS end params do - requires :module_version, type: String, desc: 'Module version' + requires :module_version, type: String, desc: 'The version of the Go module' end get ':module_version.zip', requirements: MODULE_VERSION_REQUIREMENTS do ver = find_version content_type 'application/zip' env['api.format'] = :binary - header['Content-Disposition'] = ActionDispatch::Http::ContentDisposition.format(disposition: 'attachment', filename: ver.name + '.zip') + header['Content-Disposition'] = + ActionDispatch::Http::ContentDisposition.format(disposition: 'attachment', filename: "#{ver.name}.zip") header['Content-Transfer-Encoding'] = 'binary' status :ok body ver.archive.string diff --git a/lib/api/group_avatar.rb b/lib/api/group_avatar.rb index 9063040c763..0820011fd89 100644 --- a/lib/api/group_avatar.rb +++ b/lib/api/group_avatar.rb @@ -7,11 +7,13 @@ module API feature_category :subgroups params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, type: String, desc: 'The ID of the group' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Download the group avatar' do detail 'This feature was introduced in GitLab 14.0' + tags %w[group_avatar] + success code: 200 end get ':id/avatar' do avatar = user_group.avatar diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index edaa32c26c4..de5ca0f86ae 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -16,8 +16,14 @@ module API requires :id, type: String, desc: 'The ID of the group' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get all clusters from the group' do + desc 'List group clusters' do + detail 'This feature was introduced in GitLab 12.1. Returns a list of group clusters.' success Entities::Cluster + failure [ + { code: 403, message: 'Forbidden' } + ] + is_array true + tags %w[clusters] end params do use :pagination @@ -28,8 +34,14 @@ module API present paginate(clusters_for_current_user), with: Entities::Cluster end - desc 'Get specific cluster for the group' do + desc 'Get a single group cluster' do + detail 'This feature was introduced in GitLab 12.1. Gets a single group cluster.' success Entities::ClusterGroup + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: 'The cluster ID' @@ -40,8 +52,15 @@ module API present cluster, with: Entities::ClusterGroup end - desc 'Adds an existing cluster' do + desc 'Add existing cluster to group' do + detail 'This feature was introduced in GitLab 12.1. Adds an existing Kubernetes cluster to the group.' success Entities::ClusterGroup + failure [ + { code: 400, message: 'Validation error' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :name, type: String, desc: 'Cluster name' @@ -73,8 +92,15 @@ module API end end - desc 'Update an existing cluster' do + desc 'Edit group cluster' do + detail 'This feature was introduced in GitLab 12.1. Updates an existing group cluster.' success Entities::ClusterGroup + failure [ + { code: 400, message: 'Validation error' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: 'The cluster ID' @@ -104,8 +130,14 @@ module API end end - desc 'Remove a cluster' do + desc 'Delete group cluster' do + detail 'This feature was introduced in GitLab 12.1. Deletes an existing group cluster. Does not remove existing resources within the connected Kubernetes cluster.' success Entities::ClusterGroup + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: 'The Cluster ID' diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb index b834d177a12..753f0db10c1 100644 --- a/lib/api/group_container_repositories.rb +++ b/lib/api/group_container_repositories.rb @@ -16,12 +16,19 @@ module API tag_name: API::NO_SLASH_URL_PART_REGEX) params do - requires :id, type: String, desc: "Group's ID or path" + requires :id, types: [String, Integer], + desc: 'The ID or URL-encoded path of the group accessible by the authenticated user' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get a list of all repositories within a group' do - detail 'This feature was introduced in GitLab 12.2.' + desc 'List registry repositories within a group' do + detail 'Get a list of registry repositories in a group. This feature was introduced in GitLab 12.2.' success Entities::ContainerRegistry::Repository + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Group Not Found' } + ] + is_array true + tags %w[container_registry] end params do use :pagination diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 2948960a9b4..eb0a01e0d3d 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -15,6 +15,16 @@ module API resource :groups, requirements: { id: %r{[^/]+} } do desc 'Download export' do detail 'This feature was introduced in GitLab 12.5.' + tags %w[group_export] + produces %w[application/octet-stream application/json] + success code: 200 + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end get ':id/export/download' do check_rate_limit! :group_download_export, scope: [current_user, user_group] @@ -32,6 +42,15 @@ module API desc 'Start export' do detail 'This feature was introduced in GitLab 12.5.' + tags %w[group_export] + success code: 202 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 429, message: 'Too many requests' }, + { code: 503, message: 'Service unavailable' } + ] end post ':id/export' do check_rate_limit! :group_export, scope: current_user @@ -47,6 +66,14 @@ module API desc 'Start relations export' do detail 'This feature was introduced in GitLab 13.12' + tags %w[group_export] + success code: 202 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end post ':id/export_relations' do response = ::BulkImports::ExportService.new(portable: user_group, user: current_user).execute @@ -60,6 +87,15 @@ module API desc 'Download relations export' do detail 'This feature was introduced in GitLab 13.12' + produces %w[application/octet-stream application/json] + tags %w[group_export] + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end params do requires :relation, type: String, desc: 'Group relation name' @@ -77,6 +113,15 @@ module API desc 'Relations export status' do detail 'This feature was introduced in GitLab 13.12' + is_array true + tags %w[group_export] + success code: 200, model: Entities::BulkImports::ExportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end get ':id/export_relations/status' do present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb index cef9b542c9e..609a7ed0ef0 100644 --- a/lib/api/group_import.rb +++ b/lib/api/group_import.rb @@ -32,6 +32,7 @@ module API resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Workhorse authorize the group import upload' do detail 'This feature was introduced in GitLab 12.8' + tags ['group_import'] end post 'import/authorize' do require_gitlab_workhorse! @@ -49,7 +50,15 @@ module API desc 'Create a new group import' do detail 'This feature was introduced in GitLab 12.8' - success Entities::Group + success code: 202 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 400, message: 'Bad request' }, + { code: 503, message: 'Service unavailable' } + ] + consumes ['multipart/form-data'] + tags ['group_import'] end params do requires :path, type: String, desc: 'Group path' diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb index 72d67b41c31..c2b4cbf732f 100644 --- a/lib/api/group_packages.rb +++ b/lib/api/group_packages.rb @@ -14,13 +14,19 @@ module API helpers ::API::Helpers::PackagesHelpers params do - requires :id, type: String, desc: "Group's ID or path" + requires :id, types: [String, Integer], desc: 'ID or URL-encoded path of the group' optional :exclude_subgroups, type: Boolean, default: false, desc: 'Determines if subgroups should be excluded' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get all project packages within a group' do - detail 'This feature was introduced in GitLab 12.5' + desc 'List packages within a group' do + detail 'Get a list of project packages at the group level. This feature was introduced in GitLab 12.5' success ::API::Entities::Package + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Group Not Found' } + ] + is_array true + tags %w[group_packages] end params do use :pagination @@ -53,10 +59,13 @@ module API packages = Packages::GroupPackagesFinder.new( current_user, user_group, - declared(params).slice(:exclude_subgroups, :order_by, :sort, :package_type, :package_name, :include_versionless, :status) + declared(params).slice( + :exclude_subgroups, :order_by, :sort, :package_type, :package_name, :include_versionless, :status + ) ).execute - present paginate(packages), with: ::API::Entities::Package, user: current_user, group: true, namespace: user_group + present paginate(packages), with: ::API::Entities::Package, user: current_user, group: true, + namespace: user_group end end end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 2235746b254..a42f9045b9d 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -11,12 +11,14 @@ module API helpers ::API::Helpers::VariablesHelpers params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, type: String, desc: 'The ID of a group or URL-encoded path of the group owned by the authenticated + user' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get group-level variables' do + desc 'Get a list of group-level variables' do success Entities::Ci::Variable + tags %w[ci_variables] end params do use :pagination @@ -26,8 +28,10 @@ module API present paginate(variables), with: Entities::Ci::Variable end - desc 'Get a specific variable from a group' do + desc 'Get the details of a group’s specific variable' do success Entities::Ci::Variable + failure [{ code: 404, message: 'Group Variable Not Found' }] + tags %w[ci_variables] end params do requires :key, type: String, desc: 'The key of the variable' @@ -42,14 +46,19 @@ module API desc 'Create a new variable in a group' do success Entities::Ci::Variable + failure [{ code: 400, message: '400 Bad Request' }] + tags %w[ci_variables] end + route_setting :log_safety, { safe: %w[key], unsafe: %w[value] } params do - requires :key, type: String, desc: 'The key of the variable' - requires :value, type: String, desc: 'The value of the variable' + requires :key, type: String, desc: 'The ID of a group or URL-encoded path of the group owned by the + authenticated user' + requires :value, type: String, desc: 'The value of a variable' optional :protected, type: String, desc: 'Whether the variable is protected' optional :masked, type: String, desc: 'Whether the variable is masked' - optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' - + optional :raw, type: String, desc: 'Whether the variable will be expanded' + optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of the variable. Default: env_var' + optional :environment_scope, type: String, desc: 'The environment scope of a variable' use :optional_group_variable_params_ee end post ':id/variables' do @@ -73,13 +82,18 @@ module API desc 'Update an existing variable from a group' do success Entities::Ci::Variable + failure [{ code: 400, message: '400 Bad Request' }, { code: 404, message: 'Group Variable Not Found' }] + tags %w[ci_variables] end + route_setting :log_safety, { safe: %w[key], unsafe: %w[value] } params do - optional :key, type: String, desc: 'The key of the variable' - optional :value, type: String, desc: 'The value of the variable' + optional :key, type: String, desc: 'The key of a variable' + optional :value, type: String, desc: 'The value of a variable' optional :protected, type: String, desc: 'Whether the variable is protected' optional :masked, type: String, desc: 'Whether the variable is masked' - optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' + optional :raw, type: String, desc: 'Whether the variable will be expanded' + optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of the variable. Default: env_var' + optional :environment_scope, type: String, desc: 'The environment scope of a variable' use :optional_group_variable_params_ee end @@ -106,9 +120,11 @@ module API desc 'Delete an existing variable from a group' do success Entities::Ci::Variable + failure [{ code: 404, message: 'Group Variable Not Found' }] + tags %w[ci_variables] end params do - requires :key, type: String, desc: 'The key of the variable' + requires :key, type: String, desc: 'The key of a variable' end delete ':id/variables/:key' do variable = find_variable(user_group, params) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 0eb4fbb196c..75e7612bd5b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -592,19 +592,19 @@ module API end end - def present_artifacts_file!(file, project:, **args) + def present_artifacts_file!(file, **args) log_artifacts_filesize(file&.model) - present_carrierwave_file!(file, project: project, **args) + present_carrierwave_file!(file, **args) end - def present_carrierwave_file!(file, project: nil, supports_direct_download: true) + def present_carrierwave_file!(file, supports_direct_download: true) return not_found! unless file&.exists? if file.file_storage? present_disk_file!(file.path, file.filename) elsif supports_direct_download && file.class.direct_download_enabled? - redirect(cdn_fronted_url(file, project)) + redirect(cdn_fronted_url(file)) else header(*Gitlab::Workhorse.send_url(file.url)) status :ok @@ -612,9 +612,9 @@ module API end end - def cdn_fronted_url(file, project) + def cdn_fronted_url(file) if file.respond_to?(:cdn_enabled_url) - result = file.cdn_enabled_url(project, ip_address) + result = file.cdn_enabled_url(ip_address) Gitlab::ApplicationContext.push(artifact_used_cdn: result.used_cdn) result.url else @@ -673,7 +673,6 @@ module API finder_params[:with_issues_enabled] = true if params[:with_issues_enabled].present? finder_params[:with_merge_requests_enabled] = true if params[:with_merge_requests_enabled].present? - finder_params[:without_deleted] = true finder_params[:search_namespaces] = true if params[:search_namespaces].present? finder_params[:user] = params.delete(:user) if params[:user] finder_params[:id_after] = sanitize_id_param(params[:id_after]) if params[:id_after] diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index e03f029a6ef..56db6ee4c5c 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -124,7 +124,12 @@ module API repository: repository.gitaly_repository.to_h, address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags( + user: ::Feature::Gitaly.user_actor(actor.user), + repository: repository, + project: ::Feature::Gitaly.project_actor(repository.container), + group: ::Feature::Gitaly.group_actor(repository.container) + ) } end end diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index 8572cc89e71..b3ba962666f 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -137,9 +137,10 @@ module API end def create_service_params(parent) - if parent.is_a?(Project) + case parent + when Project { project: parent } - elsif parent.is_a?(Group) + when Group { group: parent } else raise TypeError, 'Parent type is not supported' diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb index 85648cd166d..eed9fa30d3c 100644 --- a/lib/api/helpers/merge_requests_helpers.rb +++ b/lib/api/helpers/merge_requests_helpers.rb @@ -8,6 +8,9 @@ module API UNPROCESSABLE_ERROR_KEYS = [:project_access, :branch_conflict, :validate_fork, :base].freeze + params :ee_approval_params do + end + params :merge_requests_negatable_params do optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' optional :author_username, type: String, desc: 'Return merge requests which are authored by the user with the given username' @@ -136,3 +139,5 @@ module API end end end + +API::Helpers::MergeRequestsHelpers.prepend_mod_with('API::Helpers::MergeRequestsHelpers') diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index dc81e5e1b51..1ae863a5a25 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -45,7 +45,7 @@ module API raise ArgumentError, "Can't find application setting for package_type #{package_type}" unless application_setting_name - if target.present? && Feature.enabled?(:cascade_package_forwarding_settings, target) + if target.present? target.public_send(application_setting_name) # rubocop:disable GitlabSecurity/PublicSend else ::Gitlab::CurrentSettings diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index 34e126c73fc..352d77f472c 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -19,7 +19,7 @@ module API strong_memoize(:project) do case endpoint_scope when :project - user_project + user_project(action: :read_package) when :instance # Simulate the same behavior as #user_project by re-using #find_project! # but take care if the project_id is nil as #find_project! is not designed diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index 687c8330cc8..96a10d43401 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -3,6 +3,8 @@ module API module Helpers module PackagesHelpers + extend ::Gitlab::Utils::Override + MAX_PACKAGE_FILE_SIZE = 50.megabytes.freeze def require_packages_enabled! @@ -48,6 +50,34 @@ module API require_gitlab_workhorse! end + override :user_project + def user_project(action: :read_project) + case action + when :read_project + super() + when :read_package + user_project_with_read_package + else + raise ArgumentError, "unexpected action: #{action}" + end + end + + # This function is similar to the `find_project!` function, but it considers the `read_package` ability. + def user_project_with_read_package + strong_memoize(:user_project_with_read_package) do + project = find_project(params[:id]) + + next forbidden! unless authorized_project_scope?(project) + + next project if can?(current_user, :read_package, project&.packages_policy_subject) + # guest users can have :read_project but not :read_package + next forbidden! if can?(current_user, :read_project, project) + next unauthorized! if authenticate_non_public? + + not_found!('Project') + end + end + def track_package_event(event_name, scope, **args) ::Packages::CreateEventService.new(nil, current_user, event_name: event_name, scope: scope).execute category = args.delete(:category) || self.options[:for].name diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 9839828a5b4..c95bf0f0c21 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -65,6 +65,7 @@ module API optional :suggestion_commit_message, type: String, desc: 'The commit message used to apply merge request suggestions' optional :merge_commit_template, type: String, desc: 'Template used to create merge commit message' optional :squash_commit_template, type: String, desc: 'Template used to create squash commit message' + optional :issue_branch_template, type: String, desc: 'Template used to create a branch from an issue' optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" optional :ci_default_git_depth, type: Integer, desc: 'Default number of revisions for shallow cloning' optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' @@ -96,7 +97,7 @@ module API end params :optional_update_params_ce do - optional :ci_forward_deployment_enabled, type: Boolean, desc: 'Skip older deployment jobs that are still pending' + optional :ci_forward_deployment_enabled, type: Boolean, desc: 'Prevent older deployment jobs that are still pending' optional :ci_allow_fork_pipelines_to_run_in_parent_project, type: Boolean, desc: 'Allow fork merge request pipelines to run in parent project' optional :ci_separated_caches, type: Boolean, desc: 'Enable or disable separated caches based on branch protection.' optional :restrict_user_defined_variables, type: Boolean, desc: 'Restrict use of user-defined variables when triggering a pipeline' @@ -174,6 +175,7 @@ module API :suggestion_commit_message, :merge_commit_template, :squash_commit_template, + :issue_branch_template, :repository_storage, :packages_enabled, :service_desk_enabled, diff --git a/lib/api/helpers/users_helpers.rb b/lib/api/helpers/users_helpers.rb index 1a019283bc6..e80b89488a2 100644 --- a/lib/api/helpers/users_helpers.rb +++ b/lib/api/helpers/users_helpers.rb @@ -18,6 +18,13 @@ module API error_messages[:bio] = error_messages.delete(:"user_detail.bio") if error_messages.has_key?(:"user_detail.bio") end end + + # rubocop: disable CodeReuse/ActiveRecord + def find_user_by_id(params) + id = params[:user_id] || params[:id] + User.find_by(id: id) || not_found!('User') + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/web_hooks_helpers.rb b/lib/api/helpers/web_hooks_helpers.rb index a71e56af4c3..5d5067bc70e 100644 --- a/lib/api/helpers/web_hooks_helpers.rb +++ b/lib/api/helpers/web_hooks_helpers.rb @@ -6,7 +6,7 @@ module API extend Grape::API::Helpers params :requires_url do - requires :url, type: String, desc: "The URL to send the request to" + requires :url, type: String, desc: "The URL to send the request to", documentation: { example: 'http://example.com/hook' } end params :optional_url do @@ -15,8 +15,8 @@ module API params :url_variables do optional :url_variables, type: Array, desc: 'URL variables for interpolation' do - requires :key, type: String, desc: 'Name of the variable' - requires :value, type: String, desc: 'Value of the variable' + requires :key, type: String, desc: 'Name of the variable', documentation: { example: 'token' } + requires :value, type: String, desc: 'Value of the variable', documentation: { example: '123' } end end diff --git a/lib/api/import_bitbucket_server.rb b/lib/api/import_bitbucket_server.rb index 0f2d6239d0d..f315ae5afff 100644 --- a/lib/api/import_bitbucket_server.rb +++ b/lib/api/import_bitbucket_server.rb @@ -22,6 +22,14 @@ module API desc 'Import a BitBucket Server repository' do detail 'This feature was introduced in GitLab 13.2.' success ::ProjectEntity + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 422, message: 'Unprocessable entity' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_import_bitbucket'] end params do diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index 493cc038f46..d742e3732a8 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -2,6 +2,8 @@ module API class ImportGithub < ::API::Base + before { authenticate! } + feature_category :importers urgency :low @@ -35,7 +37,15 @@ module API desc 'Import a GitHub project' do detail 'This feature was introduced in GitLab 11.3.4.' - success ::ProjectEntity + success code: 201, model: ::ProjectEntity + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 422, message: 'Unprocessable entity' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_import_github'] end params do requires :personal_access_token, type: String, desc: 'GitHub personal access token' @@ -56,6 +66,18 @@ module API end end + desc 'Cancel GitHub project import' do + detail 'This feature was introduced in GitLab 15.5' + success code: 200, model: ProjectImportEntity + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_import_github'] + end params do requires :project_id, type: Integer, desc: 'ID of importing project to be canceled' end diff --git a/lib/api/integrations.rb b/lib/api/integrations.rb index 71c55704ddf..408fa038b0d 100644 --- a/lib/api/integrations.rb +++ b/lib/api/integrations.rb @@ -3,6 +3,8 @@ module API class Integrations < ::API::Base feature_category :integrations + INTEGRATIONS_TAGS = %w[integrations].freeze + integrations = Helpers::IntegrationsHelpers.integrations integration_classes = Helpers::IntegrationsHelpers.integration_classes @@ -65,14 +67,21 @@ module API # The support for `:id/services` can be dropped if we create an API V5. [':id/services', ':id/integrations'].each do |path| params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authenticate! } before { authorize_admin_project } - desc 'Get all active project integrations' do + desc 'List all active integrations' do + detail 'Get a list of all active project integrations.' success Entities::ProjectIntegrationBasic + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags INTEGRATIONS_TAGS end get path do integrations = user_project.integrations.active @@ -81,7 +90,16 @@ module API end INTEGRATIONS.each do |slug, settings| - desc "Set #{slug} integration for project" + desc "Create/Edit #{slug.titleize} integration" do + detail "Set #{slug.titleize} integration for a project." + success Entities::ProjectIntegrationBasic + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags INTEGRATIONS_TAGS + end params do settings.each do |setting| if setting[:required] @@ -103,7 +121,16 @@ module API end end - desc "Delete an integration from a project" + desc "Disable an integration" do + detail "Disable the integration for a project. Integration settings are preserved." + success code: 204 + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags INTEGRATIONS_TAGS + end params do requires :slug, type: String, values: INTEGRATIONS.keys, desc: 'The name of the integration' end @@ -124,8 +151,15 @@ module API end end - desc 'Get the integration settings for a project' do + desc "Get an integration settings" do + detail "Get the integration settings for a project." success Entities::ProjectIntegration + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags INTEGRATIONS_TAGS end params do requires :slug, type: String, values: INTEGRATIONS.keys, desc: 'The name of the integration' @@ -149,11 +183,16 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Trigger a slash command for #{integration_slug}" do detail 'Added in GitLab 8.13' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags INTEGRATIONS_TAGS end params do settings.each do |setting| diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 6f964d5636b..d06d1e9862a 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -61,15 +61,6 @@ module API Guest.can?(:download_code, project) || agent.has_access_to?(project) end - def count_events - strong_memoize(:count_events) do - events = params.slice(:gitops_sync_count, :k8s_api_proxy_request_count) - events.transform_keys! { |event| event.to_s.chomp('_count') } - events = params[:counters]&.slice(:gitops_sync, :k8s_api_proxy_request) unless events.present? - events - end - end - def increment_unique_events events = params[:unique_counters]&.slice(:agent_users_using_ci_tunnel) @@ -77,6 +68,12 @@ module API increment_unique_values(event, entity_ids) end end + + def increment_count_events + events = params[:counters]&.slice(:gitops_sync, :k8s_api_proxy_request) + + Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_event_counts(events) + end end namespace 'internal' do @@ -144,26 +141,17 @@ module API detail 'Updates usage metrics for agent' end params do - # Todo: Remove gitops_sync_count and k8s_api_proxy_request_count in the next milestone - # https://gitlab.com/gitlab-org/gitlab/-/issues/369489 - # We're only keeping it for backwards compatibility until KAS is released - # using `counts:` instead - optional :gitops_sync_count, type: Integer, desc: 'The count to increment the gitops_sync metric by' - optional :k8s_api_proxy_request_count, type: Integer, desc: 'The count to increment the k8s_api_proxy_request_count metric by' optional :counters, type: Hash do optional :gitops_sync, type: Integer, desc: 'The count to increment the gitops_sync metric by' - optional :k8s_api_proxy_request, type: Integer, desc: 'The count to increment the k8s_api_proxy_request_count metric by' + optional :k8s_api_proxy_request, type: Integer, desc: 'The count to increment the k8s_api_proxy_request metric by' end - mutually_exclusive :counters, :gitops_sync_count - mutually_exclusive :counters, :k8s_api_proxy_request_count optional :unique_counters, type: Hash do optional :agent_users_using_ci_tunnel, type: Set[Integer], desc: 'A set of user ids that have interacted a CI Tunnel to' end end post '/' do - Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_event_counts(count_events) if count_events - + increment_count_events increment_unique_events no_content! diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb index 6be2679af14..771059053ac 100644 --- a/lib/api/internal/pages.rb +++ b/lib/api/internal/pages.rb @@ -5,6 +5,7 @@ module API module Internal class Pages < ::API::Base feature_category :pages + urgency :low before do authenticate_gitlab_pages_request! diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 6fb3eca0ba8..6aefdf146cf 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -18,11 +18,12 @@ module API desc 'Invite non-members by email address to a group or project.' do detail 'This feature was introduced in GitLab 13.6' success Entities::Invitation + tags %w[invitations] end params do requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' - optional :email, types: [String, Array[String]], email_or_email_list: true, desc: 'The email address to invite, or multiple emails separated by comma' - optional :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.' + optional :email, type: Array[String], email_or_email_list: true, coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The email address to invite, or multiple emails separated by comma' + optional :user_id, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api' optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' @@ -44,8 +45,12 @@ module API desc 'Get a list of group or project invitations viewable by the authenticated user' do detail 'This feature was introduced in GitLab 13.6' success Entities::Invitation + is_array true + tags %w[invitations] end params do + optional :page, type: Integer, desc: 'Page to retrieve' + optional :per_page, type: Integer, desc: 'Number of member invitations to return per page' optional :query, type: String, desc: 'A query string to search for members' use :pagination end @@ -62,6 +67,7 @@ module API desc 'Updates a group or project invitation.' do success Entities::Member + tags %w[invitations] end params do requires :email, type: String, desc: 'The email address of the invitation' @@ -93,7 +99,15 @@ module API end end - desc 'Removes an invitation from a group or project.' + desc 'Removes an invitation from a group or project.' do + success code: 204 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Could not delete invitation' } + ] + tags %w[invitations] + end params do requires :email, type: String, desc: 'The email address of the invitation' end diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index 563fb3358ed..020b02248a0 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -6,16 +6,27 @@ module API before { authenticate! } + ISSUE_LINKS_TAGS = %w[issue_links].freeze + feature_category :team_planning urgency :low params do - requires :id, type: String, desc: 'The ID of a project' - requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + requires :id, types: [String, Integer], + desc: 'The ID or URL-encoded path of the project owned by the authenticated user' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project’s issue' end resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get related issues' do + desc 'List issue relations' do + detail 'Get a list of a given issue’s linked issues, sorted by the relationship creation datetime (ascending).'\ + 'Issues are filtered according to the user authorizations.' success Entities::RelatedIssue + is_array true + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags ISSUE_LINKS_TAGS end get ':id/issues/:issue_iid/links' do source_issue = find_project_issue(params[:issue_iid]) @@ -30,14 +41,23 @@ module API include_subscribed: false end - desc 'Relate issues' do + desc 'Create an issue link' do + detail 'Creates a two-way relation between two issues.'\ + 'The user must be allowed to update both issues to succeed.' success Entities::IssueLink + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' } + ] + tags ISSUE_LINKS_TAGS end params do - requires :target_project_id, type: String, desc: 'The ID of the target project' - requires :target_issue_iid, type: Integer, desc: 'The IID of the target issue' + requires :target_project_id, types: [String, Integer], + desc: 'The ID or URL-encoded path of a target project' + requires :target_issue_iid, types: [String, Integer], desc: 'The internal ID of a target project’s issue' optional :link_type, type: String, values: IssueLink.link_types.keys, - desc: 'The type of the relation' + desc: 'The type of the relation (“relates_to”, “blocks”, “is_blocked_by”),'\ + 'defaults to “relates_to”)' end # rubocop: disable CodeReuse/ActiveRecord post ':id/issues/:issue_iid/links' do @@ -61,12 +81,17 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'Get issues relation' do - detail 'This feature was introduced in GitLab 15.1.' + desc 'Get an issue link' do + detail 'Gets details about an issue link. This feature was introduced in GitLab 15.1.' success Entities::IssueLink + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags ISSUE_LINKS_TAGS end params do - requires :issue_link_id, type: Integer, desc: 'The ID of an issue link' + requires :issue_link_id, types: [String, Integer], desc: 'ID of an issue relationship' end get ':id/issues/:issue_iid/links/:issue_link_id' do issue = find_project_issue(params[:issue_iid]) @@ -77,11 +102,17 @@ module API present issue_link, with: Entities::IssueLink end - desc 'Remove issues relation' do + desc 'Delete an issue link' do + detail 'Deletes an issue link, thus removes the two-way relationship.' success Entities::IssueLink + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags ISSUE_LINKS_TAGS end params do - requires :issue_link_id, type: Integer, desc: 'The ID of an issue link' + requires :issue_link_id, types: [String, Integer], desc: 'The ID of an issue relationship' end delete ':id/issues/:issue_iid/links/:issue_link_id' do issue = find_project_issue(params[:issue_iid]) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b8b4019765d..b08819e34e3 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -198,7 +198,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include TimeTrackingEndpoints diff --git a/lib/api/keys.rb b/lib/api/keys.rb index fb1bedd5e92..77952bac01a 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -9,8 +9,13 @@ module API resource :keys do desc 'Get single ssh key by id. Only available to admin users' do + detail 'Get SSH key with user by ID of an SSH key. Note only administrators can lookup SSH key with user by ID\ + of an SSH key' success Entities::SSHKeyWithUser end + params do + requires :id, types: [String, Integer], desc: 'The ID of an SSH key', documentation: { example: '2' } + end get ":id" do authenticated_as_admin! @@ -19,11 +24,14 @@ module API present key, with: Entities::SSHKeyWithUser, current_user: current_user end - desc 'Get SSH Key information' do + desc 'Get user by fingerprint of SSH key' do success Entities::UserWithAdmin + detail 'You can search for a user that owns a specific SSH key. Note only administrators can lookup SSH key\ + with the fingerprint of an SSH key' end params do - requires :fingerprint, type: String, desc: 'Search for a SSH fingerprint' + requires :fingerprint, type: String, desc: 'The fingerprint of an SSH key', + documentation: { example: 'ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1' } end get do authenticated_with_can_read_all_resources! diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 0a107a96d61..2e00affbbdf 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -15,7 +15,7 @@ module API label_id: API::NO_SLASH_URL_PART_REGEX) params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: LABEL_ENDPOINT_REQUIREMENTS do desc 'Get all labels of the project' do diff --git a/lib/api/lint.rb b/lib/api/lint.rb index f65ecf3b4a6..89787ba00c2 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -15,12 +15,18 @@ module API end namespace :ci do - desc 'Validation of .gitlab-ci.yml content' + desc 'Validates the .gitlab-ci.yml content' do + detail 'Checks if CI/CD YAML configuration is valid' + success code: 200, model: Entities::Ci::Lint::Result + tags %w[ci_lint] + end params do - requires :content, type: String, desc: 'Content of .gitlab-ci.yml' - optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response' - optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' + requires :content, type: String, desc: 'The CI/CD configuration content' + optional :include_merged_yaml, type: Boolean, desc: 'If the expanded CI/CD configuration should be included in the response' + optional :include_jobs, type: Boolean, desc: 'If the list of jobs should be included in the response. This is + false by default' end + post '/lint', urgency: :low do unauthorized! unless can_lint_ci? @@ -36,16 +42,21 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Validation of .gitlab-ci.yml content' do - detail 'This feature was introduced in GitLab 13.5.' + desc 'Validates a CI YAML configuration with a namespace' do + detail 'Checks if a project’s latest (HEAD of the project’s default branch) .gitlab-ci.yml configuration is + valid' + success Entities::Ci::Lint::Result + tags %w[ci_lint] end params do - optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.' - optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' + optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check. This is false by default' + optional :include_jobs, type: Boolean, desc: 'If the list of jobs that would exist in a static check or pipeline + simulation should be included in the response. This is false by default' optional :ref, type: String, desc: 'Branch or tag used to execute a dry run. Defaults to the default branch of the project. Only used when dry_run is true' end + get ':id/ci/lint', urgency: :low do - authorize! :download_code, user_project + authorize! :read_code, user_project if user_project.commit.present? content = user_project.repository.gitlab_ci_yml_for(user_project.commit.id, user_project.ci_config_path_or_default) @@ -60,15 +71,19 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Validation of .gitlab-ci.yml content' do - detail 'This feature was introduced in GitLab 13.6.' + desc 'Validate a CI YAML configuration with a namespace' do + detail 'Checks if CI/CD YAML configuration is valid. This endpoint has namespace specific context' + success code: 200, model: Entities::Ci::Lint::Result + tags %w[ci_lint] end params do requires :content, type: String, desc: 'Content of .gitlab-ci.yml' - optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.' - optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' - optional :ref, type: String, desc: 'Branch or tag used to execute a dry run. Defaults to the default branch of the project. Only used when dry_run is true' + optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check. This is false by default' + optional :include_jobs, type: Boolean, desc: 'If the list of jobs that would exist in a static check or pipeline + simulation should be included in the response. This is false by default' + optional :ref, type: String, desc: 'When dry_run is true, sets the branch or tag to use. Defaults to the project’s default branch when not set' end + post ':id/ci/lint', urgency: :low do authorize! :create_pipeline, user_project diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index 1f8255fd6a4..276560f3433 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -7,13 +7,19 @@ module API feature_category :team_planning params do - requires :text, type: String, desc: "The markdown text to render" - optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown" - optional :project, type: String, desc: "The full path of a project to use as the context when creating references using GitLab Flavored Markdown" + requires :text, type: String, desc: "The Markdown text to render" + optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown. Default is false" + optional :project, type: String, desc: "Use project as a context when creating references using GitLab Flavored Markdown" end resource :markdown do - desc "Render markdown text" do + desc "Render an arbitrary Markdown document" do detail "This feature was introduced in GitLab 11.0." + success ::API::Entities::Markdown + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' } + ] + tags %w[markdown] end post do context = { only_path: false, current_user: current_user } @@ -29,7 +35,7 @@ module API context[:skip_project_check] = true end - { html: Banzai.render_and_post_process(params[:text], context) } + present({ html: Banzai.render_and_post_process(params[:text], context) }, with: Entities::Markdown) end end end diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 72313d6a588..30cdaba76ba 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -220,7 +220,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Download the maven package file' do @@ -232,18 +232,20 @@ module API end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + project = user_project(action: :read_package) + # return a similar failure to user_project - unless Feature.enabled?(:maven_central_request_forwarding, user_project&.root_ancestor) + unless Feature.enabled?(:maven_central_request_forwarding, project&.root_ancestor) not_found!('Project') unless path_exists?(params[:path]) end - authorize_read_package!(user_project) + authorize_read_package!(project) file_name, format = extract_format(params[:file_name]) - package = fetch_package(file_name: file_name, project: user_project) + package = fetch_package(file_name: file_name, project: project) - find_and_present_package_file(package, file_name, format, params.merge(target: user_project)) + find_and_present_package_file(package, file_name, format, params.merge(target: project)) end desc 'Workhorse authorize the maven package file upload' do @@ -268,7 +270,7 @@ module API params do requires :path, type: String, desc: 'Package path' requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.maven_file_name_regex - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true put ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb index 71ca8331ed6..7622ec717cc 100644 --- a/lib/api/merge_request_approvals.rb +++ b/lib/api/merge_request_approvals.rb @@ -6,10 +6,9 @@ module API feature_category :source_code_management - helpers do - params :ee_approval_params do - end + helpers ::API::Helpers::MergeRequestsHelpers + helpers do def present_approval(merge_request) present merge_request, with: ::API::Entities::MergeRequestApprovals, current_user: current_user end @@ -24,7 +23,12 @@ module API # merge_request_iid (required) - IID of MR # Examples: # GET /projects/:id/merge_requests/:merge_request_iid/approvals - desc 'List approvals for merge request' + desc 'List approvals for merge request' do + success ::API::Entities::MergeRequestApprovals + failure [ + { code: 404, message: 'Not found' } + ] + end get 'approvals', urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -39,7 +43,13 @@ module API # Examples: # POST /projects/:id/merge_requests/:merge_request_iid/approve # - desc 'Approve a merge request' + desc 'Approve a merge request' do + success code: 201, model: ::API::Entities::MergeRequestApprovals + failure [ + { code: 404, message: 'Not found' }, + { code: 401, message: 'Unauthorized' } + ] + end params do optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' @@ -60,7 +70,13 @@ module API present_approval(merge_request) end - desc 'Remove an approval from a merge request' + desc 'Remove an approval from a merge request' do + success code: 201, model: ::API::Entities::MergeRequestApprovals + failure [ + { code: 404, message: 'Not found' }, + { code: 401, message: 'Unauthorized' } + ] + end post 'unapprove', urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid], :approve_merge_request) diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 87623568a04..c7f0f88eacc 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -10,16 +10,18 @@ module API feature_category :code_review params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of merge request diff versions' do detail 'This feature was introduced in GitLab 8.12.' success Entities::MergeRequestDiff + tags %w[merge_requests] + is_array true end params do - requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The internal ID of the merge request' use :pagination end get ":id/merge_requests/:merge_request_iid/versions" do @@ -31,11 +33,12 @@ module API desc 'Get a single merge request diff version' do detail 'This feature was introduced in GitLab 8.12.' success Entities::MergeRequestDiffFull + tags %w[merge_requests] end params do - requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' - requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' + requires :merge_request_iid, type: Integer, desc: 'The internal ID of the merge request' + requires :version_id, type: Integer, desc: 'The ID of the merge request diff version' end get ":id/merge_requests/:merge_request_iid/versions/:version_id", urgency: :low do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index a0e7d0b10cd..bb2861aa221 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -170,7 +170,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include TimeTrackingEndpoints diff --git a/lib/api/metadata.rb b/lib/api/metadata.rb index 3e42ffe336a..788d9843c63 100644 --- a/lib/api/metadata.rb +++ b/lib/api/metadata.rb @@ -9,6 +9,8 @@ module API before { authenticate! } + METADATA_TAGS = %w[metadata].freeze + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned METADATA_QUERY = <<~EOF @@ -21,6 +23,7 @@ module API externalUrl version } + enterprise } } EOF @@ -35,30 +38,13 @@ module API end end - desc 'Retrieve metadata information for this GitLab instance.' do + desc 'Retrieve metadata information for this GitLab instance' do detail 'This feature was introduced in GitLab 15.2.' - success [ - { - code: 200, - model: Entities::Metadata, - message: 'successful operation', - examples: { - successful_response: { - 'value' => { - version: "15.0-pre", - revision: "c401a659d0c", - kas: { - enabled: true, - externalUrl: "grpc://gitlab.example.com:8150", - version: "15.0.0" - } - } - } - } - } + success Entities::Metadata + failure [ + { code: 401, message: 'Unauthorized' } ] - failure [{ code: 401, message: 'unauthorized operation' }] - tags %w[metadata] + tags METADATA_TAGS end get '/metadata' do run_metadata_query @@ -66,31 +52,14 @@ module API # Support the deprecated `/version` route. # See https://gitlab.com/gitlab-org/gitlab/-/issues/366287 - desc 'Get the version information of the GitLab instance.' do + desc 'Retrieves version information for the GitLab instance' do detail 'This feature was introduced in GitLab 8.13 and deprecated in 15.5. ' \ 'We recommend you instead use the Metadata API.' - success [ - { - code: 200, - model: Entities::Metadata, - message: 'successful operation', - examples: { - 'Example' => { - 'value' => { - version: "15.0-pre", - revision: "c401a659d0c", - kas: { - enabled: true, - externalUrl: "grpc://gitlab.example.com:8150", - version: "15.0.0" - } - } - } - } - } + success Entities::Metadata + failure [ + { code: 401, message: 'Unauthorized' } ] - failure [{ code: 401, message: 'unauthorized operation' }] - tags %w[metadata] + tags METADATA_TAGS end get '/version' do diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb index 478adcdce70..6ba154191be 100644 --- a/lib/api/metrics/dashboard/annotations.rb +++ b/lib/api/metrics/dashboard/annotations.rb @@ -7,8 +7,15 @@ module API feature_category :metrics urgency :low - desc 'Create a new monitoring dashboard annotation' do + desc 'Create a new annotation' do + detail 'Creates a new monitoring dashboard annotation' success Entities::Metrics::Dashboard::Annotation + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[dashboard_annotations] end ANNOTATIONS_SOURCES = [ @@ -20,12 +27,16 @@ module API resource annotations_source[:resource] do params do requires :starting_at, type: DateTime, - desc: 'Date time indicating starting moment to which the annotation relates.' + desc: 'Date time string, ISO 8601 formatted, such as 2016-03-11T03:45:40Z.'\ + 'Timestamp marking start point of annotation.' optional :ending_at, type: DateTime, - desc: 'Date time indicating ending moment to which the annotation relates.' + desc: 'Date time string, ISO 8601 formatted, such as 2016-03-11T03:45:40Z.'\ + 'Timestamp marking end point of annotation.'\ + 'When not supplied, an annotation displays as a single event at the start point.' requires :dashboard_path, type: String, coerce_with: -> (val) { CGI.unescape(val) }, - desc: 'The path to a file defining the dashboard on which the annotation should be added' - requires :description, type: String, desc: 'The description of the annotation' + desc: 'ID of the dashboard which needs to be annotated.'\ + 'Treated as a CGI-escaped path, and automatically un-escaped.' + requires :description, type: String, desc: 'Description of the annotation.' end post ':id/metrics_dashboard/annotations' do @@ -33,7 +44,9 @@ module API forbidden! unless can?(current_user, :create_metrics_dashboard_annotation, annotations_source_object) - create_service_params = declared(params).merge(annotations_source[:create_service_param_key] => annotations_source_object) + create_service_params = declared(params).merge( + annotations_source[:create_service_param_key] => annotations_source_object + ) result = ::Metrics::Dashboard::Annotations::CreateService.new(current_user, create_service_params).execute diff --git a/lib/api/metrics/user_starred_dashboards.rb b/lib/api/metrics/user_starred_dashboards.rb index 4d5396acccb..0a91e914d52 100644 --- a/lib/api/metrics/user_starred_dashboards.rb +++ b/lib/api/metrics/user_starred_dashboards.rb @@ -6,14 +6,22 @@ module API feature_category :metrics urgency :low + USER_STARRED_DASHBOARDS_TAGS = %w[user_starred_dashboards].freeze + resource :projects do - desc 'Marks selected metrics dashboard as starred' do + desc 'Add a star to a dashboard' do + detail 'Marks selected metrics dashboard as starred. Introduced in GitLab 13.0.' success Entities::Metrics::UserStarredDashboard + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' } + ] + tags USER_STARRED_DASHBOARDS_TAGS end params do requires :dashboard_path, type: String, allow_blank: false, coerce_with: ->(val) { CGI.unescape(val) }, - desc: 'Url encoded path to a file defining the dashboard to which the star should be added' + desc: 'URL-encoded path to file defining the dashboard which should be marked as favorite' end post ':id/metrics/user_starred_dashboards' do @@ -26,7 +34,15 @@ module API end end - desc 'Remove star from selected metrics dashboard' + desc 'Remove a star from a dashboard' do + detail 'Remove star from selected metrics dashboard. Introduced in GitLab 13.0.' + success code: 200 + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' } + ] + tags USER_STARRED_DASHBOARDS_TAGS + end params do optional :dashboard_path, type: String, allow_blank: false, coerce_with: ->(val) { CGI.unescape(val) }, diff --git a/lib/api/ml/mlflow.rb b/lib/api/ml/mlflow.rb index 2ffb04ebcbd..56bfac1530e 100644 --- a/lib/api/ml/mlflow.rb +++ b/lib/api/ml/mlflow.rb @@ -68,10 +68,19 @@ module API def find_candidate!(iid) candidate_repository.by_iid(iid) || resource_not_found! end + + def packages_url + path = api_v4_projects_packages_generic_package_version_path( + id: user_project.id, package_name: '', file_name: '' + ) + path = path.delete_suffix('/package_version') + + "#{request.base_url}#{path}" + end end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'API to interface with MLFlow Client, REST API version 1.28.0' do @@ -130,8 +139,7 @@ module API resource :runs do desc 'Creates a Run.' do success Entities::Ml::Mlflow::Run - detail ['https://www.mlflow.org/docs/1.28.0/rest-api.html#create-run', - 'MLFlow Runs map to GitLab Candidates'] + detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#create-run' end params do requires :experiment_id, type: Integer, @@ -143,7 +151,8 @@ module API optional :tags, type: Array, desc: 'This will be ignored' end post 'create', urgency: :low do - present candidate_repository.create!(experiment, params[:start_time]), with: Entities::Ml::Mlflow::Run + present candidate_repository.create!(experiment, params[:start_time]), + with: Entities::Ml::Mlflow::Run, packages_url: packages_url end desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do @@ -155,13 +164,12 @@ module API optional :run_uuid, type: String, desc: 'This parameter is ignored' end get 'get', urgency: :low do - present candidate, with: Entities::Ml::Mlflow::Run + present candidate, with: Entities::Ml::Mlflow::Run, packages_url: packages_url end desc 'Updates a Run.' do success Entities::Ml::Mlflow::UpdateRun - detail ['https://www.mlflow.org/docs/1.28.0/rest-api.html#update-run', - 'MLFlow Runs map to GitLab Candidates'] + detail 'MLFlow Runs map to GitLab Candidates. https://www.mlflow.org/docs/1.28.0/rest-api.html#update-run' end params do requires :run_id, type: String, desc: 'UUID of the candidate.' @@ -174,7 +182,7 @@ module API post 'update', urgency: :low do candidate_repository.update(candidate, params[:status], params[:end_time]) - present candidate, with: Entities::Ml::Mlflow::UpdateRun + present candidate, with: Entities::Ml::Mlflow::UpdateRun, packages_url: packages_url end desc 'Logs a metric to a run.' do diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 166c0b755fe..494b493f5e0 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -11,7 +11,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end namespace 'projects/:id/packages/npm' do desc 'Download the NPM tarball' do diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 3e05ea13311..d549a8be035 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -35,7 +35,7 @@ module API helpers do params :file_params do - requires :package, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :package, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end def project_or_group @@ -91,7 +91,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project', regexp: ::API::Concerns::Packages::NugetEndpoints::POSITIVE_INTEGER_REGEX + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project', regexp: ::API::Concerns::Packages::NugetEndpoints::POSITIVE_INTEGER_REGEX end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/nuget' do diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index 278dc4c2044..bb9f96cdbb1 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -8,19 +8,23 @@ module API authorize_packages_access!(user_project) end + PACKAGE_FILES_TAGS = %w[package_files].freeze + feature_category :package_registry urgency :low helpers ::API::Helpers::PackagesHelpers params do - requires :id, type: String, desc: 'The ID of a project' - requires :package_id, type: Integer, desc: 'The ID of a package' + requires :id, types: [String, Integer], desc: 'ID or URL-encoded path of the project' + requires :package_id, type: Integer, desc: 'ID of a package' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get all package files' do - detail 'This feature was introduced in GitLab 11.8' + desc 'List package files' do + detail 'Get a list of package files of a single package' success ::API::Entities::PackageFile + is_array true + tags PACKAGE_FILES_TAGS end params do use :pagination @@ -35,11 +39,17 @@ module API present paginate(package_files), with: ::API::Entities::PackageFile end - desc 'Remove a package file' do + desc 'Delete a package file' do detail 'This feature was introduced in GitLab 13.12' + success code: 204 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags PACKAGE_FILES_TAGS end params do - requires :package_file_id, type: Integer, desc: 'The ID of a package file' + requires :package_file_id, type: Integer, desc: 'ID of a package file' end delete ':id/packages/:package_id/package_files/:package_file_id' do authorize_destroy_package!(user_project) diff --git a/lib/api/pages.rb b/lib/api/pages.rb index 5f695f3853d..7e230bd3c67 100644 --- a/lib/api/pages.rb +++ b/lib/api/pages.rb @@ -10,7 +10,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Unpublish pages' do diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 9cf61967ba4..967847a8e62 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -54,7 +54,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb index bdb69d0ba44..c3505780396 100644 --- a/lib/api/pagination_params.rb +++ b/lib/api/pagination_params.rb @@ -17,8 +17,9 @@ module API included do helpers do params :pagination do - optional :page, type: Integer, default: 1, desc: 'Current page number' - optional :per_page, type: Integer, default: 20, desc: 'Number of items per page', except_values: [0] + optional :page, type: Integer, default: 1, desc: 'Current page number', documentation: { example: 1 } + optional :per_page, type: Integer, default: 20, + desc: 'Number of items per page', except_values: [0], documentation: { example: 20 } end def verify_pagination_params! diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb index a2903faa4ad..66930ecd797 100644 --- a/lib/api/personal_access_tokens.rb +++ b/lib/api/personal_access_tokens.rb @@ -6,24 +6,6 @@ module API feature_category :authentication_and_authorization - desc 'Get all Personal Access Tokens' do - detail 'This feature was added in GitLab 13.3' - success Entities::PersonalAccessToken - end - params do - optional :user_id, type: Integer, desc: 'Filter PATs by User ID' - optional :revoked, type: Boolean, desc: 'Filter PATs where revoked state matches parameter' - optional :state, type: String, desc: 'Filter PATs which are either active or not', - values: %w[active inactive] - optional :created_before, type: DateTime, desc: 'Filter PATs which were created before given datetime' - optional :created_after, type: DateTime, desc: 'Filter PATs which were created after given datetime' - optional :last_used_before, type: DateTime, desc: 'Filter PATs which were used before given datetime' - optional :last_used_after, type: DateTime, desc: 'Filter PATs which were used after given datetime' - optional :search, type: String, desc: 'Filters PATs by its name' - - use :pagination - end - before do authenticate! restrict_non_admins! unless current_user.can_admin_all_resources? @@ -32,12 +14,47 @@ module API helpers ::API::Helpers::PersonalAccessTokensHelpers resources :personal_access_tokens do + desc 'List personal access tokens' do + detail 'Get all personal access tokens the authenticated user has access to.' + is_array true + success Entities::PersonalAccessToken + tags %w[personal_access_tokens] + failure [ + { code: 401, message: 'Unauthorized' } + ] + end + params do + optional :user_id, type: Integer, desc: 'Filter PATs by User ID', documentation: { example: 2 } + optional :revoked, type: Boolean, desc: 'Filter PATs where revoked state matches parameter', + documentation: { example: false } + optional :state, type: String, desc: 'Filter PATs which are either active or not', + values: %w[active inactive], documentation: { example: 'active' } + optional :created_before, type: DateTime, desc: 'Filter PATs which were created before given datetime', + documentation: { example: '2022-01-01' } + optional :created_after, type: DateTime, desc: 'Filter PATs which were created after given datetime', + documentation: { example: '2021-01-01' } + optional :last_used_before, type: DateTime, desc: 'Filter PATs which were used before given datetime', + documentation: { example: '2021-01-01' } + optional :last_used_after, type: DateTime, desc: 'Filter PATs which were used after given datetime', + documentation: { example: '2022-01-01' } + optional :search, type: String, desc: 'Filters PATs by its name', documentation: { example: 'token' } + + use :pagination + end get do tokens = PersonalAccessTokensFinder.new(finder_params(current_user), current_user).execute present paginate(tokens), with: Entities::PersonalAccessToken end + desc 'Get single personal access token' do + detail 'Get a personal access token by using the ID of the personal access token.' + success Entities::PersonalAccessToken + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + end get ':id' do token = PersonalAccessToken.find_by_id(params[:id]) @@ -51,6 +68,13 @@ module API end end + desc 'Revoke a personal access token' do + detail 'Revoke a personal access token by using the ID of the personal access token.' + success code: 204 + failure [ + { code: 400, message: 'Bad Request' } + ] + end delete ':id' do token = find_token(params[:id]) diff --git a/lib/api/personal_access_tokens/self_information.rb b/lib/api/personal_access_tokens/self_information.rb index 89850614f94..5735fe49f33 100644 --- a/lib/api/personal_access_tokens/self_information.rb +++ b/lib/api/personal_access_tokens/self_information.rb @@ -17,10 +17,28 @@ module API before { authenticate! } resource :personal_access_tokens do + desc "Get single personal access token" do + detail 'Get the details of a personal access token by passing it to the API in a header' + success code: 200, model: Entities::PersonalAccessToken + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[personal_access_tokens] + end get 'self' do present access_token, with: Entities::PersonalAccessToken end + desc "Revoke a personal access token" do + detail 'Revoke a personal access token by passing it to the API in a header' + success code: 204 + failure [ + { code: 400, message: 'Bad Request' } + ] + tags %w[personal_access_tokens] + end + delete 'self' do revoke_token(access_token) end diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index 4644d38ea80..21f1ee69613 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -13,12 +13,17 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of the project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get all clusters from the project' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'List project clusters' do + detail 'This feature was introduced in GitLab 11.7. Returns a list of project clusters.' success Entities::Cluster + failure [ + { code: 403, message: 'Forbidden' } + ] + is_array true + tags %w[clusters] end params do use :pagination @@ -29,9 +34,14 @@ module API present paginate(clusters_for_current_user), with: Entities::Cluster end - desc 'Get specific cluster for the project' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Get a single project cluster' do + detail 'This feature was introduced in GitLab 11.7. Gets a single project cluster.' success Entities::ClusterProject + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: 'The cluster ID' @@ -42,9 +52,15 @@ module API present cluster, with: Entities::ClusterProject end - desc 'Adds an existing cluster' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Add existing cluster to project' do + detail 'This feature was introduced in GitLab 11.7. Adds an existing Kubernetes cluster to the project.' success Entities::ClusterProject + failure [ + { code: 400, message: 'Validation error' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :name, type: String, desc: 'Cluster name' @@ -76,9 +92,15 @@ module API end end - desc 'Update an existing cluster' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Edit project cluster' do + detail 'This feature was introduced in GitLab 11.7. Updates an existing project cluster.' success Entities::ClusterProject + failure [ + { code: 400, message: 'Validation error' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: 'The cluster ID' @@ -108,9 +130,14 @@ module API end end - desc 'Remove a cluster' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Delete project cluster' do + detail 'This feature was introduced in GitLab 11.7. Deletes an existing project cluster. Does not remove existing resources within the connected Kubernetes cluster.' success Entities::ClusterProject + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[clusters] end params do requires :cluster_id, type: Integer, desc: 'The Cluster ID' diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 6a6275ed02a..c5add42decc 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -16,7 +16,7 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end route_setting :authentication, job_token_allowed: true, job_token_scope: :project resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -47,8 +47,12 @@ module API end delete ':id/registry/repositories/:repository_id', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_admin_container_image! + repository.delete_scheduled! + + unless Feature.enabled?(:container_registry_delete_repository_with_cron_worker) + DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker + end - DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker track_package_event('delete_repository', :container, user: current_user, project: user_project, namespace: user_project.namespace) status :accepted diff --git a/lib/api/project_debian_distributions.rb b/lib/api/project_debian_distributions.rb index b8ca9428fa3..1e27f5c8856 100644 --- a/lib/api/project_debian_distributions.rb +++ b/lib/api/project_debian_distributions.rb @@ -3,7 +3,7 @@ module API class ProjectDebianDistributions < ::API::Base params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end before do diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb index e8829216336..d90ce32c354 100644 --- a/lib/api/project_events.rb +++ b/lib/api/project_events.rb @@ -12,10 +12,15 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' + optional :action, type: String, desc: 'Include only events of a particular action type' + optional :target_type, type: String, desc: 'Include only events of a particular target type' + optional :before, type: DateTime, desc: 'Include only events created before a particular date' + optional :after, type: DateTime, desc: 'Include only events created after a particular date' + optional :sort, type: String, desc: 'Sort events in asc or desc order by created_at. Default is desc' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc "List a Project's visible events" do + desc "List a project's visible events" do success Entities::Event end params do diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 29fdfe45566..e4e950fb603 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -11,12 +11,19 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get export status' do detail 'This feature was introduced in GitLab 10.6.' - success Entities::ProjectExportStatus + success code: 200, model: Entities::ProjectExportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] end get ':id/export' do present user_project, with: Entities::ProjectExportStatus @@ -24,6 +31,15 @@ module API desc 'Download export' do detail 'This feature was introduced in GitLab 10.6.' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] + produces %w[application/octet-stream application/json] end get ':id/export/download' do check_rate_limit! :project_download_export, scope: [current_user, user_project.namespace] @@ -41,6 +57,16 @@ module API desc 'Start export' do detail 'This feature was introduced in GitLab 10.6.' + success code: 202 + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 429, message: 'Too many requests' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] end params do optional :description, type: String, desc: 'Override the project description' @@ -86,6 +112,15 @@ module API desc 'Start relations export' do detail 'This feature was introduced in GitLab 14.4' + success code: 202 + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] end post ':id/export_relations' do response = ::BulkImports::ExportService.new(portable: user_project, user: current_user).execute @@ -93,12 +128,23 @@ module API if response.success? accepted! else - render_api_error!(message: 'Project relations export could not be started.') + render_api_error!('Project relations export could not be started.', 500) end end desc 'Download relations export' do detail 'This feature was introduced in GitLab 14.4' + success code: 200 + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 500, message: 'Internal Server Error' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] + produces %w[application/octet-stream application/json] end params do requires :relation, @@ -119,6 +165,15 @@ module API desc 'Relations export status' do detail 'This feature was introduced in GitLab 14.4' + is_array true + success code: 200, model: Entities::BulkImports::ExportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] end get ':id/export_relations/status' do present user_project.bulk_import_exports, with: Entities::BulkImports::ExportStatus diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 466e80d68c8..ced8ecec883 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -4,6 +4,8 @@ module API class ProjectHooks < ::API::Base include PaginationParams + project_hooks_tags = %w[project_hooks] + before { authenticate! } before { authorize_admin_project } @@ -37,15 +39,18 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/hooks' do mount ::API::Hooks::UrlVariables end - desc 'Get project hooks' do + desc 'List project hooks' do + detail 'Get a list of project hooks' success Entities::ProjectHook + is_array true + tags project_hooks_tags end params do use :pagination @@ -54,8 +59,13 @@ module API present paginate(user_project.hooks), with: Entities::ProjectHook end - desc 'Get a project hook' do + desc 'Get project hook' do + detail 'Get a specific hook for a project' success Entities::ProjectHook + failure [ + { code: 404, message: 'Not found' } + ] + tags project_hooks_tags end params do requires :hook_id, type: Integer, desc: 'The ID of a project hook' @@ -65,8 +75,15 @@ module API present hook, with: Entities::ProjectHook end - desc 'Add hook to project' do + desc 'Add project hook' do + detail 'Adds a hook to a specified project' success Entities::ProjectHook + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags project_hooks_tags end params do use :requires_url @@ -79,11 +96,18 @@ module API save_hook(hook, Entities::ProjectHook) end - desc 'Update an existing hook' do + desc 'Edit project hook' do + detail 'Edits a hook for a specified project.' success Entities::ProjectHook + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags project_hooks_tags end params do - requires :hook_id, type: Integer, desc: "The ID of the hook to update" + requires :hook_id, type: Integer, desc: 'The ID of the project hook' use :optional_url use :common_hook_parameters end @@ -91,11 +115,16 @@ module API update_hook(entity: Entities::ProjectHook) end - desc 'Deletes project hook' do + desc 'Delete a project hook' do + detail 'Removes a hook from a project. This is an idempotent method and can be called multiple times. Either the hook is available or not.' success Entities::ProjectHook + failure [ + { code: 404, message: 'Not found' } + ] + tags project_hooks_tags end params do - requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' + requires :hook_id, type: Integer, desc: 'The ID of the project hook' end delete ":id/hooks/:hook_id" do hook = find_hook diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 0da8c1ecedd..02f0d9a2a70 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -40,6 +40,7 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Workhorse authorize the project import upload' do detail 'This feature was introduced in GitLab 12.9' + tags ['project_import'] end post 'import/authorize' do require_gitlab_workhorse! @@ -77,7 +78,16 @@ module API end desc 'Create a new project import' do detail 'This feature was introduced in GitLab 10.6.' - success Entities::ProjectImportStatus + success code: 201, model: Entities::ProjectImportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_import'] + consumes ['multipart/form-data'] end post 'import' do require_gitlab_workhorse! @@ -108,11 +118,19 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end desc 'Get a project import status' do detail 'This feature was introduced in GitLab 10.6.' - success Entities::ProjectImportStatus + success code: 200, model: Entities::ProjectImportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_import'] end route_setting :skip_authentication, true get ':id/import' do @@ -133,7 +151,17 @@ module API end desc 'Create a new project import using a remote object storage path' do detail 'This feature was introduced in GitLab 13.2.' - success Entities::ProjectImportStatus + consumes ['multipart/form-data'] + tags ['project_import'] + success code: 201, model: Entities::ProjectImportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' }, + { code: 429, message: 'Too many requests' }, + { code: 503, message: 'Service unavailable' } + ] end post 'remote-import' do check_rate_limit! :project_import, scope: [current_user, :project_import] @@ -176,7 +204,17 @@ module API end desc 'Create a new project import using a file from AWS S3' do detail 'This feature was introduced in GitLab 14.9.' - success Entities::ProjectImportStatus + consumes ['multipart/form-data'] + tags ['project_import'] + success code: 201, model: Entities::ProjectImportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' }, + { code: 429, message: 'Too many requests' }, + { code: 503, message: 'Service unavailable' } + ] end post 'remote-import-s3' do not_found! unless ::Feature.enabled?(:import_project_from_remote_file_s3) diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 9f82dbf9813..a7a583aaa23 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -11,7 +11,7 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of project milestones' do diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 800966408fc..d09c481403f 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -14,7 +14,7 @@ module API helpers ::API::Helpers::PackagesHelpers params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all project packages' do diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb index ab5d8b3a888..5777b8754e7 100644 --- a/lib/api/project_repository_storage_moves.rb +++ b/lib/api/project_repository_storage_moves.rb @@ -11,7 +11,8 @@ module API resource :project_repository_storage_moves do desc 'Get a list of all project repository storage moves' do detail 'This feature was introduced in GitLab 13.0.' - success Entities::Projects::RepositoryStorageMove + is_array true + success code: 200, model: Entities::Projects::RepositoryStorageMove end params do use :pagination @@ -24,7 +25,7 @@ module API desc 'Get a project repository storage move' do detail 'This feature was introduced in GitLab 13.0.' - success Entities::Projects::RepositoryStorageMove + success code: 200, model: Entities::Projects::RepositoryStorageMove end params do requires :repository_storage_move_id, type: Integer, desc: 'The ID of a project repository storage move' @@ -37,6 +38,7 @@ module API desc 'Schedule bulk project repository storage moves' do detail 'This feature was introduced in GitLab 13.7.' + success code: 202 end params do requires :source_storage_name, type: String, desc: 'The source storage shard', values: -> { Gitlab.config.repositories.storages.keys } @@ -53,12 +55,13 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of all project repository storage moves' do detail 'This feature was introduced in GitLab 13.1.' - success Entities::Projects::RepositoryStorageMove + is_array true + success code: 200, model: Entities::Projects::RepositoryStorageMove end params do use :pagination @@ -71,7 +74,7 @@ module API desc 'Get a project repository storage move' do detail 'This feature was introduced in GitLab 13.1.' - success Entities::Projects::RepositoryStorageMove + success code: 200, model: Entities::Projects::RepositoryStorageMove end params do requires :repository_storage_move_id, type: Integer, desc: 'The ID of a project repository storage move' @@ -84,14 +87,14 @@ module API desc 'Schedule a project repository storage move' do detail 'This feature was introduced in GitLab 13.1.' - success Entities::Projects::RepositoryStorageMove + success code: 201, model: Entities::Projects::RepositoryStorageMove end params do optional :destination_storage_name, type: String, desc: 'The destination storage shard' end post ':id/repository_storage_moves' do storage_move = user_project.repository_storage_moves.build( - declared_params.merge(source_storage_name: user_project.repository_storage) + declared_params.compact.merge(source_storage_name: user_project.repository_storage) ) if storage_move.schedule diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb index d33d2976b1c..d2ed7f75fb7 100644 --- a/lib/api/project_snapshots.rb +++ b/lib/api/project_snapshots.rb @@ -11,6 +11,11 @@ module API resource :projects do desc 'Download a (possibly inconsistent) snapshot of a repository' do detail 'This feature was introduced in GitLab 10.7' + success File + produces 'application/x-tar' + failure [ + { code: 401, message: 'Unauthorized' } + ] end params do optional :wiki, type: Boolean, desc: 'Set to true to receive the wiki repository' diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 14792730eae..93ffb23fea8 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -9,7 +9,7 @@ module API feature_category :snippets params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers Helpers::SnippetsHelpers @@ -34,6 +34,11 @@ module API desc 'Get all project snippets' do success Entities::ProjectSnippet + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[project_snippets] + is_array true end params do use :pagination @@ -46,6 +51,10 @@ module API desc 'Get a single project snippet' do success Entities::ProjectSnippet + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[project_snippets] end params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' @@ -60,6 +69,12 @@ module API desc 'Create a new project snippet' do success Entities::ProjectSnippet + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[project_snippets] end params do requires :title, type: String, allow_blank: false, desc: 'The title of the snippet' @@ -91,6 +106,12 @@ module API desc 'Update an existing project snippet' do success Entities::ProjectSnippet + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[project_snippets] end params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' @@ -132,7 +153,14 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'Delete a project snippet' + desc 'Delete a project snippet' do + success code: 204 + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' } + ] + tags %w[project_snippets] + end params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' end @@ -156,7 +184,13 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'Get a raw project snippet' + desc 'Get a raw project snippet' do + success Entities::ProjectSnippet + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[project_snippets] + end params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' end @@ -168,7 +202,13 @@ module API present content_for(snippet) end - desc 'Get raw project snippet file contents from the repository' + desc 'Get raw project snippet file contents from the repository' do + success Entities::ProjectSnippet + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[project_snippets] + end params do use :raw_file_params end @@ -182,6 +222,10 @@ module API desc 'Get the user agent details for a project snippet' do success Entities::UserAgentDetail + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[project_snippets] end params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb index 3db8d20ebac..859e53b6981 100644 --- a/lib/api/project_statistics.rb +++ b/lib/api/project_statistics.rb @@ -10,10 +10,18 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get the list of project fetch statistics for the last 30 days' + desc 'Get the list of project fetch statistics for the last 30 days' do + success Entities::ProjectDailyStatistics + failure [ + { code: 404, message: '404 Project Not Found' }, + { code: 401, message: '401 Unauthorized' } + ] + tags %w[projects] + end + get ":id/statistics" do statistic_finder = ::Projects::DailyStatisticsFinder.new(user_project) diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index f6e1286d616..8ec67988e39 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -15,12 +15,18 @@ module API feature_category :source_code_management params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of templates available to this project' do detail 'This endpoint was introduced in GitLab 11.4' + is_array true + success Entities::TemplatesList + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] end params do use :pagination @@ -33,13 +39,24 @@ module API desc 'Download a template available to this project' do detail 'This endpoint was introduced in GitLab 11.4' + success Entities::License + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] end params do - requires :name, type: String, desc: 'The name of the template' + requires :name, type: String, + desc: 'The key of the template, as obtained from the collection endpoint.', documentation: { example: 'MIT' } optional :source_template_project_id, type: Integer, - desc: 'The project id where a given template is being stored. This is useful when multiple templates from different projects have the same name' - optional :project, type: String, desc: 'The project name to use when expanding placeholders in the template. Only affects licenses' - optional :fullname, type: String, desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses' + desc: 'The project id where a given template is being stored. This is useful when multiple templates from different projects have the same name', + documentation: { example: 1 } + optional :project, type: String, + desc: 'The project name to use when expanding placeholders in the template. Only affects licenses', + documentation: { example: 'GitLab' } + optional :fullname, type: String, + desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses', + documentation: { example: 'GitLab B.V.' } end get ':id/templates/:type/:name', requirements: TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index bb97f4fa7ce..fc898c30a71 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -151,7 +151,6 @@ module API project_params = project_finder_params support_order_by_similarity!(project_params) verify_project_filters!(project_params) - ProjectsFinder.new(current_user: current_user, params: project_params).execute end @@ -336,7 +335,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a single project' do @@ -424,7 +423,7 @@ module API end desc 'Check pages access of this project' - get ':id/pages_access', feature_category: :pages do + get ':id/pages_access', urgency: :low, feature_category: :pages do authorize! :read_pages_content, user_project unless user_project.public_pages? status 200 end @@ -654,7 +653,7 @@ module API desc 'Upload a file' params do - requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded' + requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded', documentation: { type: 'file' } end post ":id/uploads", feature_category: :not_owned do # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned log_if_upload_exceed_max_size(user_project, params[:file]) diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index fb782b49f02..bb1420534f1 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -10,9 +10,13 @@ module API execute_batch_counting(projects_relation) + postload_relation(projects_relation, options) + preload_repository_cache(projects_relation) Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects_relation, options[:current_user]).execute if options[:current_user] + + options[:current_user].preloaded_member_roles_for_projects(projects_relation) if options[:current_user] Preloaders::SingleHierarchyProjectGroupPlansPreloader.new(projects_relation).execute if options[:single_hierarchy] preload_groups(projects_relation) if options[:with] == Entities::Project diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 38bafac25b2..786045684b8 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -13,15 +13,23 @@ module API helpers Helpers::ProtectedBranchesHelpers params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, + types: [String, Integer], + desc: 'The ID or URL-encoded path of the project', + documentation: { example: 'gitlab-org/gitlab' } end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get a project's protected branches" do - success Entities::ProtectedBranch + success code: 200, model: Entities::ProtectedBranch + is_array true + failure [ + { code: 404, message: '404 Project Not Found' }, + { code: 401, message: '401 Unauthorized' } + ] end params do use :pagination - optional :search, type: String, desc: 'Search for a protected branch by name' + optional :search, type: String, desc: 'Search for a protected branch by name', documentation: { example: 'mai' } end # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_branches' do @@ -36,10 +44,14 @@ module API # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single protected branch' do - success Entities::ProtectedBranch + success code: 200, model: Entities::ProtectedBranch + failure [ + { code: 404, message: '404 Project Not Found' }, + { code: 401, message: '401 Unauthorized' } + ] end params do - requires :name, type: String, desc: 'The name of the branch or wildcard' + requires :name, type: String, desc: 'The name of the branch or wildcard', documentation: { example: 'main' } end # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do @@ -50,10 +62,16 @@ module API # rubocop: enable CodeReuse/ActiveRecord desc 'Protect a single branch' do - success Entities::ProtectedBranch + success code: 201, model: Entities::ProtectedBranch + failure [ + { code: 422, message: 'name is missing' }, + { code: 409, message: "Protected branch 'main' already exists" }, + { code: 404, message: '404 Project Not Found' }, + { code: 401, message: '401 Unauthorized' } + ] end params do - requires :name, type: String, desc: 'The name of the protected branch' + requires :name, type: String, desc: 'The name of the protected branch', documentation: { example: 'main' } optional :push_access_level, type: Integer, values: ProtectedBranch::PushAccessLevel.allowed_access_levels, desc: 'Access levels allowed to push (defaults: `40`, maintainer access level)' @@ -86,9 +104,47 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'Update a protected branch' do + success code: 200, model: Entities::ProtectedBranch + failure [ + { code: 422, message: 'Push access levels access level has already been taken' }, + { code: 404, message: '404 Project Not Found' }, + { code: 401, message: '401 Unauthorized' } + ] + end + params do + requires :name, type: String, desc: 'The name of the branch', documentation: { example: 'main' } + optional :allow_force_push, type: Boolean, + desc: 'Allow force push for all users with push access.' + + use :optional_params_ee + end + # rubocop: disable CodeReuse/ActiveRecord + patch ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + protected_branch = user_project.protected_branches.find_by!(name: params[:name]) + + declared_params = declared_params(include_missing: false) + api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params) + protected_branch = api_service.update(protected_branch) + + if protected_branch.valid? + present protected_branch, with: Entities::ProtectedBranch, project: user_project + else + render_api_error!(protected_branch.errors.full_messages, 422) + end + end + # rubocop: enable CodeReuse/ActiveRecord + desc 'Unprotect a single branch' params do - requires :name, type: String, desc: 'The name of the protected branch' + requires :name, type: String, desc: 'The name of the protected branch', documentation: { example: 'main' } + end + desc 'Unprotect a single branch' do + success code: 204 + failure [ + { code: 404, message: '404 Project Not Found' }, + { code: 401, message: '401 Unauthorized' } + ] end # rubocop: disable CodeReuse/ActiveRecord delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS, urgency: :low do diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb index 4611ee58479..7b55b1fd61d 100644 --- a/lib/api/protected_tags.rb +++ b/lib/api/protected_tags.rb @@ -13,12 +13,18 @@ module API helpers Helpers::ProtectedTagsHelpers params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get a project's protected tags" do detail 'This feature was introduced in GitLab 11.3.' - success Entities::ProtectedTag + is_array true + success code: 200, model: Entities::ProtectedTag + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[protected_tags] end params do use :pagination @@ -33,10 +39,15 @@ module API desc 'Get a single protected tag' do detail 'This feature was introduced in GitLab 11.3.' - success Entities::ProtectedTag + success code: 200, model: Entities::ProtectedTag + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[protected_tags] end params do - requires :name, type: String, desc: 'The name of the tag or wildcard' + requires :name, type: String, desc: 'The name of the tag or wildcard', documentation: { example: 'release*' } end # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do @@ -48,13 +59,21 @@ module API desc 'Protect a single tag or wildcard' do detail 'This feature was introduced in GitLab 11.3.' - success Entities::ProtectedTag + success code: 201, model: Entities::ProtectedTag + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[protected_tags] end params do - requires :name, type: String, desc: 'The name of the protected tag' - optional :create_access_level, type: Integer, - values: ProtectedTag::CreateAccessLevel.allowed_access_levels, - desc: 'Access levels allowed to create (defaults: `40`, maintainer access level)' + requires :name, type: String, desc: 'The name of the protected tag', documentation: { example: 'release-1-0' } + optional :create_access_level, + type: Integer, + values: ProtectedTag::CreateAccessLevel.allowed_access_levels, + desc: 'Access levels allowed to create (defaults: `40`, maintainer access level)', + documentation: { example: 30 } use :optional_params_ee end post ':id/protected_tags' do @@ -76,9 +95,16 @@ module API desc 'Unprotect a single tag' do detail 'This feature was introduced in GitLab 11.3.' + success code: 204 + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' }, + { code: 412, message: 'Precondition Failed' } + ] + tags %w[protected_tags] end params do - requires :name, type: String, desc: 'The name of the protected tag' + requires :name, type: String, desc: 'The name of the protected tag', documentation: { example: 'release-1-0' } end # rubocop: disable CodeReuse/ActiveRecord delete ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 1f27fcce879..6c649483da1 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -95,9 +95,9 @@ module API find_authorized_group! end - def ensure_project! + def project!(action: :read_package) find_project(params[:id]) || not_found! - authorized_user_project + authorized_user_project(action: action) end end @@ -157,14 +157,10 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do - ensure_project! - end - namespace ':id/packages/pypi' do desc 'The PyPi package download endpoint' do detail 'This feature was introduced in GitLab 12.10' @@ -176,8 +172,7 @@ module API route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get 'files/:sha256/*file_identifier' do - project = authorized_user_project - authorize_read_package!(project) + project = project! filename = "#{params[:file_identifier]}.#{params[:format]}" package = Packages::Pypi::PackageFinder.new(current_user, project, { filename: filename, sha256: params[:sha256] }).execute @@ -196,7 +191,7 @@ module API # PyPi simple API returns a list of packages as a simple HTML file. route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get 'simple', format: :txt do - present_simple_index(authorized_user_project) + present_simple_index(project!) end desc 'The PyPi Simple Project Package Endpoint' do @@ -211,7 +206,7 @@ module API # PyPi simple API returns the package descriptor as a simple HTML file. route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get 'simple/*package_name', format: :txt do - present_simple_package(authorized_user_project) + present_simple_package(project!) end desc 'The PyPi Package upload endpoint' do @@ -219,7 +214,7 @@ module API end params do - requires :content, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :content, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } requires :name, type: String requires :version, type: String optional :requires_python, type: String @@ -229,15 +224,16 @@ module API route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth post do - authorize_upload!(authorized_user_project) - bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size) + project = project!(action: :read_project) + authorize_upload!(project) + bad_request!('File is too large') if project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size) - track_package_event('push_package', :pypi, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace) + track_package_event('push_package', :pypi, project: project, user: current_user, namespace: project.namespace) unprocessable_entity! if Gitlab::FIPS.enabled? && declared_params[:md5_digest].present? ::Packages::Pypi::CreatePackageService - .new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)) + .new(project, current_user, declared_params.merge(build: current_authenticated_job)) .execute created! @@ -249,10 +245,11 @@ module API route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth post 'authorize' do + project = project!(action: :read_project) authorize_workhorse!( - subject: authorized_user_project, + subject: project, has_length: false, - maximum_size: authorized_user_project.actual_limits.pypi_max_file_size + maximum_size: project.actual_limits.pypi_max_file_size ) end end diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index 8b9380b332e..c72f90dfdf3 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -5,6 +5,8 @@ module API class Links < ::API::Base include PaginationParams + release_links_tags = %w[release_links] + RELEASE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) @@ -14,17 +16,23 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, type: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do - requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + requires :tag_name, type: String, desc: 'The tag associated with the release', as: :tag end resource 'releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do resource :assets do - desc 'Get a list of links of a release' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'List links of a release' do + detail 'Get assets as links from a release. This feature was introduced in GitLab 11.7.' success Entities::Releases::Link + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags release_links_tags end params do use :pagination @@ -36,15 +44,24 @@ module API present paginate(release.links.sorted), with: Entities::Releases::Link end - desc 'Create a link of a release' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Create a release link' do + detail 'Create an asset as a link from a release. This feature was introduced in GitLab 11.7.' success Entities::Releases::Link + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' } + ] + tags release_links_tags end params do - requires :name, type: String, desc: 'The name of the link' - requires :url, type: String, desc: 'The URL of the link' - optional :filepath, type: String, desc: 'The filepath of the link' - optional :link_type, type: String, desc: 'The link type, one of: "runbook", "image", "package" or "other"' + requires :name, type: String, desc: 'The name of the link. Link names must be unique in the release' + requires :url, type: String, desc: 'The URL of the link. Link URLs must be unique in the release.' + optional :filepath, type: String, desc: 'Optional path for a direct asset link' + optional :link_type, + type: String, + values: %w[other runbook image package], + default: 'other', + desc: 'The type of the link: `other`, `runbook`, `image`, or `package`. Defaults to `other`' end route_setting :authentication, job_token_allowed: true post 'links' do @@ -60,12 +77,17 @@ module API end params do - requires :link_id, type: String, desc: 'The ID of the link' + requires :link_id, type: Integer, desc: 'The ID of the link' end resource 'links/:link_id' do - desc 'Get a link detail of a release' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Get a release link' do + detail 'Get an asset as a link from a release. This feature was introduced in GitLab 11.7.' success Entities::Releases::Link + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags release_links_tags end route_setting :authentication, job_token_allowed: true get do @@ -74,15 +96,25 @@ module API present link, with: Entities::Releases::Link end - desc 'Update a link of a release' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Update a release link' do + detail 'Update an asset as a link from a release. This feature was introduced in GitLab 11.7.' success Entities::Releases::Link + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' } + ] + tags release_links_tags end params do optional :name, type: String, desc: 'The name of the link' optional :url, type: String, desc: 'The URL of the link' - optional :filepath, type: String, desc: 'The filepath of the link' - optional :link_type, type: String, desc: 'The link type' + optional :filepath, type: String, desc: 'Optional path for a direct asset link' + optional :link_type, + type: String, + values: %w[other runbook image package], + default: 'other', + desc: 'The type of the link: `other`, `runbook`, `image`, or `package`. Defaults to `other`' + at_least_one_of :name, :url end route_setting :authentication, job_token_allowed: true @@ -96,9 +128,14 @@ module API end end - desc 'Delete a link of a release' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Delete a release link' do + detail 'Deletes an asset as a link from a release. This feature was introduced in GitLab 11.7.' success Entities::Releases::Link + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' } + ] + tags release_links_tags end route_setting :authentication, job_token_allowed: true delete do diff --git a/lib/api/releases.rb b/lib/api/releases.rb index cdfcce9dddb..e6884e66200 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -4,6 +4,8 @@ module API class Releases < ::API::Base include PaginationParams + releases_tags = %w[releases] + RELEASE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) RELEASE_CLI_USER_AGENT = 'GitLab-release-cli' @@ -12,20 +14,37 @@ module API urgency :low params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the group' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authorize_read_group_releases! } - desc 'Get a list of releases for projects in this group.' do + desc 'List group releases' do + detail 'Returns a list of group releases.' success Entities::Release + failure [ + { code: 400, message: 'Bad request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags releases_tags end params do - requires :id, type: Integer, desc: 'The ID of the group to get releases for' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return projects sorted in ascending and descending order by released_at' - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' + requires :id, + types: [String, Integer], + desc: 'The ID or URL-encoded path of the group owned by the authenticated user' + + optional :sort, + type: String, + values: %w[asc desc], + default: 'desc', + desc: 'The direction of the order. Either `desc` (default) for descending order or `asc` for ascending order' + + optional :simple, + type: Boolean, + default: false, + desc: 'Return only limited fields for each release' use :pagination end @@ -42,26 +61,38 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authorize_read_releases! } after { track_release_event } - desc 'Get a project releases' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'List Releases' do + detail 'Returns a paginated list of releases. This feature was introduced in GitLab 11.7.' named 'get_releases' + is_array true success Entities::Release + tags releases_tags end params do use :pagination - optional :order_by, type: String, values: %w[released_at created_at], default: 'released_at', - desc: 'Return releases ordered by `released_at` or `created_at`.' - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return releases sorted in `asc` or `desc` order.' - optional :include_html_description, type: Boolean, - desc: 'If `true`, a response includes HTML rendered markdown of the release description.' + + optional :order_by, + type: String, + values: %w[released_at created_at], + default: 'released_at', + desc: 'The field to use as order. Either `released_at` (default) or `created_at`' + + optional :sort, + type: String, + values: %w[asc desc], + default: 'desc', + desc: 'The direction of the order. Either `desc` (default) for descending order or `asc` for ascending order' + + optional :include_html_description, + type: Boolean, + desc: 'If `true`, a response includes HTML rendered markdown of the release description' end route_setting :authentication, job_token_allowed: true get ':id/releases' do @@ -81,19 +112,26 @@ module API include_html_description: params[:include_html_description] end - desc 'Get a single project release' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Get a release by a tag name' do + detail 'Gets a release for the given tag. This feature was introduced in GitLab 11.7.' named 'get_release' success Entities::Release + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags releases_tags end params do - requires :tag_name, type: String, desc: 'The name of the tag', as: :tag - optional :include_html_description, type: Boolean, - desc: 'If `true`, a response includes HTML rendered markdown of the release description.' + requires :tag_name, type: String, desc: 'The Git tag the release is associated with', as: :tag + + optional :include_html_description, + type: Boolean, + desc: 'If `true`, a response includes HTML rendered markdown of the release description' end route_setting :authentication, job_token_allowed: true get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do - authorize_download_code! + authorize_read_code! not_found! unless release @@ -103,17 +141,23 @@ module API desc 'Download a project release asset file' do detail 'This feature was introduced in GitLab 15.4.' named 'download_release_asset_file' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags releases_tags end params do - requires :tag_name, type: String, - desc: 'The name of the tag.', as: :tag - requires :file_path, type: String, - file_path: true, - desc: 'The path to the file to download, as specified when creating the release asset.' + requires :tag_name, type: String, desc: 'The Git tag the release is associated with', as: :tag + + requires :file_path, + type: String, + file_path: true, + desc: 'The path to the file to download, as specified when creating the release asset' end route_setting :authentication, job_token_allowed: true get ':id/releases/:tag_name/downloads/*file_path', format: false, requirements: RELEASE_ENDPOINT_REQUIREMENTS do - authorize_download_code! + authorize_read_code! not_found! unless release @@ -127,13 +171,21 @@ module API desc 'Get the latest project release' do detail 'This feature was introduced in GitLab 15.4.' named 'get_latest_release' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags releases_tags end params do - requires :suffix_path, type: String, file_path: true, desc: 'The path to be suffixed to the latest release' + requires :suffix_path, + type: String, + file_path: true, + desc: 'The path to be suffixed to the latest release' end route_setting :authentication, job_token_allowed: true get ':id/releases/permalink/latest(/)(*suffix_path)', format: false, requirements: RELEASE_ENDPOINT_REQUIREMENTS do - authorize_download_code! + authorize_read_code! # Try to find the latest release latest_release = find_latest_release @@ -156,27 +208,50 @@ module API redirect redirect_url end - desc 'Create a new release' do - detail 'This feature was introduced in GitLab 11.7.' + desc 'Create a release' do + detail 'Creates a release. Developer level access to the project is required to create a release. This feature was introduced in GitLab 11.7.' named 'create_release' success Entities::Release + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags releases_tags end params do - requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + requires :tag_name, type: String, desc: 'The tag where the release is created from', as: :tag optional :tag_message, type: String, desc: 'Message to use if creating a new annotated tag' - optional :name, type: String, desc: 'The name of the release' - optional :description, type: String, desc: 'The release notes' - optional :ref, type: String, desc: 'Commit SHA or branch name to use if creating a new tag' + optional :name, type: String, desc: 'The release name' + optional :description, type: String, desc: 'The description of the release. You can use Markdown' + + optional :ref, + type: String, + desc: "If a tag specified in `tag_name` doesn't exist, the release is created from `ref` and tagged " \ + "with `tag_name`. It can be a commit SHA, another tag name, or a branch name." + optional :assets, type: Hash do optional :links, type: Array do - requires :name, type: String, desc: 'The name of the link' - requires :url, type: String, desc: 'The URL of the link' - optional :filepath, type: String, desc: 'The filepath of the link' - optional :link_type, type: String, desc: 'The link type, one of: "runbook", "image", "package" or "other"' + requires :name, type: String, desc: 'The name of the link. Link names must be unique within the release' + requires :url, type: String, desc: 'The URL of the link. Link URLs must be unique within the release' + optional :filepath, type: String, desc: 'Optional path for a direct asset link' + optional :link_type, type: String, desc: 'The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`' end end - optional :milestones, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The titles of the related milestones', default: [] - optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready. Defaults to the current time.' + + optional :milestones, + type: Array[String], + coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + desc: 'The title of each milestone the release is associated with. GitLab Premium customers can specify group milestones', + default: [] + + optional :released_at, + type: DateTime, + desc: 'Date and time for the release. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). ' \ + 'Only provide this field if creating an upcoming or historical release.' end route_setting :authentication, job_token_allowed: true post ':id/releases' do @@ -196,16 +271,27 @@ module API end desc 'Update a release' do - detail 'This feature was introduced in GitLab 11.7.' + detail 'Updates a release. Developer level access to the project is required to update a release. This feature was introduced in GitLab 11.7.' named 'update_release' success Entities::Release + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags releases_tags end params do - requires :tag_name, type: String, desc: 'The name of the tag', as: :tag - optional :name, type: String, desc: 'The name of the release' - optional :description, type: String, desc: 'Release notes with markdown support' - optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready.' - optional :milestones, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The titles of the related milestones' + requires :tag_name, type: String, desc: 'The Git tag the release is associated with', as: :tag + optional :name, type: String, desc: 'The release name' + optional :description, type: String, desc: 'The description of the release. You can use Markdown' + optional :released_at, type: DateTime, desc: 'The date when the release is/was ready. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`)' + + optional :milestones, + type: Array[String], + coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + desc: 'The title of each milestone to associate with the release. GitLab Premium customers can specify group milestones. To remove all milestones from the release, specify `[]`' end route_setting :authentication, job_token_allowed: true put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do @@ -226,12 +312,19 @@ module API end desc 'Delete a release' do - detail 'This feature was introduced in GitLab 11.7.' + detail "Delete a release. Deleting a release doesn't delete the associated tag. Maintainer level access to the project is required to delete a release. This feature was introduced in GitLab 11.7." named 'delete_release' success Entities::Release + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags releases_tags end params do - requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + requires :tag_name, type: String, desc: 'The Git tag the release is associated with', as: :tag end route_setting :authentication, job_token_allowed: true delete ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do @@ -280,6 +373,10 @@ module API authorize! :download_code, user_project end + def authorize_read_code! + authorize! :read_code, user_project + end + def authorize_create_evidence! # extended in EE end diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb index 8de155312fb..f7ea5a6ad2b 100644 --- a/lib/api/remote_mirrors.rb +++ b/lib/api/remote_mirrors.rb @@ -11,11 +11,17 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "List the project's remote mirrors" do - success Entities::RemoteMirror + success code: 200, model: Entities::RemoteMirror + is_array true + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[remote_mirrors] end params do use :pagination @@ -26,7 +32,12 @@ module API end desc 'Get a single remote mirror' do - success Entities::RemoteMirror + success code: 200, model: Entities::RemoteMirror + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[remote_mirrors] end params do requires :mirror_id, type: String, desc: 'The ID of a remote mirror' @@ -38,13 +49,21 @@ module API end desc 'Create remote mirror for a project' do - success Entities::RemoteMirror + success code: 201, model: Entities::RemoteMirror + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[remote_mirrors] end params do - requires :url, type: String, desc: 'The URL for a remote mirror' - optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled' - optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored' - optional :keep_divergent_refs, type: Boolean, desc: 'Determines if divergent refs are kept on the target' + requires :url, type: String, desc: 'The URL for a remote mirror', documentation: { example: 'https://*****:*****@example.com/gitlab/example.git' } + optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled', documentation: { example: false } + optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored', + documentation: { example: false } + optional :keep_divergent_refs, type: Boolean, desc: 'Determines if divergent refs are kept on the target', + documentation: { example: false } end post ':id/remote_mirrors' do create_params = declared_params(include_missing: false) @@ -59,13 +78,21 @@ module API end desc 'Update the attributes of a single remote mirror' do - success Entities::RemoteMirror + success code: 200, model: Entities::RemoteMirror + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[remote_mirrors] end params do requires :mirror_id, type: String, desc: 'The ID of a remote mirror' - optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled' - optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored' - optional :keep_divergent_refs, type: Boolean, desc: 'Determines if divergent refs are kept on the target' + optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled', documentation: { example: true } + optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored', + documentation: { example: false } + optional :keep_divergent_refs, type: Boolean, desc: 'Determines if divergent refs are kept on the target', + documentation: { example: false } end put ':id/remote_mirrors/:mirror_id' do mirror = user_project.remote_mirrors.find(params[:mirror_id]) @@ -88,6 +115,13 @@ module API desc 'Delete a single remote mirror' do detail 'This feature was introduced in GitLab 14.10' + success code: 204 + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[remote_mirrors] end params do requires :mirror_id, type: String, desc: 'The ID of a remote mirror' diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index c6a2d582d8a..70535496b12 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -15,33 +15,40 @@ module API requires :version, type: String, regexp: Gitlab::Regex.unbounded_semver_regex, - desc: 'The version of the release, using the semantic versioning format' + desc: 'The version of the release, using the semantic versioning format', + documentation: { example: '1.0.0' } optional :from, type: String, - desc: 'The first commit in the range of commits to use for the changelog' + desc: 'The first commit in the range of commits to use for the changelog', + documentation: { example: 'ed899a2f4b50b4370feeea94676502b42383c746' } optional :to, type: String, - desc: 'The last commit in the range of commits to use for the changelog' + desc: 'The last commit in the range of commits to use for the changelog', + documentation: { example: '6104942438c14ec7bd21c6cd5bd995272b3faff6' } optional :date, type: DateTime, - desc: 'The date and time of the release' + desc: 'The date and time of the release', + documentation: { type: 'dateTime', example: '2021-09-20T11:50:22.001+00:00' } optional :trailer, type: String, desc: 'The Git trailer to use for determining if commits are to be included in the changelog', - default: ::Repositories::ChangelogService::DEFAULT_TRAILER + default: ::Repositories::ChangelogService::DEFAULT_TRAILER, + documentation: { example: 'Changelog' } end end - before { authorize! :download_code, user_project } + before { authorize! :read_code, user_project } feature_category :source_code_management params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], + desc: 'The ID or URL-encoded path of the project', + documentation: { example: 1 } end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do @@ -56,7 +63,7 @@ module API end def assign_blob_vars!(limit:) - authorize! :download_code, user_project + authorize! :read_code, user_project @repo = user_project.repository @@ -94,15 +101,19 @@ module API success Entities::TreeObject end params do - optional :ref, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' - optional :path, type: String, desc: 'The path of the tree' + optional :ref, type: String, + desc: 'The name of a repository branch or tag, if not given the default branch is used', + documentation: { example: 'main' } + optional :path, type: String, desc: 'The path of the tree', documentation: { example: 'files/html' } optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' use :pagination optional :pagination, type: String, values: %w(legacy keyset none), default: 'legacy', desc: 'Specify the pagination method ("none" is only valid if "recursive" is true)' given pagination: ->(value) { value == 'keyset' } do - optional :page_token, type: String, desc: 'Record from which to start the keyset pagination' + optional :page_token, type: String, + desc: 'Record from which to start the keyset pagination', + documentation: { example: 'a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba' } end given pagination: ->(value) { value == 'none' } do @@ -123,7 +134,8 @@ module API desc 'Get raw blob contents from the repository' params do - requires :sha, type: String, desc: 'The commit hash' + requires :sha, type: String, + desc: 'The commit hash', documentation: { example: '7d70e02340bac451f281cecf0a980907974bd8be' } end get ':id/repository/blobs/:sha/raw' do # Load metadata enough to ask Workhorse to load the whole blob @@ -136,7 +148,8 @@ module API desc 'Get a blob from the repository' params do - requires :sha, type: String, desc: 'The commit hash' + requires :sha, type: String, + desc: 'The commit hash', documentation: { example: '7d70e02340bac451f281cecf0a980907974bd8be' } end get ':id/repository/blobs/:sha' do assign_blob_vars!(limit: -1) @@ -151,9 +164,12 @@ module API desc 'Get an archive of the repository' params do - optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' - optional :format, type: String, desc: 'The archive format' - optional :path, type: String, desc: 'Subfolder of the repository to be downloaded' + optional :sha, type: String, + desc: 'The commit sha of the archive to be downloaded', + documentation: { example: '7d70e02340bac451f281cecf0a980907974bd8be' } + optional :format, type: String, desc: 'The archive format', documentation: { example: 'tar.gz' } + optional :path, type: String, + desc: 'Subfolder of the repository to be downloaded', documentation: { example: 'files/archives' } end get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do check_archive_rate_limit!(current_user, user_project) do @@ -171,9 +187,13 @@ module API success Entities::Compare end params do - requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' - requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' - optional :from_project_id, type: String, desc: 'The project to compare from' + requires :from, type: String, + desc: 'The commit, branch name, or tag name to start comparison', + documentation: { example: 'main' } + requires :to, type: String, + desc: 'The commit, branch name, or tag name to stop comparison', + documentation: { example: 'feature' } + optional :from_project_id, type: Integer, desc: 'The project to compare from', documentation: { example: 1 } optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false end get ':id/repository/compare', urgency: :low do @@ -215,7 +235,10 @@ module API success Entities::Commit end params do - requires :refs, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce + requires :refs, type: Array[String], + coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + desc: 'The refs to find the common ancestor of, multiple refs can be passed', + documentation: { example: 'main' } end get ':id/repository/merge_base' do refs = params[:refs] @@ -241,12 +264,14 @@ module API desc 'Generates a changelog section for a release and returns it' do detail 'This feature was introduced in GitLab 14.6' + success Entities::Changelog end params do use :release_params optional :config_file, type: String, + documentation: { example: '.gitlab/changelog_config.yml' }, desc: "The file path to the configuration file as stored in the project's Git repository. Defaults to '.gitlab/changelog_config.yml'" end get ':id/repository/changelog' do @@ -264,26 +289,31 @@ module API desc 'Generates a changelog section for a release and commits it in a changelog file' do detail 'This feature was introduced in GitLab 13.9' + success code: 200 end params do use :release_params optional :branch, type: String, - desc: 'The branch to commit the changelog changes to' + desc: 'The branch to commit the changelog changes to', + documentation: { example: 'main' } optional :config_file, type: String, + documentation: { example: '.gitlab/changelog_config.yml' }, desc: "The file path to the configuration file as stored in the project's Git repository. Defaults to '.gitlab/changelog_config.yml'" optional :file, type: String, desc: 'The file to commit the changelog changes to', - default: ::Repositories::ChangelogService::DEFAULT_FILE + default: ::Repositories::ChangelogService::DEFAULT_FILE, + documentation: { example: 'CHANGELOG.md' } optional :message, type: String, - desc: 'The commit message to use when committing the changelog' + desc: 'The commit message to use when committing the changelog', + documentation: { example: 'Initial commit' } end post ':id/repository/changelog' do branch = params[:branch] || user_project.default_branch_or_main diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb index 2ba109b7092..754dfadb5fc 100644 --- a/lib/api/resource_access_tokens.rb +++ b/lib/api/resource_access_tokens.rb @@ -4,6 +4,8 @@ module API class ResourceAccessTokens < ::API::Base include PaginationParams + ALLOWED_RESOURCE_ACCESS_LEVELS = Gitlab::Access.options_with_owner.freeze + before { authenticate! } feature_category :authentication_and_authorization @@ -12,9 +14,12 @@ module API resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get list of all access tokens for the specified resource' do detail 'This feature was introduced in GitLab 13.9.' + is_array true + tags ["#{source_type}_access_tokens"] + success Entities::ResourceAccessToken end params do - requires :id, type: String, desc: "The #{source_type} ID" + requires :id, types: [String, Integer], desc: "ID or URL-encoded path of the #{source_type}" end get ":id/access_tokens" do resource = find_source(source_type, params[:id]) @@ -29,9 +34,11 @@ module API desc 'Get an access token for the specified resource by ID' do detail 'This feature was introduced in GitLab 14.10.' + tags ["#{source_type}_access_tokens"] + success Entities::ResourceAccessToken end params do - requires :id, type: String, desc: "The #{source_type} ID" + requires :id, types: [String, Integer], desc: "ID or URL-encoded path of the #{source_type}" requires :token_id, type: String, desc: "The ID of the token" end get ":id/access_tokens/:token_id" do @@ -51,6 +58,12 @@ module API desc 'Revoke a resource access token' do detail 'This feature was introduced in GitLab 13.9.' + tags ["#{source_type}_access_tokens"] + success code: 204 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 404, message: 'Not found' } + ] end params do requires :id, type: String, desc: "The #{source_type} ID" @@ -75,13 +88,21 @@ module API desc 'Create a resource access token' do detail 'This feature was introduced in GitLab 13.9.' + tags ["#{source_type}_access_tokens"] + success Entities::ResourceAccessTokenWithToken end params do - requires :id, type: String, desc: "The #{source_type} ID" - requires :name, type: String, desc: "Resource access token name" - requires :scopes, type: Array[String], desc: "The permissions of the token" - optional :access_level, type: Integer, desc: "The access level of the token in the #{source_type}" - optional :expires_at, type: Date, desc: "The expiration date of the token" + requires :id, type: String, desc: "The #{source_type} ID", documentation: { example: 2 } + requires :name, type: String, desc: "Resource access token name", documentation: { example: 'test' } + requires :scopes, type: Array[String], values: ::Gitlab::Auth.resource_bot_scopes.map(&:to_s), + desc: "The permissions of the token", + documentation: { example: %w[api read_repository] } + optional :access_level, type: Integer, + values: ALLOWED_RESOURCE_ACCESS_LEVELS.values, + default: Gitlab::Access::MAINTAINER, + desc: "The access level of the token in the #{source_type}", + documentation: { example: 40 } + optional :expires_at, type: Date, desc: "The expiration date of the token", documentation: { example: '"2021-01-31' } end post ':id/access_tokens' do resource = find_source(source_type, params[:id]) diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb index 04d71faa56a..5640e88ae6e 100644 --- a/lib/api/resource_milestone_events.rb +++ b/lib/api/resource_milestone_events.rb @@ -5,6 +5,8 @@ module API include PaginationParams helpers ::API::Helpers::NotesHelpers + resource_milestone_events_tags = %w[resource_milestone_events] + before { authenticate! } { @@ -15,17 +17,19 @@ module API eventables_str = eventable_type.to_s.underscore.pluralize params do - requires :id, type: String, desc: "The ID of a #{parent_type}" + requires :id, types: [String, Integer], desc: "The ID or URL-encoded path of the #{parent_type}" end resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc "Get a list of #{eventable_type.to_s.downcase} resource milestone events" do + desc "List project #{eventable_type.underscore.humanize} milestone events" do + detail "Gets a list of all milestone events for a single #{eventable_type.underscore.humanize}" success Entities::ResourceMilestoneEvent + is_array true + tags resource_milestone_events_tags end params do requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' use :pagination end - get ":id/#{eventables_str}/:eventable_id/resource_milestone_events", feature_category: feature_category, urgency: :low do eventable = find_noteable(eventable_type, params[:eventable_id]) @@ -34,8 +38,13 @@ module API present paginate(events), with: Entities::ResourceMilestoneEvent end - desc "Get a single #{eventable_type.to_s.downcase} resource milestone event" do + desc "Get single #{eventable_type.underscore.humanize} milestone event" do + detail "Returns a single milestone event for a specific project #{eventable_type.underscore.humanize}" success Entities::ResourceMilestoneEvent + failure [ + { code: 404, message: 'Not found' } + ] + tags resource_milestone_events_tags end params do requires :event_id, type: String, desc: 'The ID of a resource milestone event' diff --git a/lib/api/rpm_project_packages.rb b/lib/api/rpm_project_packages.rb index d17470ae92d..40b8d022c6c 100644 --- a/lib/api/rpm_project_packages.rb +++ b/lib/api/rpm_project_packages.rb @@ -21,7 +21,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/rpm' do @@ -30,7 +30,14 @@ module API requires :file_name, type: String, desc: 'Repository metadata file name' end get 'repodata/*file_name', requirements: { file_name: API::NO_SLASH_URL_PART_REGEX } do - not_found! + authorize_read_package!(authorized_user_project) + + repository_file = Packages::Rpm::RepositoryFile.find_by_project_id_and_file_name!( + authorized_user_project.id, + "#{params['file_name']}.#{params['format']}" + ) + + present_carrierwave_file!(repository_file.file) end desc 'Download RPM package files' @@ -39,6 +46,13 @@ module API requires :file_name, type: String, desc: 'RPM package file name' end get '*package_file_id/*file_name', requirements: { file_name: API::NO_SLASH_URL_PART_REGEX } do + track_package_event( + 'pull_package', + :rpm, + category: self.class.name, + project: authorized_user_project, + namespace: authorized_user_project.namespace + ) not_found! end @@ -50,6 +64,15 @@ module API bad_request!('File is too large') end + track_package_event( + 'push_package', + :rpm, + user: current_user, + category: self.class.name, + project: authorized_user_project, + namespace: authorized_user_project.namespace + ) + not_found! end diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index b4d02613e4c..87cf1f66223 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -93,7 +93,7 @@ module API detail 'This feature was introduced in GitLab 13.9' end params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end post 'gems' do authorize_upload!(user_project) diff --git a/lib/api/search.rb b/lib/api/search.rb index ff17696ed3e..cf6a1385783 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -67,8 +67,8 @@ module API Gitlab::Metrics::GlobalSearchSlis.record_apdex( elapsed: @search_duration_s, - search_type: search_type, - search_level: search_service.level, + search_type: search_type(additional_params), + search_level: search_service(additional_params).level, search_scope: search_scope ) @@ -81,7 +81,7 @@ module API # with a 200 status code, but an empty @search_duration_s. Gitlab::Metrics::GlobalSearchSlis.record_error_rate( error: @search_duration_s.nil? || (status < 200 || status >= 400), - search_type: search_type, + search_type: search_type(additional_params), search_level: search_service(additional_params).level, search_scope: search_scope ) @@ -171,7 +171,7 @@ module API detail 'This feature was introduced in GitLab 10.5.' end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' requires :search, type: String, desc: 'The expression it should be searched for' requires :scope, type: String, diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 8c8b6c0a1ba..26b7e58bc7a 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -53,6 +53,7 @@ module API optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' + optional :disable_admin_oauth_scopes, type: Boolean, desc: 'Stop administrators from connecting to non-trusted OAuth applications.' optional :disable_feed_token, type: Boolean, desc: 'Disable display of RSS/Atom and Calendar `feed_tokens`' optional :disabled_oauth_sign_in_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Disable certain OAuth sign-in sources' optional :domain_denylist_enabled, type: Boolean, desc: 'Enable domain denylist for sign ups' diff --git a/lib/api/snippet_repository_storage_moves.rb b/lib/api/snippet_repository_storage_moves.rb index e3034191641..92eb10b3bb8 100644 --- a/lib/api/snippet_repository_storage_moves.rb +++ b/lib/api/snippet_repository_storage_moves.rb @@ -11,7 +11,8 @@ module API resource :snippet_repository_storage_moves do desc 'Get a list of all snippet repository storage moves' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::Snippets::RepositoryStorageMove + is_array true + success code: 200, model: Entities::Snippets::RepositoryStorageMove end params do use :pagination @@ -24,7 +25,7 @@ module API desc 'Get a snippet repository storage move' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::Snippets::RepositoryStorageMove + success code: 200, model: Entities::Snippets::RepositoryStorageMove end params do requires :repository_storage_move_id, type: Integer, desc: 'The ID of a snippet repository storage move' @@ -37,6 +38,7 @@ module API desc 'Schedule bulk snippet repository storage moves' do detail 'This feature was introduced in GitLab 13.8.' + success code: 202 end params do requires :source_storage_name, type: String, desc: 'The source storage shard', values: -> { Gitlab.config.repositories.storages.keys } @@ -68,7 +70,8 @@ module API desc 'Get a list of all snippets repository storage moves' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::Snippets::RepositoryStorageMove + is_array true + success code: 200, model: Entities::Snippets::RepositoryStorageMove end params do use :pagination @@ -81,7 +84,7 @@ module API desc 'Get a snippet repository storage move' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::Snippets::RepositoryStorageMove + success code: 200, model: Entities::Snippets::RepositoryStorageMove end params do requires :repository_storage_move_id, type: Integer, desc: 'The ID of a snippet repository storage move' @@ -94,14 +97,14 @@ module API desc 'Schedule a snippet repository storage move' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::Snippets::RepositoryStorageMove + success code: 201, model: Entities::Snippets::RepositoryStorageMove end params do optional :destination_storage_name, type: String, desc: 'The destination storage shard' end post ':id/repository_storage_moves' do storage_move = user_snippet.repository_storage_moves.build( - declared_params.merge(source_storage_name: user_snippet.repository_storage) + declared_params.compact.merge(source_storage_name: user_snippet.repository_storage) ) if storage_move.schedule diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 5f8e6c806cb..36698a220bd 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -28,6 +28,11 @@ module API desc 'Get a snippets list for an authenticated user' do detail 'This feature was introduced in GitLab 8.15.' success Entities::Snippet + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[snippets] + is_array true end params do optional :created_after, type: DateTime, desc: 'Return snippets created after the specified time' @@ -45,6 +50,11 @@ module API desc 'List all public personal snippets current_user has access to' do detail 'This feature was introduced in GitLab 8.15.' success Entities::PersonalSnippet + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[snippets] + is_array true end params do optional :created_after, type: DateTime, desc: 'Return snippets created after the specified time' @@ -62,6 +72,10 @@ module API desc 'Get a single snippet' do detail 'This feature was introduced in GitLab 8.15.' success Entities::PersonalSnippet + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[snippets] end params do requires :id, type: Integer, desc: 'The ID of a snippet' @@ -77,6 +91,12 @@ module API desc 'Create new snippet' do detail 'This feature was introduced in GitLab 8.15.' success Entities::PersonalSnippet + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[snippets] end params do requires :title, type: String, allow_blank: false, desc: 'The title of a snippet' @@ -110,6 +130,12 @@ module API desc 'Update an existing snippet' do detail 'This feature was introduced in GitLab 8.15.' success Entities::PersonalSnippet + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[snippets] end params do @@ -154,6 +180,11 @@ module API desc 'Remove snippet' do detail 'This feature was introduced in GitLab 8.15.' success Entities::PersonalSnippet + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' } + ] + tags %w[snippets] end params do requires :id, type: Integer, desc: 'The ID of a snippet' @@ -178,6 +209,10 @@ module API desc 'Get a raw snippet' do detail 'This feature was introduced in GitLab 8.15.' + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[snippets] end params do requires :id, type: Integer, desc: 'The ID of a snippet' @@ -189,7 +224,12 @@ module API present content_for(snippet) end - desc 'Get raw snippet file contents from the repository' + desc 'Get raw snippet file contents from the repository' do + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[snippets] + end params do use :raw_file_params end @@ -202,6 +242,10 @@ module API desc 'Get the user agent details for a snippet' do success Entities::UserAgentDetail + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[snippets] end params do requires :id, type: Integer, desc: 'The ID of a snippet' diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb index a12a2ed08d7..1af83c0737a 100644 --- a/lib/api/statistics.rb +++ b/lib/api/statistics.rb @@ -10,7 +10,7 @@ module API MergeRequest, Note, Snippet, Key, Milestone].freeze desc 'Get the current application statistics' do - success Entities::ApplicationStatistics + success code: 200, model: Entities::ApplicationStatistics end get "application/statistics", urgency: :low do counts = Gitlab::Database::Count.approximate_counts(COUNTED_ITEMS) diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb index 2b51ab91c40..6638ac57f69 100644 --- a/lib/api/submodules.rb +++ b/lib/api/submodules.rb @@ -18,17 +18,34 @@ module API end params do - requires :id, type: String, desc: 'The project ID' + requires :id, + type: String, + desc: 'The ID or URL-encoded path of a project', + documentation: { example: 'gitlab-org/gitlab' } end - resource :projects, requirements: Files::FILE_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Update existing submodule reference in repository' do - success Entities::Commit + success code: 200, model: Entities::CommitDetail + failure [ + { code: 404, message: '404 Project Not Found' }, + { code: 401, message: '401 Unauthorized' }, + { code: 400, message: 'The repository is empty' } + ] end params do - requires :submodule, type: String, desc: 'Url encoded full path to submodule.' - requires :commit_sha, type: String, desc: 'Commit sha to update the submodule to.' - requires :branch, type: String, desc: 'Name of the branch to commit into.' - optional :commit_message, type: String, desc: 'Commit message. If no message is provided a default one will be set.' + requires :submodule, + type: String, + desc: 'Url encoded full path to submodule.', + documentation: { example: 'gitlab-org/gitlab-shell' } + requires :commit_sha, + type: String, + desc: 'Commit sha to update the submodule to.', + documentation: { example: 'ed899a2f4b50b4370feeea94676502b42383c746' } + requires :branch, type: String, desc: 'Name of the branch to commit into.', documentation: { example: 'main' } + optional :commit_message, + type: String, + desc: 'Commit message. If no message is provided a default one will be set.', + documentation: { example: 'Commit message' } end put ":id/repository/submodules/:submodule", requirements: Files::FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb index 0697169b49a..6260983087f 100644 --- a/lib/api/suggestions.rb +++ b/lib/api/suggestions.rb @@ -9,9 +9,10 @@ module API resource :suggestions do desc 'Apply suggestion patch in the Merge Request it was created' do success Entities::Suggestion + tags %w[suggestions] end params do - requires :id, type: String, desc: 'The suggestion ID' + requires :id, type: Integer, desc: 'The ID of the suggestion' optional :commit_message, type: String, desc: "A custom commit message to use instead of the default generated message or the project's default message" end put ':id/apply', urgency: :low do @@ -26,9 +27,10 @@ module API desc 'Apply multiple suggestion patches in the Merge Request where they were created' do success Entities::Suggestion + tags %w[suggestions] end params do - requires :ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: "An array of suggestion ID's" + requires :ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: "An array of the suggestion IDs" optional :commit_message, type: String, desc: "A custom commit message to use instead of the default generated message or the project's default message" end put 'batch_apply', urgency: :low do diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 804cedfefe9..f2019d785a0 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -4,6 +4,8 @@ module API class SystemHooks < ::API::Base include PaginationParams + system_hooks_tags = %w[system_hooks] + feature_category :integrations before do @@ -19,12 +21,13 @@ module API end params :hook_parameters do - optional :token, type: String, desc: 'The token used to validate payloads' - optional :push_events, type: Boolean, desc: "Trigger hook on push events" - optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" - optional :merge_requests_events, type: Boolean, desc: "Trigger hook on tag push events" - optional :repository_update_events, type: Boolean, desc: "Trigger hook on repository update events" - optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + optional :token, type: String, + desc: "Secret token to validate received payloads; this isn't returned in the response" + optional :push_events, type: Boolean, desc: 'When true, the hook fires on push events' + optional :tag_push_events, type: Boolean, desc: 'When true, the hook fires on new tags being pushed' + optional :merge_requests_events, type: Boolean, desc: 'Trigger hook on merge requests events' + optional :repository_update_events, type: Boolean, desc: 'Trigger hook on repository update events' + optional :enable_ssl_verification, type: Boolean, desc: 'Do SSL verification when triggering the hook' use :url_variables end end @@ -32,8 +35,11 @@ module API resource :hooks do mount ::API::Hooks::UrlVariables - desc 'Get the list of system hooks' do + desc 'List system hooks' do + detail 'Get a list of all system hooks' success Entities::Hook + is_array true + tags system_hooks_tags end params do use :pagination @@ -42,8 +48,13 @@ module API present paginate(SystemHook.all), with: Entities::Hook end - desc 'Get a hook' do + desc 'Get system hook' do + detail 'Get a system hook by its ID. Introduced in GitLab 14.9.' success Entities::Hook + failure [ + { code: 404, message: 'Not found' } + ] + tags system_hooks_tags end params do requires :hook_id, type: Integer, desc: 'The ID of the system hook' @@ -52,8 +63,15 @@ module API present find_hook, with: Entities::Hook end - desc 'Create a new system hook' do + desc 'Add new system hook' do + detail 'Add a new system hook' success Entities::Hook + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags system_hooks_tags end params do use :requires_url @@ -66,11 +84,18 @@ module API save_hook(hook, Entities::Hook) end - desc 'Update an existing system hook' do + desc 'Edit system hook' do + detail 'Edits a system hook' success Entities::Hook + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags system_hooks_tags end params do - requires :hook_id, type: Integer, desc: "The ID of the hook to update" + requires :hook_id, type: Integer, desc: 'The ID of the system hook' use :optional_url use :hook_parameters end @@ -90,8 +115,13 @@ module API kind: 'system_hooks' } - desc 'Delete a hook' do + desc 'Delete system hook' do + detail 'Deletes a system hook' success Entities::Hook + failure [ + { code: 404, message: 'Not found' } + ] + tags system_hooks_tags end params do requires :hook_id, type: Integer, desc: 'The ID of the system hook' diff --git a/lib/api/tags.rb b/lib/api/tags.rb index c8ac68189f5..b412a17bc6f 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -7,17 +7,25 @@ module API TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX) before do - authorize! :download_code, user_project + authorize! :read_code, user_project not_found! unless user_project.repo_exists? end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project repository tags' do - success Entities::Tag + is_array true + success code: 200, model: Entities::Tag + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' }, + { code: 503, message: 'Service unavailable' } + ] + tags %w[tags] end params do optional :sort, type: String, values: %w[asc desc], default: 'desc', @@ -46,7 +54,12 @@ module API end desc 'Get a single repository tag' do - success Entities::Tag + success code: 200, model: Entities::Tag + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[tags] end params do requires :tag_name, type: String, desc: 'The name of the tag' @@ -59,12 +72,18 @@ module API end desc 'Create a new repository tag' do - success Entities::Tag + success code: 201, model: Entities::Tag + failure [ + { code: 400, message: 'Bad request' }, + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[tags] end params do - requires :tag_name, type: String, desc: 'The name of the tag' - requires :ref, type: String, desc: 'The commit sha or branch name' - optional :message, type: String, desc: 'Specifying a message creates an annotated tag' + requires :tag_name, type: String, desc: 'The name of the tag', documentation: { example: 'v.1.0.0' } + requires :ref, type: String, desc: 'The commit sha or branch name', documentation: { example: '2695effb5807a22ff3d138d593fd856244e155e7' } + optional :message, type: String, desc: 'Specifying a message creates an annotated tag', documentation: { example: 'Release 1.0.0' } end post ':id/repository/tags', :release_orchestration do authorize_admin_tag @@ -81,7 +100,15 @@ module API end end - desc 'Delete a repository tag' + desc 'Delete a repository tag' do + success code: 204 + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' }, + { code: 412, message: 'Precondition failed' } + ] + tags %w[tags] + end params do requires :tag_name, type: String, desc: 'The name of the tag' end diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index 267d41e5fb9..5624784228e 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -21,7 +21,7 @@ module API module_version: SEMVER_REGEX }.freeze - feature_category :infrastructure_as_code + feature_category :package_registry urgency :low after_validation do @@ -92,11 +92,29 @@ module API authorize_read_package!(package || module_namespace) end + desc 'List versions for a module' do + detail 'List versions for a module' + success code: 200, model: Entities::Terraform::ModuleVersions + failure [ + { code: 403, message: 'Forbidden' } + ] + is_array true + tags %w[terraform_registry] + end get 'versions' do presenter = ::Terraform::ModulesPresenter.new(packages, params[:module_system]) present presenter, with: ::API::Entities::Terraform::ModuleVersions end + desc 'Get download location for the latest version of a module' do + detail 'Download the latest version of a module' + success code: 302 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[terraform_registry] + end get 'download' do latest_version = packages.order_version.last&.version @@ -115,6 +133,15 @@ module API redirect(download_path) end + desc 'Get details about the latest version of a module' do + detail 'Get details about the latest version of a module' + success code: 200, model: Entities::Terraform::ModuleVersion + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[terraform_registry] + end get do latest_package = packages.order_version.last @@ -133,6 +160,15 @@ module API not_found! unless package && package_file end + desc 'Get download location for specific version of a module' do + detail 'Download specific version of a module' + success code: 204 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[terraform_registry] + end get 'download' do module_file_path = api_v4_packages_terraform_modules_v1_module_version_file_path( module_namespace: params[:module_namespace], @@ -154,6 +190,15 @@ module API accept.token_types(:deploy_token_from_jwt, :job_token_from_jwt, :personal_access_token_from_jwt).sent_through(:token_param) end + desc 'Download specific version of a module' do + detail 'Download specific version of a module' + success File + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[terraform_registry] + end get do track_package_event('pull_package', :terraform_module, project: package.project, namespace: module_namespace, user: current_user) @@ -166,6 +211,15 @@ module API # format: false is required, otherwise grape splits the semver version into 2 params: # params[:module_version] and params[:format], # thus leading to an invalid/not found module version + desc 'Get details about specific version of a module' do + detail 'Get details about specific version of a module' + success code: 200, model: Entities::Terraform::ModuleVersion + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[terraform_registry] + end get format: false do presenter = ::Terraform::ModuleVersionPresenter.new(package, params[:module_system]) present presenter, with: ::API::Entities::Terraform::ModuleVersion @@ -189,6 +243,11 @@ module API desc 'Workhorse authorize Terraform Module package file' do detail 'This feature was introduced in GitLab 13.11' + success code: 200 + failure [ + { code: 403, message: 'Forbidden' } + ] + tags %w[terraform_registry] end put 'authorize' do @@ -200,10 +259,19 @@ module API desc 'Upload Terraform Module package file' do detail 'This feature was introduced in GitLab 13.11' + success code: 201 + failure [ + { code: 400, message: 'Invalid file' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + consumes %w[multipart/form-data] + tags %w[terraform_registry] end params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end put do diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index a19919b5e76..577d011ebad 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -27,13 +27,21 @@ module API increment_unique_values('p_terraform_state_api_unique_users', current_user.id) if Feature.enabled?(:route_hll_to_snowplow_phase2, user_project&.namespace) - Gitlab::Tracking.event('API::Terraform::State', 'p_terraform_state_api_unique_users', - namespace: user_project&.namespace, user: current_user) + Gitlab::Tracking.event( + 'API::Terraform::State', + 'terraform_state_api_request', + namespace: user_project&.namespace, + user: current_user, + project: user_project, + label: 'redis_hll_counters.terraform.p_terraform_state_api_unique_users_monthly', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: 'p_terraform_state_api_unique_users').to_context] + ) end end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -49,7 +57,19 @@ module API end end - desc 'Get a terraform state by its name' + desc 'Get a Terraform state by its name' do + detail 'Get a Terraform state by its name' + success [ + { code: 200 }, + { code: 204, message: 'Empty state' } + ] + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Validation failure' } + ] + tags %w[terraform_state] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get do remote_state_handler.find_with_lock do |state| @@ -60,7 +80,18 @@ module API end end - desc 'Add a new terraform state or update an existing one' + desc 'Add a new Terraform state or update an existing one' do + detail 'Add a new Terraform state or update an existing one' + success [ + { code: 200 }, + { code: 204, message: 'No data provided' } + ] + failure [ + { code: 403, message: 'Forbidden' }, + { code: 422, message: 'Validation failure' } + ] + tags %w[terraform_state] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth post do authorize! :admin_terraform_state, user_project @@ -76,7 +107,16 @@ module API status :ok end - desc 'Delete a terraform state of a certain name' + desc 'Delete a Terraform state of a certain name' do + detail 'Delete a Terraform state of a certain name' + success code: 200 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Validation failure' } + ] + tags %w[terraform_state] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth delete do authorize! :admin_terraform_state, user_project @@ -89,7 +129,17 @@ module API status :ok end - desc 'Lock a terraform state of a certain name' + desc 'Lock a Terraform state of a certain name' do + detail 'Lock a Terraform state of a certain name' + success code: 200 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' }, + { code: 422, message: 'Validation failure' } + ] + tags %w[terraform_state] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth params do requires :ID, type: String, limit: 255, desc: 'Terraform state lock ID' @@ -128,7 +178,17 @@ module API end end - desc 'Unlock a terraform state of a certain name' + desc 'Unlock a Terraform state of a certain name' do + detail 'Unlock a Terraform state of a certain name' + success code: 200 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' }, + { code: 422, message: 'Validation failure' } + ] + tags %w[terraform_state] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth params do optional :ID, type: String, limit: 255, desc: 'Terraform state lock ID' diff --git a/lib/api/terraform/state_version.rb b/lib/api/terraform/state_version.rb index ca37c786666..f98aeb5860e 100644 --- a/lib/api/terraform/state_version.rb +++ b/lib/api/terraform/state_version.rb @@ -14,7 +14,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -42,7 +42,15 @@ module API end end - desc 'Get a terraform state version' + desc 'Get a Terraform state version' do + detail 'Get a Terraform state version' + success File + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[terraform_state] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get do find_version(params[:serial]) do |version| @@ -52,7 +60,15 @@ module API end end - desc 'Delete a terraform state version' + desc 'Delete a Terraform state version' do + detail 'Delete a Terraform state version' + success code: 204 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[terraform_state] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth delete do authorize! :admin_terraform_state, user_project diff --git a/lib/api/topics.rb b/lib/api/topics.rb index 38cfdc44021..b16b40244d4 100644 --- a/lib/api/topics.rb +++ b/lib/api/topics.rb @@ -11,7 +11,9 @@ module API success Entities::Projects::Topic end params do - optional :search, type: String, desc: 'Return list of topics matching the search criteria' + optional :search, type: String, + desc: 'Return list of topics matching the search criteria', + documentation: { example: 'search' } optional :without_projects, type: Boolean, desc: 'Return list of topics without assigned projects' use :pagination end @@ -42,7 +44,8 @@ module API requires :name, type: String, desc: 'Slug (name)' requires :title, type: String, desc: 'Title' optional :description, type: String, desc: 'Description' - optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for topic' + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for topic', + documentation: { type: 'file' } end post 'topics' do authenticated_as_admin! @@ -65,7 +68,8 @@ module API optional :name, type: String, desc: 'Slug (name)' optional :title, type: String, desc: 'Title' optional :description, type: String, desc: 'Description' - optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for topic' + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for topic', + documentation: { type: 'file' } end put 'topics/:id' do authenticated_as_admin! diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb index 1fbd7cf5afc..38ce4bd7f32 100644 --- a/lib/api/unleash.rb +++ b/lib/api/unleash.rb @@ -4,14 +4,16 @@ module API class Unleash < ::API::Base include PaginationParams + unleash_tags = %w[unleash_api] + feature_category :feature_flags namespace :feature_flags do resource :unleash, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do requires :project_id, type: String, desc: 'The ID of a project' - optional :instance_id, type: String, desc: 'The Instance ID of Unleash Client' - optional :app_name, type: String, desc: 'The Application Name of Unleash Client' + optional :instance_id, type: String, desc: 'The instance ID of Unleash Client' + optional :app_name, type: String, desc: 'The application name of Unleash Client' end route_param :project_id do before do @@ -23,26 +25,22 @@ module API status :ok end - desc 'Get a list of features (deprecated, v2 client support)' - get 'features' do - if ::Feature.enabled?(:cache_unleash_client_api, project) - present_feature_flags - else - present :version, 1 - present :features, feature_flags, with: ::API::Entities::UnleashFeature - end + desc 'Get a list of features (deprecated, v2 client support)' do + is_array true + tags unleash_tags + end + get 'features', urgency: :low do + present_feature_flags end # We decrease the urgency of this endpoint until the maxmemory issue of redis-cache has been resolved. # See https://gitlab.com/gitlab-org/gitlab/-/issues/365575#note_1033611872 for more information. - desc 'Get a list of features' + desc 'Get a list of features' do + is_array true + tags unleash_tags + end get 'client/features', urgency: :low do - if ::Feature.enabled?(:cache_unleash_client_api, project) - present_feature_flags - else - present :version, 1 - present :features, feature_flags, with: ::API::Entities::UnleashFeature - end + present_feature_flags end post 'client/register' do @@ -50,7 +48,7 @@ module API status :ok end - post 'client/metrics' do + post 'client/metrics', urgency: :low do # not supported yet status :ok end diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb index 388aa5e375c..e9420f1e2b7 100644 --- a/lib/api/user_counts.rb +++ b/lib/api/user_counts.rb @@ -8,17 +8,12 @@ module API resource :user_counts do desc 'Return the user specific counts' do detail 'Assigned open issues, assigned MRs and pending todos count' + success Entities::UserCounts end get do unauthorized! unless current_user - { - merge_requests: current_user.assigned_open_merge_requests_count, # @deprecated - assigned_issues: current_user.assigned_open_issues_count, - assigned_merge_requests: current_user.assigned_open_merge_requests_count, - review_requested_merge_requests: current_user.review_requested_open_merge_requests_count, - todos: current_user.todos_pending_count - } + present current_user, with: Entities::UserCounts end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 7f44e46f1ca..72c121bca03 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -8,9 +8,18 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } - feature_category :users, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] - - urgency :medium, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] + feature_category :users, + %w[ + /users/:id/custom_attributes + /users/:id/custom_attributes/:key + /users/:id/associations_count + ] + + urgency :medium, + %w[ + /users/:id/custom_attributes + /users/:id/custom_attributes/:key + ] resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do include CustomAttributesEndpoints @@ -20,16 +29,10 @@ module API end helpers Helpers::UsersHelpers + helpers Gitlab::Tracking::Helpers::WeakPasswordErrorEvent helpers do # rubocop: disable CodeReuse/ActiveRecord - def find_user_by_id(params) - id = params[:user_id] || params[:id] - User.find_by(id: id) || not_found!('User') - end - # rubocop: enable CodeReuse/ActiveRecord - - # rubocop: disable CodeReuse/ActiveRecord def reorder_users(users) if params[:order_by] && params[:sort] users.reorder(order_options_with_tie_breaker) @@ -75,6 +78,31 @@ module API end end + resources ':id/associations_count' do + helpers do + def present_entity(result) + present result, + with: ::API::Entities::UserAssociationsCount + end + end + + desc "Returns a list of a specified user's count of projects, groups, issues and merge requests." + params do + requires :id, + type: Integer, + desc: 'ID of the user to query.' + end + get do + authenticate! + + user = find_user_by_id(params) + forbidden! unless can?(current_user, :get_user_associations_count, user) + not_found!('User') unless user + + present_entity(user) + end + end + desc 'Get the list of users' do success Entities::UserBasic end @@ -279,6 +307,8 @@ module API .by_username(user.username) .any? + track_weak_password_error(user, 'API::Users', 'create') + render_validation_error!(user) end end @@ -324,6 +354,7 @@ module API if result[:status] == :success present user, with: Entities::UserWithAdmin, current_user: current_user else + track_weak_password_error(user, 'API::Users', 'update') render_validation_error!(user) end end @@ -402,16 +433,16 @@ module API success Entities::SSHKey end params do - requires :id, type: Integer, desc: 'The ID of the user' + requires :user_id, type: Integer, desc: 'The ID of the user' requires :key, type: String, desc: 'The new SSH key' requires :title, type: String, desc: 'The title of the new SSH key' optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)' end # rubocop: disable CodeReuse/ActiveRecord - post ":id/keys", feature_category: :authentication_and_authorization do + post ":user_id/keys", feature_category: :authentication_and_authorization do authenticated_as_admin! - user = User.find_by(id: params.delete(:id)) + user = User.find_by(id: params.delete(:user_id)) not_found!('User') unless user key = ::Keys::CreateService.new(current_user, declared_params(include_missing: false).merge(user: user)).execute diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index c86b7785ce2..e4a26838746 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -58,7 +58,7 @@ module API project = find_project!( ::Gitlab::Jira::Dvcs.restore_full_path(**params.slice(:namespace, :project).symbolize_keys) ) - not_found! unless can?(current_user, :download_code, project) + not_found! unless can?(current_user, :read_code, project) project end diff --git a/lib/api/validations/validators/email_or_email_list.rb b/lib/api/validations/validators/email_or_email_list.rb index da665f39130..715a29c613d 100644 --- a/lib/api/validations/validators/email_or_email_list.rb +++ b/lib/api/validations/validators/email_or_email_list.rb @@ -9,7 +9,12 @@ module API return unless value - return if value.split(',').map { |v| ValidateEmail.valid?(v) }.all? + case value + when String + return if value.split(',').map { |v| ValidateEmail.valid?(v) }.all? + when Array + return if value.map { |v| ValidateEmail.valid?(v) }.all? + end raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index bb8ad5c4285..2058f5de706 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -28,6 +28,11 @@ module API desc 'Get a list of wiki pages' do success Entities::WikiPageBasic + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[wikis] + is_array true end params do optional :with_content, type: Boolean, default: false, desc: "Include pages' content" @@ -47,6 +52,10 @@ module API desc 'Get a wiki page' do success Entities::WikiPage + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[wikis] end params do requires :slug, type: String, desc: 'The slug of a wiki page' @@ -67,6 +76,12 @@ module API desc 'Create a wiki page' do success Entities::WikiPage + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[wikis] end params do requires :title, type: String, desc: 'Title of a wiki page' @@ -88,6 +103,12 @@ module API desc 'Update a wiki page' do success Entities::WikiPage + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[wikis] end params do optional :title, type: String, desc: 'Title of a wiki page' @@ -110,7 +131,14 @@ module API end end - desc 'Delete a wiki page' + desc 'Delete a wiki page' do + success code: 204 + failure [ + { code: 400, message: 'Validation error' }, + { code: 404, message: 'Not found' } + ] + tags %w[wikis] + end params do requires :slug, type: String, desc: 'The slug of a wiki page' end @@ -131,6 +159,10 @@ module API desc 'Upload an attachment to the wiki repository' do detail 'This feature was introduced in GitLab 11.3.' success Entities::WikiAttachment + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[wikis] end params do requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded', documentation: { type: 'file' } diff --git a/lib/atlassian/jira_connect/jwt/asymmetric.rb b/lib/atlassian/jira_connect/jwt/asymmetric.rb index 0611a17c005..573a8022752 100644 --- a/lib/atlassian/jira_connect/jwt/asymmetric.rb +++ b/lib/atlassian/jira_connect/jwt/asymmetric.rb @@ -12,7 +12,8 @@ module Atlassian KeyFetchError = Class.new(StandardError) ALGORITHM = 'RS256' - PUBLIC_KEY_CDN_URL = 'https://connect-install-keys.atlassian.com/' + DEFAULT_PUBLIC_KEY_CDN_URL = 'https://connect-install-keys.atlassian.com' + PROXY_PUBLIC_KEY_PATH = '/-/jira_connect/public_keys' UUID4_REGEX = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/.freeze def initialize(token, verification_claims) @@ -60,7 +61,7 @@ module Atlassian def retrieve_public_key(key_id) raise KeyFetchError unless UUID4_REGEX.match?(key_id) - public_key = Gitlab::HTTP.try_get(PUBLIC_KEY_CDN_URL + key_id).try(:body) + public_key = Gitlab::HTTP.try_get("#{public_key_cdn_url}/#{key_id}").try(:body) raise KeyFetchError if public_key.blank? @@ -74,6 +75,21 @@ module Atlassian def verification_qsh @verification_claims[:qsh] end + + def public_key_cdn_url + if public_key_cdn_url_setting.blank? || Feature.disabled?(:jira_connect_oauth_self_managed) + return DEFAULT_PUBLIC_KEY_CDN_URL + end + + public_key_cdn_url_setting + end + + def public_key_cdn_url_setting + @public_key_cdn_url_setting ||= + if Gitlab::CurrentSettings.jira_connect_proxy_url + Gitlab::Utils.append_path(Gitlab::CurrentSettings.jira_connect_proxy_url, PROXY_PUBLIC_KEY_PATH) + end + end end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 902eb8f6659..a8b3c12a2a2 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -396,13 +396,13 @@ module Backup timestamp = matched[1].to_i - if Time.at(timestamp) < (Time.now - keep_time) - begin - FileUtils.rm(file) - removed += 1 - rescue StandardError => e - puts_time "Deleting #{file} failed: #{e.message}".color(:red) - end + next unless Time.at(timestamp) < (Time.now - keep_time) + + begin + FileUtils.rm(file) + removed += 1 + rescue StandardError => e + puts_time "Deleting #{file} failed: #{e.message}".color(:red) end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index d1a0f8e5859..0a76c84efe5 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -22,12 +22,12 @@ module Banzai addressable_uri = nil end - unless internal_url?(addressable_uri) - punycode_autolink_node!(addressable_uri, node) - sanitize_link_text!(node) - add_malicious_tooltip!(addressable_uri, node) - add_nofollow!(addressable_uri, node) - end + next if internal_url?(addressable_uri) + + punycode_autolink_node!(addressable_uri, node) + sanitize_link_text!(node) + add_malicious_tooltip!(addressable_uri, node) + add_nofollow!(addressable_uri, node) end doc diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index f5c4b788ad8..f10efdccdf1 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -44,25 +44,25 @@ module Banzai node_xpath = Gitlab::Utils::Nokogiri.css_to_xpath(css) footnote_node = doc.at_xpath(node_xpath) - if footnote_node || modified_footnotes[ref_num] - link_node[:href] += rand_suffix - link_node[:id] += rand_suffix + next unless footnote_node || modified_footnotes[ref_num] - # Sanitization stripped off class - add it back in - link_node.parent.append_class('footnote-ref') + link_node[:href] += rand_suffix + link_node[:id] += rand_suffix - unless modified_footnotes[ref_num] - footnote_node[:id] += rand_suffix - backref_node = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]") + # Sanitization stripped off class - add it back in + link_node.parent.append_class('footnote-ref') - if backref_node - backref_node[:href] += rand_suffix - backref_node.append_class('footnote-backref') - end + next if modified_footnotes[ref_num] - modified_footnotes[ref_num] = true - end + footnote_node[:id] += rand_suffix + backref_node = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]") + + if backref_node + backref_node[:href] += rand_suffix + backref_node.append_class('footnote-backref') end + + modified_footnotes[ref_num] = true end doc diff --git a/lib/banzai/filter/kroki_filter.rb b/lib/banzai/filter/kroki_filter.rb index 713ff2439fc..26f42c6b194 100644 --- a/lib/banzai/filter/kroki_filter.rb +++ b/lib/banzai/filter/kroki_filter.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true -require "nokogiri" -require "asciidoctor/extensions/asciidoctor_kroki/extension" +require 'nokogiri' +require 'asciidoctor/extensions/asciidoctor_kroki/version' +require 'asciidoctor/extensions/asciidoctor_kroki/extension' module Banzai module Filter @@ -31,16 +32,16 @@ module Banzai img_tag = Nokogiri::HTML::DocumentFragment.parse(%(<img src="#{image_src}" />)) img_tag = img_tag.children.first - unless img_tag.nil? - lazy_load = diagram_src.length > MAX_CHARACTER_LIMIT - img_tag.set_attribute('hidden', '') if lazy_load - img_tag.set_attribute('class', 'js-render-kroki') + next if img_tag.nil? - img_tag.set_attribute('data-diagram', diagram_type) - img_tag.set_attribute('data-diagram-src', "data:text/plain;base64,#{Base64.strict_encode64(diagram_src)}") + lazy_load = diagram_src.length > MAX_CHARACTER_LIMIT + img_tag.set_attribute('hidden', '') if lazy_load + img_tag.set_attribute('class', 'js-render-kroki') - node.parent.replace(img_tag) - end + img_tag.set_attribute('data-diagram', diagram_type) + img_tag.set_attribute('data-diagram-src', "data:text/plain;base64,#{Base64.strict_encode64(diagram_src)}") + + node.parent.replace(img_tag) end doc diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 1ca4b2c89db..1d854d6599b 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -10,7 +10,7 @@ module Banzai # HTML filter that implements our math syntax, adding class="code math" # class MathFilter < HTML::Pipeline::Filter - CSS_MATH = 'pre.code.language-math' + CSS_MATH = 'pre[lang="math"] > code' XPATH_MATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_MATH).freeze CSS_CODE = 'code' XPATH_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_CODE).freeze @@ -32,7 +32,7 @@ module Banzai # Corresponds to the $$\n...\n$$ syntax DOLLAR_DISPLAY_BLOCK_PATTERN = %r{ ^(?<matched>\$\$\ *\n(?<math>.*)\n\$\$\ *)$ - }x.freeze + }mx.freeze # Order dependent. Handle the `$$` syntax before the `$` syntax DOLLAR_MATH_PIPELINE = [ @@ -107,27 +107,27 @@ module Banzai # We need a sibling before and after. # They should end and start with $ respectively. - if closing && opening && - closing.text? && opening.text? && - closing.content.first == DOLLAR_SIGN && - opening.content.last == DOLLAR_SIGN - - code[:class] = MATH_CLASSES - code[STYLE_ATTRIBUTE] = 'inline' - closing.content = closing.content[1..] - opening.content = opening.content[0..-2] - - @nodes_count += 1 - break if @nodes_count >= RENDER_NODES_LIMIT - end + next unless closing && opening && + closing.text? && opening.text? && + closing.content.first == DOLLAR_SIGN && + opening.content.last == DOLLAR_SIGN + + code[:class] = MATH_CLASSES + code[STYLE_ATTRIBUTE] = 'inline' + closing.content = closing.content[1..] + opening.content = opening.content[0..-2] + + @nodes_count += 1 + break if @nodes_count >= RENDER_NODES_LIMIT end end # corresponds to the "```math...```" syntax def process_math_codeblock - doc.xpath(XPATH_MATH).each do |el| - el[STYLE_ATTRIBUTE] = 'display' - el[:class] += " #{TAG_CLASS}" + doc.xpath(XPATH_MATH).each do |node| + pre_node = node.parent + pre_node[STYLE_ATTRIBUTE] = 'display' + pre_node[:class] = TAG_CLASS end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index 82f6247cf03..6a1fa64fb76 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -17,12 +17,12 @@ module Banzai img_tag = Nokogiri::HTML::DocumentFragment.parse( Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})).css('img').first - unless img_tag.nil? - img_tag.set_attribute('data-diagram', 'plantuml') - img_tag.set_attribute('data-diagram-src', "data:text/plain;base64,#{Base64.strict_encode64(node.content)}") + next if img_tag.nil? - node.parent.replace(img_tag) - end + img_tag.set_attribute('data-diagram', 'plantuml') + img_tag.set_attribute('data-diagram-src', "data:text/plain;base64,#{Base64.strict_encode64(node.content)}") + + node.parent.replace(img_tag) end doc diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index f5cf1833304..e95da735647 100644 --- a/lib/banzai/filter/repository_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -101,6 +101,7 @@ module Banzai if uri.relative? && uri.path.present? html_attr.value = rebuild_relative_uri(uri).to_s + html_attr.parent.add_class('gfm') end rescue URI::Error, Addressable::URI::InvalidURIError # noop diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 7175e99f1c7..766715d9e39 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true require 'rouge/plugins/common_mark' -require "asciidoctor/extensions/asciidoctor_kroki/extension" +require 'asciidoctor/extensions/asciidoctor_kroki/version' +require 'asciidoctor/extensions/asciidoctor_kroki/extension' # Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai @@ -13,8 +14,9 @@ module Banzai LANG_PARAMS_DELIMITER = ':' LANG_PARAMS_ATTR = 'data-lang-params' + CSS_CLASSES = 'code highlight js-syntax-highlight' - CSS = 'pre:not([data-math-style]):not([data-mermaid-style]):not([data-kroki-style]) > code:only-child' + CSS = 'pre:not([data-kroki-style]) > code:only-child' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze def call @@ -26,9 +28,9 @@ module Banzai end def highlight_node(node) - css_classes = +'code highlight js-syntax-highlight' + return if node.parent&.parent.nil? + lang, lang_params = parse_lang_params(node) - sourcepos = node.parent.attr('data-sourcepos') retried = false if use_rouge?(lang) @@ -41,7 +43,6 @@ module Banzai begin code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) - css_classes << " language-#{language}" if language rescue StandardError # Gracefully handle syntax highlighter bugs/errors to ensure users can # still access an issue/comment/etc. First, retry with the plain text @@ -56,16 +57,26 @@ module Banzai retry end - sourcepos_attr = sourcepos ? "data-sourcepos=\"#{escape_once(sourcepos)}\"" : '' + # maintain existing attributes already added. e.g math and mermaid nodes + node.children = code + pre_node = node.parent + + # ensure there are no extra children, such as a text node that might + # show up from an XSS attack + pre_node.children = node + + pre_node[:lang] = language + pre_node.add_class(CSS_CLASSES) + pre_node.add_class("language-#{language}") if language + pre_node.set_attribute('data-canonical-lang', escape_once(lang)) if lang != language + pre_node.set_attribute(LANG_PARAMS_ATTR, escape_once(lang_params)) if lang_params.present? + pre_node.set_attribute('v-pre', 'true') + pre_node.remove_attribute('data-meta') - highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code"><pre #{sourcepos_attr} class="#{css_classes}" - lang="#{language}" - #{lang != language ? "data-canonical-lang=\"#{escape_once(lang)}\"" : ""} - #{lang_params} - v-pre="true"><code>#{code}</code></pre><copy-code></copy-code></div>) + highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code">#{pre_node.to_html}<copy-code></copy-code></div>) # Extracted to a method to measure it - replace_parent_pre_element(node, highlighted) + replace_pre_element(pre_node, highlighted) end private @@ -93,9 +104,8 @@ module Banzai language, language_params = language.split(LANG_PARAMS_DELIMITER, 2) language_params = [node.attr('data-meta'), language_params].compact.join(' ') - formatted_language_params = format_language_params(language_params) - [language, formatted_language_params] + [language, language_params] end # Separate method so it can be instrumented. @@ -107,20 +117,14 @@ module Banzai (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new end - # Replace the parent `pre` element with the entire highlighted block - def replace_parent_pre_element(node, highlighted) - node.parent.replace(highlighted) + # Replace the `pre` element with the entire highlighted block + def replace_pre_element(pre_node, highlighted) + pre_node.replace(highlighted) end def use_rouge?(language) (%w(math suggestion) + ::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES).exclude?(language) end - - def format_language_params(language_params) - return if language_params.blank? - - %(#{LANG_PARAMS_ATTR}="#{escape_once(language_params)}") - end end end end diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 1c794a81d9d..d76009d08e1 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -33,17 +33,17 @@ module Banzai header_root = current_header = HeaderNode.new doc.xpath(XPATH).each do |node| - if header_content = node.children.first - id = string_to_anchor(node.text[0...255]) + next unless header_content = node.children.first - uniq = headers[id] > 0 ? "-#{headers[id]}" : '' - headers[id] += 1 - href = "#{id}#{uniq}" + id = string_to_anchor(node.text[0...255]) - current_header = HeaderNode.new(node: node, href: href, previous_header: current_header) + uniq = headers[id] > 0 ? "-#{headers[id]}" : '' + headers[id] += 1 + href = "#{id}#{uniq}" - header_content.add_previous_sibling(anchor_tag(href)) - end + current_header = HeaderNode.new(node: node, href: href, previous_header: current_header) + + header_content.add_previous_sibling(anchor_tag(href)) end push_toc(header_root.children, root: true) diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index b652d8d89cf..afd5802de22 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -7,13 +7,13 @@ module Banzai FilterArray[ Filter::AsciiDocSanitizationFilter, Filter::AssetProxyFilter, - Filter::SyntaxHighlightFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, Filter::ColorFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, Filter::WikiLinkFilter, + Filter::SyntaxHighlightFilter, Filter::AsciiDocPostProcessingFilter ] end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5e7c2f64c92..9b73e413d44 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -17,7 +17,6 @@ module Banzai Filter::SanitizationFilter, Filter::KrokiFilter, Filter::AssetProxyFilter, - Filter::SyntaxHighlightFilter, Filter::MathFilter, Filter::ColorFilter, Filter::MermaidFilter, @@ -37,7 +36,8 @@ module Banzai Filter::CustomEmojiFilter, Filter::TaskListFilter, Filter::InlineDiffFilter, - Filter::SetDirectionFilter + Filter::SetDirectionFilter, + Filter::SyntaxHighlightFilter ] end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 831baa9a778..19d91876892 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -66,6 +66,8 @@ module Banzai projects = lazy { projects_for_nodes(nodes) } project_attr = 'data-project' + preload_associations(projects, user) + nodes.select do |node| if node.has_attribute?(project_attr) can_read_reference?(user, projects[node], node) @@ -261,6 +263,14 @@ module Banzai hash[key] = {} end end + + # For any preloading of project associations + # needed to avoid N+1s. + # Note: `projects` param is a hash of { node => project }. + # See #projects_for_nodes for more information. + def preload_associations(projects, user) + ::Preloaders::ProjectPolicyPreloader.new(projects.values, user).execute + end end end end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb index 88896970bc6..c51f4976c28 100644 --- a/lib/banzai/reference_parser/commit_parser.rb +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -32,6 +32,13 @@ module Banzai commits end + def nodes_visible_to_user(user, nodes) + projects = lazy { projects_for_nodes(nodes) } + user.preloaded_member_roles_for_projects(projects.values) if user + + super + end + private def can_read_reference?(user, ref_project, node) diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index fb4a392105f..3d09bc83151 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -38,6 +38,13 @@ module Banzai range.valid_commits? ? range : nil end + def nodes_visible_to_user(user, nodes) + projects = lazy { projects_for_nodes(nodes) } + user.preloaded_member_roles_for_projects(projects.values) if user + + super + end + private def can_read_reference?(user, ref_project, node) diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 1d77757c4af..4c36f226006 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -56,8 +56,15 @@ module BulkImports def instance_version strong_memoize(:instance_version) do - response = with_error_handling do - Gitlab::HTTP.get(resource_url(:version), default_options) + response = begin + with_error_handling do + Gitlab::HTTP.get(resource_url(:version), default_options) + end + rescue BulkImports::NetworkError + # `version` endpoint is not available, try `metadata` endpoint instead + with_error_handling do + Gitlab::HTTP.get(resource_url(:metadata), default_options) + end end Gitlab::VersionInfo.parse(response.parsed_response['version']) diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index 5066f622d57..a52504d04bc 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -24,11 +24,13 @@ module BulkImports end logger.info( - bulk_import_id: context.bulk_import_id, - bulk_import_entity_id: context.entity.id, - bulk_import_entity_type: context.entity.source_type, + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_id: entity.id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, pipeline_class: self.class.name, message: "Entity #{entity.status_name}", + source_version: entity.bulk_import.source_version_info.to_s, importer: 'gitlab_migration' ) diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb index fea550b9f9d..68d511b065f 100644 --- a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb @@ -24,7 +24,7 @@ module BulkImports Gitlab::UrlBlocker.validate!(url, schemes: %w[http https], allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?) - wiki.ensure_repository + wiki.create_wiki_repository wiki.repository.fetch_as_mirror(url) end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index ef9575d1e96..81f8dee30d9 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -99,7 +99,7 @@ module BulkImports end def log_import_failure(exception, step) - attributes = { + failure_attributes = { bulk_import_entity_id: context.entity.id, pipeline_class: pipeline, pipeline_step: step, @@ -108,16 +108,18 @@ module BulkImports correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id } - error( - bulk_import_id: context.bulk_import_id, - pipeline_step: step, - exception_class: exception.class.to_s, - exception_message: exception.message, - message: "Pipeline failed", - importer: 'gitlab_migration' + log_exception( + exception, + log_params( + { + bulk_import_id: context.bulk_import_id, + pipeline_step: step, + message: 'Pipeline failed' + } + ) ) - BulkImports::Failure.create(attributes) + BulkImports::Failure.create(failure_attributes) end def info(extra = {}) @@ -128,17 +130,15 @@ module BulkImports logger.warn(log_params(extra)) end - def error(extra = {}) - logger.error(log_params(extra)) - end - def log_params(extra) defaults = { bulk_import_id: context.bulk_import_id, bulk_import_entity_id: context.entity.id, bulk_import_entity_type: context.entity.source_type, + source_full_path: context.entity.source_full_path, pipeline_class: pipeline, context_extra: context.extra, + source_version: context.entity.bulk_import.source_version_info.to_s, importer: 'gitlab_migration' } @@ -150,6 +150,19 @@ module BulkImports def logger @logger ||= Gitlab::Import::Logger.build end + + def log_exception(exception, payload) + Gitlab::ExceptionLogFormatter.format!(exception, payload) + logger.error(structured_payload(payload)) + end + + def structured_payload(payload = {}) + context = Gitlab::ApplicationContext.current.merge( + 'class' => self.class.name + ) + + payload.stringify_keys.merge(context) + end end end end diff --git a/lib/bulk_imports/projects/pipelines/references_pipeline.rb b/lib/bulk_imports/projects/pipelines/references_pipeline.rb new file mode 100644 index 00000000000..9c76f96c7be --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/references_pipeline.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ReferencesPipeline + include Pipeline + + BATCH_SIZE = 100 + + def extract(_context) + data = Enumerator.new do |enum| + add_matching_objects(portable.issues, enum) + add_matching_objects(portable.merge_requests, enum) + end + + BulkImports::Pipeline::ExtractedData.new(data: data) + end + + def transform(_context, object) + body = object_body(object).dup + + matching_urls(object).each do |old_url, new_url| + body.gsub!(old_url, new_url) + end + + object.assign_attributes(body_field(object) => body) + + object + end + + def load(_context, object) + object.save! if object_body_changed?(object) + end + + private + + def add_matching_objects(collection, enum) + collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| + batch.each do |object| + enum << object if object_has_reference?(object) + + object.notes.each_batch(of: BATCH_SIZE) do |notes_batch| + notes_batch.each do |note| + enum << note if object_has_reference?(note) + end + end + end + end + end + + def object_has_reference?(object) + object_body(object).include?(source_full_path) + end + + def object_body(object) + call_object_method(object) + end + + def object_body_changed?(object) + call_object_method(object, suffix: '_changed?') + end + + def call_object_method(object, suffix: nil) + method = body_field(object) + method = "#{method}#{suffix}" if suffix.present? + + object.public_send(method) # rubocop:disable GitlabSecurity/PublicSend + end + + def body_field(object) + object.is_a?(Note) ? 'note' : 'description' + end + + def matching_urls(object) + URI.extract(object_body(object), %w[http https]).each_with_object([]) do |url, array| + parsed_url = URI.parse(url) + + next unless source_host == parsed_url.host + next unless parsed_url.path&.start_with?("/#{source_full_path}") + + array << [url, new_url(parsed_url)] + end + end + + def new_url(parsed_old_url) + parsed_old_url.host = ::Gitlab.config.gitlab.host + parsed_old_url.port = ::Gitlab.config.gitlab.port + parsed_old_url.scheme = ::Gitlab.config.gitlab.https ? 'https' : 'http' + parsed_old_url.to_s.gsub!(source_full_path, portable.full_path) + end + + def source_host + @source_host ||= URI.parse(context.configuration.url).host + end + + def source_full_path + context.entity.source_full_path + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index acfa9163eae..2fefdb9055e 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -129,6 +129,10 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::PipelineSchedulesPipeline, stage: 5 }, + references: { + pipeline: BulkImports::Projects::Pipelines::ReferencesPipeline, + stage: 5 + }, finisher: { pipeline: BulkImports::Common::Pipelines::EntityFinisher, stage: 6 diff --git a/lib/error_tracking/sentry_client.rb b/lib/error_tracking/sentry_client.rb index 713cec7a7d6..dea60fff04c 100644 --- a/lib/error_tracking/sentry_client.rb +++ b/lib/error_tracking/sentry_client.rb @@ -17,14 +17,9 @@ module ErrorTracking attr_accessor :url, :token - def initialize(api_url, token, validate_size_guarded_by_feature_flag: false) + def initialize(api_url, token) @url = api_url @token = token - @validate_size_guarded_by_feature_flag = validate_size_guarded_by_feature_flag - end - - def validate_size_guarded_by_feature_flag? - @validate_size_guarded_by_feature_flag end private @@ -103,7 +98,7 @@ module ErrorTracking def handle_response(response) raise_error "Sentry response status code: #{response.code}" unless response.code.between?(200, 204) - validate_size(response.parsed_response) if validate_size_guarded_by_feature_flag? + validate_size(response.parsed_response) { body: response.parsed_response, headers: response.headers } end diff --git a/lib/error_tracking/sentry_client/issue.rb b/lib/error_tracking/sentry_client/issue.rb index 5e2e0787a3f..359617328cb 100644 --- a/lib/error_tracking/sentry_client/issue.rb +++ b/lib/error_tracking/sentry_client/issue.rb @@ -18,10 +18,6 @@ module ErrorTracking issues = response[:issues] pagination = response[:pagination] - # We check validate size only with feture flag disabled because when - # enabled we already check it when parsing the response. - validate_size(issues) unless validate_size_guarded_by_feature_flag? - handle_mapping_exceptions do { issues: map_to_errors(issues), diff --git a/lib/feature.rb b/lib/feature.rb index f317e8cb2c5..5841828da0e 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -301,11 +301,11 @@ module Feature end def gate_specified? - %i(user project group feature_group namespace).any? { |key| params.key?(key) } + %i(user project group feature_group namespace repository).any? { |key| params.key?(key) } end def targets - [feature_group, users, projects, groups, namespaces].flatten.compact + [feature_group, users, projects, groups, namespaces, repositories].flatten.compact end private @@ -350,6 +350,17 @@ module Feature Namespace.without_project_namespaces.find_by_full_path(arg) || (raise UnknowTargetError, "#{arg} is not found!") end end + + def repositories + return unless params.key?(:repository) + + params[:repository].split(',').map do |arg| + container, _project, _type, _path = Gitlab::RepoPath.parse(arg) + raise UnknowTargetError, "#{arg} is not found!" if container.nil? + + container.repository + end + end end end diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 0c6b9dfde7a..fd798862fa8 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -4,26 +4,60 @@ module Feature class Gitaly PREFIX = "gitaly_" + # Wrapper for feature flag actor to avoid unnecessarily SQL queries + class ActorWrapper + def initialize(klass, id) + @klass = klass + @id = id + end + + def flipper_id + "#{@klass.name}:#{@id}" + end + end + class << self - def enabled?(feature_flag, project = nil) + def enabled_for_any?(feature_flag, *actors) return false unless Feature::FlipperFeature.table_exists? - Feature.enabled?("#{PREFIX}#{feature_flag}", project, type: :undefined, default_enabled_if_undefined: false) + actors = actors.compact + return Feature.enabled?(feature_flag, type: :undefined, default_enabled_if_undefined: false) if actors.empty? + + actors.any? do |actor| + Feature.enabled?(feature_flag, actor, type: :undefined, default_enabled_if_undefined: false) + end rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad false end - def server_feature_flags(project = nil) + def server_feature_flags(repository: nil, user: nil, project: nil, group: nil) # We need to check that both the DB connection and table exists return {} unless FlipperFeature.database.cached_table_exists? + # The order of actors here is significant. Percentage-based actor selection may not work as expected if this + # order changes. We want repository actor to take highest precedence. + actors = [repository, user, project, group].compact + Feature.persisted_names .select { |f| f.start_with?(PREFIX) } .to_h do |f| - flag = f.delete_prefix(PREFIX) + ["gitaly-feature-#{f.delete_prefix(PREFIX).tr('_', '-')}", enabled_for_any?(f, *actors).to_s] + end + end - ["gitaly-feature-#{flag.tr('_', '-')}", enabled?(flag, project).to_s] - end + def user_actor(user = nil) + return ::Feature::Gitaly::ActorWrapper.new(::User, user.id) if user.is_a?(::User) + + user_id = Gitlab::ApplicationContext.current_context_attribute(:user_id) + ::Feature::Gitaly::ActorWrapper.new(::User, user_id) if user_id + end + + def project_actor(container) + ::Feature::Gitaly::ActorWrapper.new(::Project, container.id) if container.is_a?(::Project) + end + + def group_actor(container) + ::Feature::Gitaly::ActorWrapper.new(::Group, container.namespace_id) if container.is_a?(::Project) end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 1920e1443da..b6ad25e700b 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -11,6 +11,7 @@ module Gitlab LOG_KEY = Labkit::Context::LOG_KEY KNOWN_KEYS = [ :user, + :user_id, :project, :root_namespace, :client_id, @@ -98,6 +99,7 @@ module Gitlab assign_hash_if_value(hash, :artifacts_dependencies_count) hash[:user] = -> { username } if include_user? + hash[:user_id] = -> { user_id } if include_user? hash[:project] = -> { project_path } if include_project? hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:client_id] = -> { client } if include_client? @@ -147,6 +149,11 @@ module Gitlab associated_user&.username end + def user_id + associated_user = user || job_user + associated_user&.id + end + def root_namespace_path associated_routable = namespace || project || runner_project || runner_group || job_project associated_routable&.full_path_components&.first diff --git a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb index 7a68dd104a8..e8bdddca374 100644 --- a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb +++ b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb @@ -10,7 +10,7 @@ module Gitlab def increment(cache_key, expiry) with_redis do |redis| redis.pipelined do |pipeline| - pipeline.sadd(cache_key, resource_key) + pipeline.sadd?(cache_key, resource_key) pipeline.expire(cache_key, expiry) pipeline.scard(cache_key) end.last diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index a9c2dd001cb..d55f2bc8ac9 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -2,6 +2,7 @@ require 'asciidoctor' require 'asciidoctor-plantuml' +require 'asciidoctor/extensions/asciidoctor_kroki/version' require 'asciidoctor/extensions/asciidoctor_kroki/extension' require 'asciidoctor/extensions' require 'gitlab/asciidoc/html5_converter' diff --git a/lib/gitlab/audit/type/definition.rb b/lib/gitlab/audit/type/definition.rb index af5dc9f4b44..f64f66f4ca4 100644 --- a/lib/gitlab/audit/type/definition.rb +++ b/lib/gitlab/audit/type/definition.rb @@ -5,6 +5,7 @@ module Gitlab module Type class Definition include ActiveModel::Validations + include ::Gitlab::Audit::Type::Shared attr_reader :path attr_reader :attributes @@ -17,18 +18,6 @@ module Gitlab AUDIT_EVENT_TYPE_SCHEMA_PATH = Rails.root.join('config', 'audit_events', 'types', 'type_schema.json') AUDIT_EVENT_TYPE_SCHEMA = JSONSchemer.schema(AUDIT_EVENT_TYPE_SCHEMA_PATH) - # The PARAMS in config/audit_events/types/type_schema.json - PARAMS = %i[ - name - description - introduced_by_issue - introduced_by_mr - group - milestone - saved_to_database - streamed - ].freeze - PARAMS.each do |param| define_method(param) do attributes[param] diff --git a/lib/gitlab/audit/type/shared.rb b/lib/gitlab/audit/type/shared.rb new file mode 100644 index 00000000000..999b7de13e2 --- /dev/null +++ b/lib/gitlab/audit/type/shared.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# This file can contain only simple constructs as it is shared between: +# 1. `Pure Ruby`: `bin/audit-event-type` +# 2. `GitLab Rails`: `lib/gitlab/audit/type/definition.rb` + +module Gitlab + module Audit + module Type + module Shared + # The PARAMS in config/audit_events/types/type_schema.json + PARAMS = %i[ + name + description + introduced_by_issue + introduced_by_mr + group + milestone + saved_to_database + streamed + ].freeze + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb index 2ee0594d0a6..249c9d7af57 100644 --- a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb +++ b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb @@ -16,11 +16,10 @@ module Gitlab .where(has_vulnerabilities: false) end + operation_name :update_all + def perform - each_sub_batch( - operation_name: :update_all, - batching_scope: RELATION - ) do |sub_batch| + each_sub_batch(batching_scope: RELATION) do |sub_batch| sub_batch .joins(VULNERABILITY_READS_JOIN) .update_all(has_vulnerabilities: true) diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb index 35b5282360f..4ea664e2529 100644 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ b/lib/gitlab/background_migration/backfill_group_features.rb @@ -5,10 +5,10 @@ module Gitlab # Backfill group_features for an array of groups class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob job_arguments :batch_size + operation_name :upsert_group_features def perform each_sub_batch( - operation_name: :upsert_group_features, batching_arguments: { order_hint: :type }, batching_scope: ->(relation) { relation.where(type: 'Group') } ) do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb index b2d38ce6aa4..c95fed512c9 100644 --- a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb +++ b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb @@ -9,10 +9,10 @@ module Gitlab class BackfillImportedIssueSearchData < BatchedMigrationJob SUB_BATCH_SIZE = 1_000 + operation_name :update_search_data + def perform - each_sub_batch( - operation_name: :update_search_data - ) do |sub_batch| + each_sub_batch do |sub_batch| update_search_data(sub_batch) rescue ActiveRecord::StatementInvalid => e raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') diff --git a/lib/gitlab/background_migration/backfill_internal_on_notes.rb b/lib/gitlab/background_migration/backfill_internal_on_notes.rb index 300f2cff6ca..fe05b4ec3c1 100644 --- a/lib/gitlab/background_migration/backfill_internal_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_internal_on_notes.rb @@ -5,9 +5,10 @@ module Gitlab # This syncs the data to `internal` from `confidential` as we rename the column. class BackfillInternalOnNotes < BatchedMigrationJob scope_to -> (relation) { relation.where(confidential: true) } + operation_name :update_all def perform - each_sub_batch(operation_name: :update_all) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch.update_all(internal: true) end end diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb index b8a51b576b6..640d9379351 100644 --- a/lib/gitlab/background_migration/backfill_namespace_details.rb +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -4,8 +4,10 @@ module Gitlab module BackgroundMigration # Backfill namespace_details for a range of namespaces class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :backfill_namespace_details + def perform - each_sub_batch(operation_name: :backfill_namespace_details) do |sub_batch| + each_sub_batch do |sub_batch| upsert_namespace_details(sub_batch) end end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb index cd349bf3ae1..dca7f9fa921 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Sets the `namespace_id` of the existing `vulnerability_reads` records class BackfillNamespaceIdOfVulnerabilityReads < BatchedMigrationJob + operation_name :set_namespace_id + UPDATE_SQL = <<~SQL UPDATE vulnerability_reads @@ -16,7 +18,7 @@ module Gitlab SQL def perform - each_sub_batch(operation_name: :set_namespace_id) do |sub_batch| + each_sub_batch do |sub_batch| update_query = update_query_for(sub_batch) connection.execute(update_query) diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb index ce4c4a28b37..6520cd63711 100644 --- a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -17,8 +17,10 @@ module Gitlab self.table_name = 'project_features' end + operation_name :update_all + def perform - each_sub_batch(operation_name: :update_all) do |sub_batch| + each_sub_batch do |sub_batch| ProjectFeature.connection.execute( <<~SQL UPDATE project_features pf diff --git a/lib/gitlab/background_migration/backfill_project_import_level.rb b/lib/gitlab/background_migration/backfill_project_import_level.rb index 06706b729ea..21c239e0070 100644 --- a/lib/gitlab/background_migration/backfill_project_import_level.rb +++ b/lib/gitlab/background_migration/backfill_project_import_level.rb @@ -3,6 +3,8 @@ module Gitlab module BackgroundMigration class BackfillProjectImportLevel < BatchedMigrationJob + operation_name :update_import_level + LEVEL = { Gitlab::Access::NO_ACCESS => [0], Gitlab::Access::DEVELOPER => [2], @@ -11,7 +13,7 @@ module Gitlab }.freeze def perform - each_sub_batch(operation_name: :update_import_level) do |sub_batch| + each_sub_batch do |sub_batch| update_import_level(sub_batch) end end diff --git a/lib/gitlab/background_migration/backfill_project_namespace_details.rb b/lib/gitlab/background_migration/backfill_project_namespace_details.rb new file mode 100644 index 00000000000..9bee3cf21e8 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_namespace_details.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true +module Gitlab + module BackgroundMigration + # Backfill project namespace_details for a range of projects + class BackfillProjectNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :backfill_project_namespace_details + + def perform + each_sub_batch do |sub_batch| + upsert_project_namespace_details(sub_batch) + end + end + + def upsert_project_namespace_details(relation) + connection.execute( + <<~SQL + INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id) + SELECT projects.description, projects.description_html, projects.cached_markdown_version, now(), now(), projects.project_namespace_id + FROM projects + WHERE projects.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (namespace_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb index 815c346bb39..34dd3321125 100644 --- a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb +++ b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb @@ -4,21 +4,53 @@ module Gitlab module BackgroundMigration # Back-fills the `issues.namespace_id` by setting it to corresponding project.project_namespace_id class BackfillProjectNamespaceOnIssues < BatchedMigrationJob + MAX_UPDATE_RETRIES = 3 + + operation_name :update_all + def perform each_sub_batch( - operation_name: :update_all, batching_scope: -> (relation) { relation.joins("INNER JOIN projects ON projects.id = issues.project_id") .select("issues.id AS issue_id, projects.project_namespace_id").where(issues: { namespace_id: nil }) } ) do |sub_batch| - connection.execute <<~SQL + # updating issues table results in failed batches quite a bit, + # to prevent that as much as possible we try to update the same sub-batch up to 3 times. + update_with_retry(sub_batch) + end + end + + private + + # rubocop:disable Database/RescueQueryCanceled + # rubocop:disable Database/RescueStatementTimeout + def update_with_retry(sub_batch) + update_attempt = 1 + + begin + update_batch(sub_batch) + rescue ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled => e + update_attempt += 1 + + if update_attempt <= MAX_UPDATE_RETRIES + sleep(5) + retry + end + + raise e + end + end + # rubocop:enable Database/RescueQueryCanceled + # rubocop:enable Database/RescueStatementTimeout + + def update_batch(sub_batch) + connection.execute <<~SQL UPDATE issues SET namespace_id = projects.project_namespace_id FROM (#{sub_batch.to_sql}) AS projects(issue_id, project_namespace_id) WHERE issues.id = issue_id - SQL - end + SQL end end end diff --git a/lib/gitlab/background_migration/backfill_projects_with_coverage.rb b/lib/gitlab/background_migration/backfill_projects_with_coverage.rb deleted file mode 100644 index ca262c0bd59..00000000000 --- a/lib/gitlab/background_migration/backfill_projects_with_coverage.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill project_ci_feature_usages for a range of projects with coverage - class BackfillProjectsWithCoverage - class ProjectCiFeatureUsage < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'project_ci_feature_usages' - end - - COVERAGE_ENUM_VALUE = 1 - INSERT_DELAY_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - report_results = ActiveRecord::Base.connection.execute <<~SQL - SELECT DISTINCT project_id, default_branch - FROM ci_daily_build_group_report_results - WHERE id BETWEEN #{start_id} AND #{end_id} - SQL - - report_results.to_a.in_groups_of(sub_batch_size, false) do |batch| - ProjectCiFeatureUsage.insert_all(build_values(batch)) - - sleep INSERT_DELAY_SECONDS - end - end - - private - - def build_values(batch) - batch.map do |data| - { - project_id: data['project_id'], - feature: COVERAGE_ENUM_VALUE, - default_branch: data['default_branch'] - } - end - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_user_details_fields.rb b/lib/gitlab/background_migration/backfill_user_details_fields.rb new file mode 100644 index 00000000000..8d8619256b0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_user_details_fields.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will backfill the following fields from user to user_details + # * linkedin + # * twitter + # * skype + # * website_url + # * location + # * organization + class BackfillUserDetailsFields < BatchedMigrationJob + operation_name :backfill_user_details_fields + + def perform + query = <<~SQL + (COALESCE(linkedin, '') IS DISTINCT FROM '') + OR (COALESCE(twitter, '') IS DISTINCT FROM '') + OR (COALESCE(skype, '') IS DISTINCT FROM '') + OR (COALESCE(website_url, '') IS DISTINCT FROM '') + OR (COALESCE(location, '') IS DISTINCT FROM '') + OR (COALESCE(organization, '') IS DISTINCT FROM '') + SQL + field_limit = UserDetail::DEFAULT_FIELD_LENGTH + + each_sub_batch( + batching_scope: ->(relation) { + relation.where(query).select( + 'id AS user_id', + "substring(COALESCE(linkedin, '') from 1 for #{field_limit}) AS linkedin", + "substring(COALESCE(twitter, '') from 1 for #{field_limit}) AS twitter", + "substring(COALESCE(skype, '') from 1 for #{field_limit}) AS skype", + "substring(COALESCE(website_url, '') from 1 for #{field_limit}) AS website_url", + "substring(COALESCE(location, '') from 1 for #{field_limit}) AS location", + "substring(COALESCE(organization, '') from 1 for #{field_limit}) AS organization" + ) + } + ) do |sub_batch| + upsert_user_details_fields(sub_batch) + end + end + + def upsert_user_details_fields(relation) + connection.execute( + <<~SQL + INSERT INTO user_details (user_id, linkedin, twitter, skype, website_url, location, organization) + #{relation.to_sql} + ON CONFLICT (user_id) + DO UPDATE SET + "linkedin" = EXCLUDED."linkedin", + "twitter" = EXCLUDED."twitter", + "skype" = EXCLUDED."skype", + "website_url" = EXCLUDED."website_url", + "location" = EXCLUDED."location", + "organization" = EXCLUDED."organization" + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb index 0c41d6af209..37b1a37569b 100644 --- a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb +++ b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Backfills the `vulnerability_reads.casted_cluster_agent_id` column class BackfillVulnerabilityReadsClusterAgent < Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :update_all + CLUSTER_AGENTS_JOIN = <<~SQL INNER JOIN cluster_agents ON CAST(vulnerability_reads.cluster_agent_id AS bigint) = cluster_agents.id AND @@ -15,7 +17,7 @@ module Gitlab scope_to ->(relation) { relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) } def perform - each_sub_batch(operation_name: :update_all) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch .joins(CLUSTER_AGENTS_JOIN) .update_all('casted_cluster_agent_id = CAST(vulnerability_reads.cluster_agent_id AS bigint)') diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index 86d53ad798d..a020cabd1f4 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -19,10 +19,10 @@ module Gitlab } job_arguments :base_type, :base_type_id + operation_name :update_all def perform each_sub_batch( - operation_name: :update_all, batching_scope: -> (relation) { relation.where(work_item_type_id: nil) } ) do |sub_batch| first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 11d15804344..64401bc0674 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -36,6 +36,12 @@ module Gitlab 0 end + def self.operation_name(operation) + define_method('operation_name') do + operation + end + end + def self.job_arguments(*args) args.each.with_index do |arg, index| define_method(arg) do @@ -70,7 +76,7 @@ module Gitlab attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, :pause_ms, :connection - def each_sub_batch(operation_name: :default, batching_arguments: {}, batching_scope: nil) + def each_sub_batch(batching_arguments: {}, batching_scope: nil) all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) relation = filter_batch(base_relation) @@ -85,7 +91,7 @@ module Gitlab end end - def distinct_each_batch(operation_name: :default, batching_arguments: {}) + def distinct_each_batch(batching_arguments: {}) if base_relation != filter_batch(base_relation) raise 'distinct_each_batch can not be used when additional filters are defined with scope_to' end @@ -111,6 +117,10 @@ module Gitlab batching_scope.call(relation) end + + def operation_name + raise('Operation name is required, please define it with `operation_name`') + end end end end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 15e54431a44..136293242b2 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -15,11 +15,12 @@ module Gitlab # We use the provided primary_key column to perform the update. class CopyColumnUsingBackgroundMigrationJob < BatchedMigrationJob job_arguments :copy_from, :copy_to + operation_name :update_all def perform assignment_clauses = build_assignment_clauses(copy_from, copy_to) - each_sub_batch(operation_name: :update_all) do |relation| + each_sub_batch do |relation| relation.update_all(assignment_clauses) end end diff --git a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb index c3e1019b72f..f93dcf83c49 100644 --- a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb +++ b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb @@ -16,13 +16,14 @@ module Gitlab ) SQL + operation_name :delete_orphaned_operational_vulnerabilities scope_to ->(relation) do relation .where(report_type: [REPORT_TYPES[:cluster_image_scanning], REPORT_TYPES[:custom]]) end def perform - each_sub_batch(operation_name: :delete_orphaned_operational_vulnerabilities) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch .where(NOT_EXISTS_SQL) .delete_all diff --git a/lib/gitlab/background_migration/destroy_invalid_group_members.rb b/lib/gitlab/background_migration/destroy_invalid_group_members.rb index 35ac42f76ab..9eb0d4489d6 100644 --- a/lib/gitlab/background_migration/destroy_invalid_group_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_group_members.rb @@ -9,8 +9,10 @@ module Gitlab .where(namespaces: { id: nil }) end + operation_name :delete_all + def perform - each_sub_batch(operation_name: :delete_all) do |sub_batch| + each_sub_batch do |sub_batch| invalid_ids = sub_batch.map(&:id) Gitlab::AppLogger.info({ message: 'Removing invalid group member records', deleted_count: invalid_ids.size, ids: invalid_ids }) diff --git a/lib/gitlab/background_migration/destroy_invalid_members.rb b/lib/gitlab/background_migration/destroy_invalid_members.rb index 7d78795bea9..17a141860ec 100644 --- a/lib/gitlab/background_migration/destroy_invalid_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_members.rb @@ -4,9 +4,10 @@ module Gitlab module BackgroundMigration class DestroyInvalidMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation scope_to ->(relation) { relation.where(member_namespace_id: nil) } + operation_name :delete_all def perform - each_sub_batch(operation_name: :delete_all) do |sub_batch| + each_sub_batch do |sub_batch| deleted_members_data = sub_batch.map do |m| { id: m.id, source_id: m.source_id, source_type: m.source_type } end diff --git a/lib/gitlab/background_migration/destroy_invalid_project_members.rb b/lib/gitlab/background_migration/destroy_invalid_project_members.rb index 3c60f765c29..53b4712ef6e 100644 --- a/lib/gitlab/background_migration/destroy_invalid_project_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_project_members.rb @@ -4,9 +4,10 @@ module Gitlab module BackgroundMigration class DestroyInvalidProjectMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation scope_to ->(relation) { relation.where(source_type: 'Project') } + operation_name :delete_all def perform - each_sub_batch(operation_name: :delete_all) do |sub_batch| + each_sub_batch do |sub_batch| invalid_project_members = sub_batch .joins('LEFT OUTER JOIN projects ON members.source_id = projects.id') .where(projects: { id: nil }) diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb index 824054b31f2..b32e88581dd 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb @@ -7,6 +7,8 @@ module Gitlab PUBLIC = 20 THRESHOLD_DATE = '2022-02-17 09:00:00' + operation_name :disable_legacy_open_source_licence_for_recent_public_projects + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -14,7 +16,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :disable_legacy_open_source_licence_for_recent_public_projects, batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC).where('created_at >= ?', THRESHOLD_DATE) } diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb index e759d504f8d..5685b782a71 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb @@ -8,6 +8,8 @@ module Gitlab PUBLIC = 20 LAST_ACTIVITY_DATE = '2021-07-01' + operation_name :disable_legacy_open_source_license_available + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -15,7 +17,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :disable_legacy_open_source_license_available, batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC).where('last_activity_at < ?', LAST_ACTIVITY_DATE) } diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb index 019c3d15b3e..b5e5555bd2d 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb @@ -6,6 +6,8 @@ module Gitlab class DisableLegacyOpenSourceLicenseForNoIssuesNoRepoProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob PUBLIC = 20 + operation_name :disable_legacy_open_source_license_for_no_issues_no_repo_projects + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -13,7 +15,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :disable_legacy_open_source_license_for_no_issues_no_repo_projects, batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC) } ) do |sub_batch| no_issues_no_repo_projects = diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb index 3a9049b1f19..89863458676 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb @@ -6,6 +6,8 @@ module Gitlab class DisableLegacyOpenSourceLicenseForOneMemberNoRepoProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob PUBLIC = 20 + operation_name :disable_legacy_open_source_license_for_one_member_no_repo_projects + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -13,7 +15,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :disable_legacy_open_source_license_for_one_member_no_repo_projects, batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC) } ) do |sub_batch| one_member_no_repo_projects = diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb index 6e4d5d8ddcb..7d93f2d4fda 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb @@ -5,9 +5,10 @@ module Gitlab # Set `project_settings.legacy_open_source_license_available` to false for projects less than 1 MB class DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob scope_to ->(relation) { relation.where(legacy_open_source_license_available: true) } + operation_name :disable_legacy_open_source_license_for_projects_less_than_one_mb def perform - each_sub_batch(operation_name: :disable_legacy_open_source_license_for_projects_less_than_one_mb) do |sub_batch| + each_sub_batch do |sub_batch| updates = { legacy_open_source_license_available: false, updated_at: Time.current } sub_batch diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb index e1805d40bab..961dea028c9 100644 --- a/lib/gitlab/background_migration/encrypt_static_object_token.rb +++ b/lib/gitlab/background_migration/encrypt_static_object_token.rb @@ -40,8 +40,9 @@ module Gitlab encrypted_tokens_sql = user_encrypted_tokens.compact.map { |(id, token)| "(#{id}, '#{token}')" }.join(',') - if user_encrypted_tokens.present? - User.connection.execute(<<~SQL) + next unless user_encrypted_tokens.present? + + User.connection.execute(<<~SQL) WITH cte(cte_id, cte_token) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT * FROM (VALUES #{encrypted_tokens_sql}) AS t (id, token) @@ -50,8 +51,7 @@ module Gitlab SET static_object_token_encrypted = cte_token FROM cte WHERE cte_id = id - SQL - end + SQL end mark_job_as_succeeded(start_id, end_id) diff --git a/lib/gitlab/background_migration/expire_o_auth_tokens.rb b/lib/gitlab/background_migration/expire_o_auth_tokens.rb index 595e4ac9dc8..08bcdb8a789 100644 --- a/lib/gitlab/background_migration/expire_o_auth_tokens.rb +++ b/lib/gitlab/background_migration/expire_o_auth_tokens.rb @@ -4,9 +4,10 @@ module Gitlab module BackgroundMigration # Add expiry to all OAuth access tokens class ExpireOAuthTokens < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :update_oauth_tokens + def perform each_sub_batch( - operation_name: :update_oauth_tokens, batching_scope: ->(relation) { relation.where(expires_in: nil) } ) do |sub_batch| update_oauth_tokens(sub_batch) diff --git a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb index 3f04e04fc4d..3dd867fa1fe 100644 --- a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb +++ b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb @@ -6,8 +6,10 @@ module Gitlab # monitor_access_level, deployments_access_level, infrastructure_access_level. # The operations_access_level setting is being split into three seperate toggles. class PopulateOperationVisibilityPermissionsFromOperations < BatchedMigrationJob + operation_name :populate_operations_visibility + def perform - each_sub_batch(operation_name: :populate_operations_visibility) do |batch| + each_sub_batch do |batch| batch.update_all('monitor_access_level=operations_access_level,' \ 'infrastructure_access_level=operations_access_level,' \ ' feature_flags_access_level=operations_access_level,'\ diff --git a/lib/gitlab/background_migration/populate_projects_star_count.rb b/lib/gitlab/background_migration/populate_projects_star_count.rb new file mode 100644 index 00000000000..085d576637e --- /dev/null +++ b/lib/gitlab/background_migration/populate_projects_star_count.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to populates the star counter of projects + class PopulateProjectsStarCount < BatchedMigrationJob + MAX_UPDATE_RETRIES = 3 + + operation_name :update_all + + def perform + each_sub_batch do |sub_batch| + update_with_retry(sub_batch) + end + end + + private + + # rubocop:disable Database/RescueQueryCanceled + # rubocop:disable Database/RescueStatementTimeout + def update_with_retry(sub_batch) + update_attempt = 1 + + begin + update_batch(sub_batch) + rescue ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled => e + update_attempt += 1 + + if update_attempt <= MAX_UPDATE_RETRIES + sleep(5) + retry + end + + raise e + end + end + # rubocop:enable Database/RescueQueryCanceled + # rubocop:enable Database/RescueStatementTimeout + + def update_batch(sub_batch) + ApplicationRecord.connection.execute <<~SQL + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql}) + UPDATE projects + SET star_count = ( + SELECT COUNT(*) + FROM users_star_projects + INNER JOIN users + ON users_star_projects.user_id = users.id + WHERE users_star_projects.project_id = batched_relation.id + AND users.state = 'active' + ) + FROM batched_relation + WHERE projects.id = batched_relation.id + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/recount_epic_cache_counts.rb b/lib/gitlab/background_migration/recount_epic_cache_counts.rb new file mode 100644 index 00000000000..42f84a33a5a --- /dev/null +++ b/lib/gitlab/background_migration/recount_epic_cache_counts.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class RecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +# we just want to re-enqueue the previous BackfillEpicCacheCounts migration, +# because it's a EE-only migation and it's a module, we just prepend new +# RecountEpicCacheCounts with existing batched migration module (which is same in both cases) +Gitlab::BackgroundMigration::RecountEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb index d30263976e8..dc7c16d7947 100644 --- a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb +++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb @@ -6,6 +6,8 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723. # These job artifacts will not be deleted and will have their `expire_at` removed. class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob + operation_name :update_all + # The migration would have backfilled `expire_at` # to midnight on the 22nd of the month of the local timezone, # storing it as UTC time in the database. @@ -32,9 +34,7 @@ module Gitlab } def perform - each_sub_batch( - operation_name: :update_all - ) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch.update_all(expire_at: nil) end end diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb index 5b1d630bb03..a284c04d4f5 100644 --- a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb +++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb @@ -4,10 +4,10 @@ module Gitlab module BackgroundMigration # Removes obsolete wiki notes class RemoveSelfManagedWikiNotes < BatchedMigrationJob + operation_name :delete_all + def perform - each_sub_batch( - operation_name: :delete_all - ) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch.where(noteable_type: 'Wiki').delete_all end end diff --git a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb index 718fb0aaa71..1b13c2ab7ef 100644 --- a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb +++ b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb @@ -13,8 +13,10 @@ module Gitlab relation.where(system_note_metadata: { action: :task }) } + operation_name :update_all + def perform - each_sub_batch(operation_name: :update_all) do |sub_batch| + each_sub_batch do |sub_batch| ApplicationRecord.connection.execute <<~SQL UPDATE notes SET note = REGEXP_REPLACE(notes.note,'#{REPLACE_REGEX}', '#{TEXT_REPLACEMENT}') diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb index 952f3b0e3c3..832385fd662 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb @@ -4,8 +4,10 @@ module Gitlab module BackgroundMigration # A job to nullify duplicate token_encrypted values in ci_runners table in batches class ResetDuplicateCiRunnersTokenEncryptedValues < BatchedMigrationJob + operation_name :nullify_duplicate_ci_runner_token_encrypted_values + def perform - each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_encrypted_values) do |sub_batch| + each_sub_batch do |sub_batch| # Reset duplicate runner encrypted tokens that would prevent creating an unique index. nullify_duplicate_ci_runner_token_encrypted_values(sub_batch) end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb index cfd6a4e4091..5f552accd8d 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb @@ -4,8 +4,10 @@ module Gitlab module BackgroundMigration # A job to nullify duplicate token values in ci_runners table in batches class ResetDuplicateCiRunnersTokenValues < BatchedMigrationJob + operation_name :nullify_duplicate_ci_runner_token_values + def perform - each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_values) do |sub_batch| + each_sub_batch do |sub_batch| # Reset duplicate runner tokens that would prevent creating an unique index. nullify_duplicate_ci_runner_token_values(sub_batch) end diff --git a/lib/gitlab/background_migration/sanitize_confidential_todos.rb b/lib/gitlab/background_migration/sanitize_confidential_todos.rb new file mode 100644 index 00000000000..d3ef6ac3019 --- /dev/null +++ b/lib/gitlab/background_migration/sanitize_confidential_todos.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Iterates through confidential notes and removes any its todos if user can + # not read the note + # + # Warning: This migration is not properly isolated. The reason for this is + # that we need to check permission for notes and it would be difficult + # to extract all related logic. + # Details in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908#note_952459215 + class SanitizeConfidentialTodos < BatchedMigrationJob + scope_to ->(relation) { relation.where(confidential: true) } + + operation_name :delete_invalid_todos + + def perform + each_sub_batch do |sub_batch| + delete_ids = invalid_todo_ids(sub_batch) + + Todo.where(id: delete_ids).delete_all if delete_ids.present? + end + end + + private + + def invalid_todo_ids(notes_batch) + todos = Todo.where(note_id: notes_batch.select(:id)).includes(:note, :user) + + todos.each_with_object([]) do |todo, ids| + ids << todo.id if invalid_todo?(todo) + end + end + + def invalid_todo?(todo) + return false unless todo.note + return false if Ability.allowed?(todo.user, :read_todo, todo) + + logger.info( + message: "#{self.class.name} deleting invalid todo", + attributes: todo.attributes + ) + + true + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb index a0cfeed618a..dfd71bb8b5f 100644 --- a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -7,9 +7,10 @@ module Gitlab DISMISSED_STATE = 2 scope_to ->(relation) { relation.where.not(dismissed_at: nil) } + operation_name :update_vulnerabilities_state def perform - each_sub_batch(operation_name: :update_vulnerabilities_state) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch.update_all(state: DISMISSED_STATE) end end diff --git a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb index e85b1bc402a..4ae7ad897cf 100644 --- a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb +++ b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb @@ -6,6 +6,8 @@ module Gitlab class SetLegacyOpenSourceLicenseAvailableForNonPublicProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob PUBLIC = 20 + operation_name :set_legacy_open_source_license_available + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -13,7 +15,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :set_legacy_open_source_license_available, batching_scope: ->(relation) { relation.where.not(visibility_level: PUBLIC) } ) do |sub_batch| ProjectSetting.where(project_id: sub_batch).update_all(legacy_open_source_license_available: false) diff --git a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb index 04a2ceebef8..b2cf8298e4f 100644 --- a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb +++ b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb @@ -10,10 +10,10 @@ module Gitlab self.table_name = 'namespace_settings' end + operation_name :set_delayed_project_removal_to_null_for_user_namespace + def perform - each_sub_batch( - operation_name: :set_delayed_project_removal_to_null_for_user_namespace - ) do |sub_batch| + each_sub_batch do |sub_batch| set_delayed_project_removal_to_null_for_user_namespace(sub_batch) end end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9209c9b4927..b2630a7ad7a 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -85,7 +85,7 @@ module Gitlab end def load_from_cache - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) self.status = nil if self.status.empty? @@ -93,13 +93,13 @@ module Gitlab end def store_in_cache - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.del(cache_key) end end @@ -107,7 +107,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.exists?(cache_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -125,6 +125,10 @@ module Gitlab project.commit end end + + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 4e7a7f326a5..7fec6584ba3 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -33,7 +33,7 @@ module Gitlab # timeout - The new timeout of the key if the key is to be refreshed. def self.read(raw_key, timeout: TIMEOUT) key = cache_key_for(raw_key) - value = Redis::Cache.with { |redis| redis.get(key) } + value = with_redis { |redis| redis.get(key) } if value.present? # We refresh the expiration time so frequently used keys stick @@ -44,7 +44,7 @@ module Gitlab # did not find a matching GitLab user. In that case we _don't_ want to # refresh the TTL so we automatically pick up the right data when said # user were to register themselves on the GitLab instance. - Redis::Cache.with { |redis| redis.expire(key, timeout) } + with_redis { |redis| redis.expire(key, timeout) } end value @@ -69,7 +69,7 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.set(key, value, ex: timeout) end @@ -85,7 +85,7 @@ module Gitlab def self.increment(raw_key, timeout: TIMEOUT) key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| value = redis.incr(key) redis.expire(key, timeout) @@ -105,7 +105,7 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.incrby(key, value) redis.expire(key, timeout) end @@ -121,9 +121,9 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.multi do |m| - m.sadd(key, value) + m.sadd?(key, value) m.expire(key, timeout) end end @@ -149,7 +149,7 @@ module Gitlab def self.values_from_set(raw_key) key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.smembers(key) end end @@ -160,14 +160,16 @@ module Gitlab # key_prefix - prefix inserted before each key # timeout - The time after which the cache key should expire. def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT) - Redis::Cache.with do |redis| - redis.pipelined do |multi| - mapping.each do |raw_key, value| - key = cache_key_for("#{key_prefix}#{raw_key}") + with_redis do |redis| + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.pipelined do |multi| + mapping.each do |raw_key, value| + key = cache_key_for("#{key_prefix}#{raw_key}") - validate_redis_value!(value) + validate_redis_value!(value) - multi.set(key, value, ex: timeout) + multi.set(key, value, ex: timeout) + end end end end @@ -180,7 +182,7 @@ module Gitlab def self.expire(raw_key, timeout) key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.expire(key, timeout) end end @@ -199,7 +201,7 @@ module Gitlab validate_redis_value!(value) key = cache_key_for(raw_key) - val = Redis::Cache.with do |redis| + val = with_redis do |redis| redis .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout]) end @@ -218,7 +220,7 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.multi do |m| m.hset(key, field, value) m.expire(key, timeout) @@ -232,7 +234,7 @@ module Gitlab def self.values_from_hash(raw_key) key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.hgetall(key) end end @@ -241,6 +243,10 @@ module Gitlab "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" end + def self.with_redis(&block) + Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end + def self.validate_redis_value!(value) value_as_string = value.to_s return if value_as_string.is_a?(String) diff --git a/lib/gitlab/cache/metrics.rb b/lib/gitlab/cache/metrics.rb new file mode 100644 index 00000000000..0143052beb1 --- /dev/null +++ b/lib/gitlab/cache/metrics.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# Instrumentation for cache efficiency metrics +module Gitlab + module Cache + class Metrics + DEFAULT_BUCKETS = [0, 1, 5].freeze + VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze + DEFAULT_BACKING_RESOURCE = :unknown + + def initialize( + caller_id:, + cache_identifier:, + feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT, + backing_resource: DEFAULT_BACKING_RESOURCE + ) + @caller_id = caller_id + @cache_identifier = cache_identifier + @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) + @backing_resource = fetch_backing_resource!(backing_resource) + end + + # Increase cache hit counter + # + def increment_cache_hit + counter.increment(labels.merge(cache_hit: true)) + end + + # Increase cache miss counter + # + def increment_cache_miss + counter.increment(labels.merge(cache_hit: false)) + end + + # Measure the duration of cacheable action + # + # @example + # observe_cache_generation do + # cacheable_action + # end + # + def observe_cache_generation(&block) + real_start = Gitlab::Metrics::System.monotonic_time + + value = yield + + histogram.observe({}, Gitlab::Metrics::System.monotonic_time - real_start) + + value + end + + private + + attr_reader :caller_id, :cache_identifier, :feature_category, :backing_resource + + def counter + @counter ||= Gitlab::Metrics.counter(:redis_hit_miss_operations_total, "Hit/miss Redis cache counter") + end + + def histogram + @histogram ||= Gitlab::Metrics.histogram( + :redis_cache_generation_duration_seconds, + 'Duration of Redis cache generation', + labels, + DEFAULT_BUCKETS + ) + end + + def labels + @labels ||= { + caller_id: caller_id, + cache_identifier: cache_identifier, + feature_category: feature_category, + backing_resource: backing_resource + } + end + + def fetch_backing_resource!(resource) + return resource if VALID_BACKING_RESOURCES.include?(resource) + + raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env? + + DEFAULT_BACKING_RESOURCE + end + end + end +end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 2ab702aa4f9..19819ff7275 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -312,9 +312,10 @@ module Gitlab normalized_section = section_to_class_name(section) - if action == "start" + case action + when "start" handle_section_start(normalized_section, timestamp) - elsif action == "end" + when "end" handle_section_end(normalized_section, timestamp) end end diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index ddf40296809..78f6c5bf0aa 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -107,9 +107,10 @@ module Gitlab section_name = sanitize_section_name(section) - if action == 'start' + case action + when 'start' handle_section_start(scanner, section_name, timestamp, options) - elsif action == 'end' + when 'end' handle_section_end(scanner, section_name, timestamp) else raise 'unsupported action' diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index 7dc375e05eb..84f8eae8deb 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -24,10 +24,11 @@ module Gitlab end def initialize(image) - if image.is_a?(String) + case image + when String @name = image @ports = [] - elsif image.is_a?(Hash) + when Hash @alias = image[:alias] @command = image[:command] @entrypoint = image[:entrypoint] diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index aebd81e7b07..c55615bb83b 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -9,20 +9,30 @@ module Gitlab MAX_PATTERN_COMPARISONS = 10_000 def initialize(globs) - globs = Array(globs) - - @top_level_only = globs.all?(&method(:top_level_glob?)) - @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?)) + @globs = Array(globs) + @top_level_only = @globs.all?(&method(:top_level_glob?)) end def satisfied_by?(_pipeline, context) paths = worktree_paths(context) + exact_globs, pattern_globs = separate_globs(context) - exact_matches?(paths) || pattern_matches?(paths) + exact_matches?(paths, exact_globs) || pattern_matches?(paths, pattern_globs) end private + def separate_globs(context) + expanded_globs = expand_globs(context) + expanded_globs.partition(&method(:exact_glob?)) + end + + def expand_globs(context) + @globs.map do |glob| + ExpandVariables.expand_existing(glob, -> { context.variables_hash }) + end + end + def worktree_paths(context) return [] unless context.project @@ -33,13 +43,16 @@ module Gitlab end end - def exact_matches?(paths) - @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } } + def exact_matches?(paths, exact_globs) + exact_globs.any? do |glob| + paths.bsearch { |path| glob <=> path } + end end - def pattern_matches?(paths) + def pattern_matches?(paths, pattern_globs) comparisons = 0 - @pattern_globs.any? do |glob| + + pattern_globs.any? do |glob| paths.any? do |path| comparisons += 1 comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path) diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 661c6fb87e3..ee537f4efe5 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -73,6 +73,10 @@ module Gitlab root.variables_entry.value_with_data end + def variables_with_prefill_data + root.variables_entry.value_with_prefill_data + end + def stages root.stages_value end diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index 73742298628..ee99354cb28 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -18,7 +18,7 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS with_options allow_nil: true do - validates :when, inclusion: { + validates :when, type: String, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 7513936a18a..8e7f6ba4326 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -21,7 +21,7 @@ module Gitlab validates :script, presence: true with_options allow_nil: true do - validates :when, inclusion: { + validates :when, type: String, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 2d2032b1d8c..e0a052ffdfd 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -60,6 +60,7 @@ module Gitlab entry :variables, ::Gitlab::Ci::Config::Entry::Variables, description: 'Environment variables available for this job.', + metadata: { allowed_value_data: %i[value expand] }, inherit: false entry :inherit, ::Gitlab::Ci::Config::Entry::Inherit, diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 1d7d8617c74..a30e6a0d9c3 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -50,7 +50,7 @@ module Gitlab entry :variables, Entry::Variables, description: 'Environment variables that will be used.', - metadata: { allowed_value_data: %i[value description], allow_array_value: true }, + metadata: { allowed_value_data: %i[value description expand], allow_array_value: true }, reserved: true entry :stages, Entry::Stages, diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index 54c153c8b07..16091758916 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -33,6 +33,10 @@ module Gitlab def value_with_data { value: @config.to_s } end + + def value_with_prefill_data + value_with_data + end end class ComplexVariable < ::Gitlab::Config::Entry::Node @@ -48,6 +52,9 @@ module Gitlab validates :key, alphanumeric: true validates :config_value, alphanumeric: true, allow_nil: false, if: :config_value_defined? validates :config_description, alphanumeric: true, allow_nil: false, if: :config_description_defined? + validates :config_expand, boolean: true, + allow_nil: false, + if: -> { ci_raw_variables_in_yaml_config_enabled? && config_expand_defined? } validate do allowed_value_data = Array(opt(:allowed_value_data)) @@ -67,7 +74,22 @@ module Gitlab end def value_with_data - { value: value, description: config_description }.compact + if ci_raw_variables_in_yaml_config_enabled? + { + value: value, + raw: (!config_expand if config_expand_defined?) + }.compact + else + { + value: value + }.compact + end + end + + def value_with_prefill_data + value_with_data.merge( + description: config_description + ).compact end def config_value @@ -78,6 +100,10 @@ module Gitlab @config[:description] end + def config_expand + @config[:expand] + end + def config_value_defined? config.key?(:value) end @@ -85,6 +111,14 @@ module Gitlab def config_description_defined? config.key?(:description) end + + def config_expand_defined? + config.key?(:expand) + end + + def ci_raw_variables_in_yaml_config_enabled? + YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) + end end class ComplexArrayVariable < ComplexVariable @@ -110,8 +144,10 @@ module Gitlab config_value.first end - def value_with_data - super.merge(value_options: config_value).compact + def value_with_prefill_data + super.merge( + value_options: config_value + ).compact end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 4430a11dda7..ef4f74b9f56 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -29,6 +29,12 @@ module Gitlab end end + def value_with_prefill_data + @entries.to_h do |key, entry| + [key.to_s, entry.value_with_prefill_data] + end + end + private def composable_class(_name, _config) diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 1244c7f7475..21a57640aee 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -42,29 +42,20 @@ module Gitlab context&.parent_pipeline&.project end - def validate_content! - return unless ensure_preconditions_satisfied! - - errors.push("File `#{masked_location}` is empty!") unless content.present? - end - - def ensure_preconditions_satisfied! - unless creating_child_pipeline? - errors.push('Including configs from artifacts is only allowed when triggering child pipelines') - return false - end - - unless job_name.present? - errors.push("Job must be provided when including configs from artifacts") - return false - end - - unless artifact_job.present? - errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!") - return false + def validate_context! + context.logger.instrument(:config_file_artifact_validate_context) do + if !creating_child_pipeline? + errors.push('Including configs from artifacts is only allowed when triggering child pipelines') + elsif !job_name.present? + errors.push("Job must be provided when including configs from artifacts") + elsif !artifact_job.present? + errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!") + end end + end - true + def validate_content! + errors.push("File `#{masked_location}` is empty!") unless content.present? end def artifact_job diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 89da0796906..57ff606c9ee 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,12 +47,11 @@ module Gitlab end def validate! - context.logger.instrument(:config_file_validation) do - validate_execution_time! - validate_location! - validate_content! if errors.none? - validate_hash! if errors.none? - end + validate_execution_time! + validate_location! + validate_context! if valid? + fetch_and_validate_content! if valid? + load_and_validate_expanded_hash! if valid? end def metadata @@ -100,6 +99,34 @@ module Gitlab end end + def validate_context! + raise NotImplementedError, 'subclass must implement validate_context' + end + + def fetch_and_validate_content! + context.logger.instrument(:config_file_fetch_content) do + content # calling the method fetches then memoizes the result + end + + return if errors.any? + + context.logger.instrument(:config_file_validate_content) do + validate_content! + end + end + + def load_and_validate_expanded_hash! + context.logger.instrument(:config_file_fetch_content_hash) do + content_hash # calling the method loads then memoizes the result + end + + context.logger.instrument(:config_file_expand_content_includes) do + expanded_content_hash # calling the method expands then memoizes the result + end + + validate_hash! + end + def validate_content! if content.blank? errors.push("Included file `#{masked_location}` is empty or does not exist!") diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 36fc5c656fc..0912a732158 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -31,10 +31,14 @@ module Gitlab private + def validate_context! + return if context.project&.repository + + errors.push("Local file `#{masked_location}` does not have project!") + end + def validate_content! - if context.project&.repository.nil? - errors.push("Local file `#{masked_location}` does not have project!") - elsif content.nil? + if content.nil? errors.push("Local file `#{masked_location}` does not exist!") elsif content.blank? errors.push("Local file `#{masked_location}` is empty!") diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 89418bd6a21..553cbd819ad 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -39,12 +39,16 @@ module Gitlab private - def validate_content! + def validate_context! if !can_access_local_content? errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") elsif sha.nil? errors.push("Project `#{masked_project_name}` reference `#{masked_ref_name}` does not exist!") - elsif content.nil? + end + end + + def validate_content! + if content.nil? errors.push("Project `#{masked_project_name}` file `#{masked_location}` does not exist!") elsif content.blank? errors.push("Project `#{masked_project_name}` file `#{masked_location}` is empty!") @@ -58,7 +62,11 @@ module Gitlab end def can_access_local_content? - Ability.allowed?(context.user, :download_code, project) + strong_memoize(:can_access_local_content) do + context.logger.instrument(:config_file_project_validate_access) do + Ability.allowed?(context.user, :download_code, project) + end + end end def fetch_local_content diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 3984bf9e4f8..b0c540685d4 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -30,6 +30,10 @@ module Gitlab private + def validate_context! + # no-op + end + def validate_location! super diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 5fcf7c71bdf..53236cb317b 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -33,6 +33,10 @@ module Gitlab private + def validate_context! + # no-op + end + def validate_location! super diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 2a1060a6059..fc03ac125fd 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -8,13 +8,15 @@ module Gitlab include Gitlab::Utils::StrongMemoize FILE_CLASSES = [ - External::File::Remote, - External::File::Template, External::File::Local, External::File::Project, + External::File::Remote, + External::File::Template, External::File::Artifact ].freeze + FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze + Error = Class.new(StandardError) AmbigiousSpecificationError = Class.new(Error) TooManyIncludesError = Class.new(Error) @@ -120,9 +122,13 @@ module Gitlab file_class.new(location, context) end.select(&:matching?) - raise AmbigiousSpecificationError, "Include `#{masked_location(location.to_json)}` needs to match exactly one accessor!" unless matching.one? - - matching.first + if matching.one? + matching.first + elsif matching.empty? + raise AmbigiousSpecificationError, "`#{masked_location(location.to_json)}` does not have a valid subkey for include. Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + else + raise AmbigiousSpecificationError, "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + end end def verify!(location_object) diff --git a/lib/gitlab/ci/parsers/codequality/code_climate.rb b/lib/gitlab/ci/parsers/codequality/code_climate.rb index 628d50b84cb..14ea907edd8 100644 --- a/lib/gitlab/ci/parsers/codequality/code_climate.rb +++ b/lib/gitlab/ci/parsers/codequality/code_climate.rb @@ -5,23 +5,36 @@ module Gitlab module Parsers module Codequality class CodeClimate - def parse!(json_data, codequality_report) + def parse!(json_data, codequality_report, metadata = {}) root = Gitlab::Json.parse(json_data) - parse_all(root, codequality_report) + parse_all(root, codequality_report, metadata) rescue JSON::ParserError => e codequality_report.set_error_message("JSON parsing failed: #{e}") end private - def parse_all(root, codequality_report) + def parse_all(root, codequality_report, metadata) return unless root.present? root.each do |degradation| - break unless codequality_report.add_degradation(degradation) + break unless codequality_report.valid_degradation?(degradation) + + degradation['web_url'] = web_url(degradation, metadata) + codequality_report.add_degradation(degradation) end end + + def web_url(degradation, metadata) + return unless metadata[:project].present? && metadata[:commit_sha].present? + + path = degradation.dig('location', 'path') + line = degradation.dig('location', 'lines', 'begin') || + degradation.dig('location', 'positions', 'begin', 'line') + "#{Routing.url_helpers.project_blob_url( + metadata[:project], File.join(metadata[:commit_sha], path))}#L#{line}" + end end end end diff --git a/lib/gitlab/ci/parsers/coverage/sax_document.rb b/lib/gitlab/ci/parsers/coverage/sax_document.rb index 27cce0e3a3b..ddd9c80f5ea 100644 --- a/lib/gitlab/ci/parsers/coverage/sax_document.rb +++ b/lib/gitlab/ci/parsers/coverage/sax_document.rb @@ -76,7 +76,12 @@ module Gitlab # | /builds/foo/test/something | something | # | /builds/foo/test/ | nil | # | /builds/foo/test | nil | - node.split("#{project_path}/", 2)[1] + # | D:\builds\foo\bar\app\ | app\ | + unixify(node).split("#{project_path}/", 2)[1] + end + + def unixify(path) + path.tr('\\', '/') end def remove_matched_filenames diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index aa594ca4049..bc62fbe55ec 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -61,23 +61,19 @@ module Gitlab end def parse_components - data['components']&.each do |component_data| - type = component_data['type'] - next unless supported_component_type?(type) - + data['components']&.each_with_index do |component_data, index| component = ::Gitlab::Ci::Reports::Sbom::Component.new( - type: type, + type: component_data['type'], name: component_data['name'], + purl: component_data['purl'], version: component_data['version'] ) - report.add_component(component) + report.add_component(component) if component.ingestible? + rescue ::Sbom::PackageUrl::InvalidPackageUrl + report.add_error("/components/#{index}/purl is invalid") end end - - def supported_component_type?(type) - ::Enums::Sbom.component_types.include?(type.to_sym) - end end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 0c117d5f214..0ac012b9fd1 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -41,7 +41,7 @@ module Gitlab private - attr_reader :json_data, :report, :validate + attr_reader :json_data, :report, :validate, :project def valid? return true unless validate @@ -157,13 +157,7 @@ module Gitlab signature_value: value ) - if signature.valid? - signature - else - e = SecurityReportParserError.new("Vulnerability tracking signature is not valid: #{signature}") - Gitlab::ErrorTracking.track_exception(e) - nil - end + signature if signature.valid? end.compact end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 627a1f58715..ab5203252a2 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4] }.freeze VERSIONS_TO_REMOVE_IN_16_0 = [].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..3a859ca8bcf --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/cluster-image-scanning-report-format.json @@ -0,0 +1,984 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/container-scanning-report-format.json new file mode 100644 index 00000000000..95f9ce90af7 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/container-scanning-report-format.json @@ -0,0 +1,916 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..b2f39d6f070 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/coverage-fuzzing-report-format.json @@ -0,0 +1,874 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dast-report-format.json new file mode 100644 index 00000000000..2b86d7e40c9 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dast-report-format.json @@ -0,0 +1,1279 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dependency-scanning-report-format.json new file mode 100644 index 00000000000..29ba60b895e --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dependency-scanning-report-format.json @@ -0,0 +1,982 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/sast-report-format.json new file mode 100644 index 00000000000..238003f8eb2 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/sast-report-format.json @@ -0,0 +1,869 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/secret-detection-report-format.json new file mode 100644 index 00000000000..5cc55ea6409 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/secret-detection-report-format.json @@ -0,0 +1,893 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 76d4a05bf30..5ec04b4889e 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -117,7 +117,7 @@ module Gitlab logger.observe(:pipeline_size_count, pipeline.total_size) metrics.pipeline_size_histogram - .observe({ source: pipeline.source.to_s }, pipeline.total_size) + .observe({ source: pipeline.source.to_s, plan: project.actual_plan_name }, pipeline.total_size) end def observe_jobs_count_in_alive_pipelines diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb index 3dd9b85d9b2..1b9dd158733 100644 --- a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -16,18 +16,7 @@ module Gitlab private def ensure_environment(build) - return unless build.instance_of?(::Ci::Build) && build.has_environment? - - environment = ::Gitlab::Ci::Pipeline::Seed::Environment - .new(build, merge_request: @command.merge_request) - .to_resource - - if environment.persisted? - build.persisted_environment = environment - build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name }) - else - build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure) - end + ::Environments::CreateForBuildService.new.execute(build, merge_request: @command.merge_request) end end end diff --git a/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb index 8b26416edf7..2bb32a316be 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb @@ -21,7 +21,10 @@ module Gitlab class: self.class.name, message: MESSAGE, project_id: project.id, - plan: project.actual_plan_name) + plan: project.actual_plan_name, + project_path: project.path, + jobs_in_alive_pipelines_count: count_jobs_in_alive_pipelines + ) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 4bec8355732..654e24be8e1 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -25,8 +25,6 @@ module Gitlab return error('Failed to build the pipeline!') end - set_pipeline_name - raise Populate::PopulateError if pipeline.persisted? end @@ -36,15 +34,6 @@ module Gitlab private - def set_pipeline_name - return if Feature.disabled?(:pipeline_name, pipeline.project) || - @command.yaml_processor_result.workflow_name.blank? - - name = @command.yaml_processor_result.workflow_name - - pipeline.build_pipeline_metadata(project: pipeline.project, title: name) - end - def stage_names # We filter out `.pre/.post` stages, as they alone are not considered # a complete pipeline: diff --git a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb new file mode 100644 index 00000000000..35b907b669c --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class PopulateMetadata < Chain::Base + include Chain::Helpers + + def perform! + set_pipeline_name + return if pipeline.pipeline_metadata.nil? || pipeline.pipeline_metadata.valid? + + message = pipeline.pipeline_metadata.errors.full_messages.join(', ') + error("Failed to build pipeline metadata! #{message}") + end + + def break? + pipeline.pipeline_metadata&.errors&.any? + end + + private + + def set_pipeline_name + return if Feature.disabled?(:pipeline_name, pipeline.project) || + @command.yaml_processor_result.workflow_name.blank? + + name = @command.yaml_processor_result.workflow_name + name = ExpandVariables.expand(name, -> { global_context.variables.sort_and_expand_all }) + + pipeline.build_pipeline_metadata(project: pipeline.project, name: name) + end + + def global_context + Gitlab::Ci::Build::Context::Global.new( + pipeline, yaml_variables: @command.pipeline_seed.root_variables) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/deployment.rb b/lib/gitlab/ci/pipeline/seed/deployment.rb deleted file mode 100644 index 69dfd6be8d5..00000000000 --- a/lib/gitlab/ci/pipeline/seed/deployment.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Seed - class Deployment < Seed::Base - attr_reader :job, :environment - - def initialize(job, environment) - @job = job - @environment = environment - end - - def to_resource - return job.deployment if job.deployment - return unless job.starts_environment? - - deployment = ::Deployment.new(attributes) - - # If there is a validation error on environment creation, such as - # the name contains invalid character, the job will fall back to a - # non-environment job. - return unless deployment.valid? && deployment.environment.persisted? - - if cluster = deployment.environment.deployment_platform&.cluster - # double write cluster_id until 12.9: https://gitlab.com/gitlab-org/gitlab/issues/202628 - deployment.cluster_id = cluster.id - deployment.deployment_cluster = ::DeploymentCluster.new( - cluster_id: cluster.id, - kubernetes_namespace: cluster.kubernetes_namespace_for(deployment.environment, deployable: job) - ) - end - - # Allocate IID for deployments. - # This operation must be outside of transactions of pipeline creations. - deployment.ensure_project_iid! - - deployment - end - - private - - def attributes - { - project: job.project, - environment: environment, - user: job.user, - ref: job.ref, - tag: job.tag, - sha: job.sha, - on_stop: job.on_stop - } - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb deleted file mode 100644 index 8353bc523bf..00000000000 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Seed - class Environment < Seed::Base - attr_reader :job, :merge_request - - delegate :simple_variables, to: :job - - def initialize(job, merge_request: nil) - @job = job - @merge_request = merge_request - end - - def to_resource - environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| - # Initialize the attributes at creation - environment.auto_stop_in = expanded_auto_stop_in - environment.tier = deployment_tier - environment.merge_request = merge_request - end - end - - private - - def environments - job.project.environments - end - - def auto_stop_in - job.environment_auto_stop_in - end - - def deployment_tier - job.environment_tier_from_options - end - - def expanded_environment_name - job.expanded_environment_name - end - - def expanded_auto_stop_in - return unless auto_stop_in - - ExpandVariables.expand(auto_stop_in, -> { simple_variables.sort_and_expand_all }) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb index e1a15fb8d5b..9e609debeed 100644 --- a/lib/gitlab/ci/pipeline/seed/pipeline.rb +++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb @@ -32,6 +32,10 @@ module Gitlab end end + def root_variables + @context.root_variables + end + private def stage_seeds diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb index 353d359fde8..3196bf3fc6d 100644 --- a/lib/gitlab/ci/reports/codequality_reports.rb +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -37,8 +37,6 @@ module Gitlab end.to_h end - private - def valid_degradation?(degradation) JSONSchemer.schema(Pathname.new(CODECLIMATE_SCHEMA_PATH)).valid?(degradation) rescue StandardError => _ diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 198b34451b4..5188304f4ed 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -7,11 +7,34 @@ module Gitlab class Component attr_reader :component_type, :name, :version - def initialize(type:, name:, version:) + def initialize(type:, name:, purl:, version:) @component_type = type @name = name + @purl = purl @version = version end + + def ingestible? + supported_component_type? && supported_purl_type? + end + + def purl + return unless @purl + + ::Sbom::PackageUrl.parse(@purl) + end + + private + + def supported_component_type? + ::Enums::Sbom.component_types.include?(component_type.to_sym) + end + + def supported_purl_type? + return true unless purl + + ::Enums::Sbom.purl_types.include?(purl.type.to_sym) + end end end end diff --git a/lib/gitlab/ci/reports/sbom/report.rb b/lib/gitlab/ci/reports/sbom/report.rb index 4f84d12f78c..51fa8ce0d2e 100644 --- a/lib/gitlab/ci/reports/sbom/report.rb +++ b/lib/gitlab/ci/reports/sbom/report.rb @@ -12,6 +12,10 @@ module Gitlab @errors = [] end + def valid? + errors.empty? + end + def add_error(error) errors << error end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 911a7f5d358..dd9b9cc6d55 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -156,6 +156,14 @@ module Gitlab signatures.present? end + def false_positive? + flags.any?(&:false_positive?) + end + + def remediation_byte_offsets + remediations.map(&:byte_offsets).compact + end + def raw_metadata @raw_metadata ||= original_data.to_json end @@ -176,6 +184,10 @@ module Gitlab original_data['location'] end + def assets + original_data['assets'] || [] + end + # Returns either the max priority signature hex # or the location fingerprint def location_fingerprint diff --git a/lib/gitlab/ci/reports/security/flag.rb b/lib/gitlab/ci/reports/security/flag.rb index 8370dd60418..e1fbd4c0eff 100644 --- a/lib/gitlab/ci/reports/security/flag.rb +++ b/lib/gitlab/ci/reports/security/flag.rb @@ -27,6 +27,10 @@ module Gitlab description: description }.compact end + + def false_positive? + flag_type == :false_positive + end end end end diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb index b6372349f68..5c08381d5cc 100644 --- a/lib/gitlab/ci/reports/security/reports.rb +++ b/lib/gitlab/ci/reports/security/reports.rb @@ -23,6 +23,10 @@ module Gitlab end def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states, report_types = []) + if Feature.enabled?(:require_approval_on_scan_removal, pipeline.project) && scan_removed?(target_reports) + return true + end + unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) > vulnerabilities_allowed end @@ -36,6 +40,10 @@ module Gitlab new_uuids = unsafe_findings_uuids(severity_levels, report_types) - target_reports&.unsafe_findings_uuids(severity_levels, report_types).to_a new_uuids.count end + + def scan_removed?(target_reports) + (target_reports&.reports&.keys.to_a - reports.keys).any? + end end end end diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 004c2897b60..fb062683397 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -41,3 +41,4 @@ deploy1: stage: deploy script: - echo "Do your deploy here" + environment: production diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 01697f67b89..2474bc569d5 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -26,7 +26,7 @@ variables: before_script: - apt-get update -qq && apt-get install -y -qq unzip - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > ~/.sdkman/etc/config + - echo sdkman_auto_answer=true >> ~/.sdkman/etc/config - source ~/.sdkman/bin/sdkman-init.sh - sdk install gradle $GRADLE_VERSION < /dev/null - sdk use gradle $GRADLE_VERSION diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index d1018f1e769..fcf2ac7de7a 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html browser_performance: stage: performance diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index bb7e020b159..04b7dacf2dd 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html browser_performance: stage: performance diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 071eccbab0d..fc1f4f0cce8 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.19.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.21.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 071eccbab0d..fc1f4f0cce8 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.19.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.21.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index d994ed70ea9..7a208584c4c 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 7ad71625436..292b0a0036d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 10c843f60a6..ba03ad6304f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index eea1c397108..936d8751fe1 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -6,7 +6,7 @@ load_performance: DOCKER_TLS_CERTDIR: "" K6_IMAGE: loadimpact/k6 K6_VERSION: 0.27.0 - K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js + K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index 0513aae00a8..77048037915 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -38,7 +38,7 @@ kics-iac-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index c0ca821ebff..4600468ef30 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -200,7 +200,7 @@ nodejs-scan-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/package.json' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -221,7 +221,7 @@ phpcs-security-audit-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.php' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -242,7 +242,7 @@ pmd-apex-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.cls' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -263,7 +263,7 @@ security-code-scan-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.csproj' - '**/*.vbproj' @@ -287,7 +287,7 @@ semgrep-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.py' - '**/*.js' @@ -326,7 +326,7 @@ sobelow-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - 'mix.exs' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -351,7 +351,7 @@ spotbugs-sast: when: never - if: $SAST_DISABLED when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.groovy' - '**/*.scala' diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml index e6eba6f6406..6603ee4268e 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml @@ -29,7 +29,7 @@ secret_detection: rules: - if: $SECRET_DETECTION_DISABLED when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 1bd527a6ec0..5863da142f0 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -2,6 +2,9 @@ # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +# NOTE: This template is intended for internal GitLab use only and likely will not work properly +# in any other project. Do not include it in your pipeline configuration. +# For information on how to set up and use DAST, visit https://docs.gitlab.com/ee/user/application_security/dast/ stages: - build diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 701e08ba56d..733ba4e4954 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -2,6 +2,9 @@ # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +# NOTE: This template is intended for internal GitLab use only and likely will not work properly +# in any other project. Do not include it in your pipeline configuration. +# For information on how to set up and use DAST, visit https://docs.gitlab.com/ee/user/application_security/dast/ stages: - build diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml index 5b6af37977e..c75ff2e9ff8 100644 --- a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml @@ -2,6 +2,9 @@ # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml +# NOTE: This template is intended for internal GitLab use only and likely will not work properly +# in any other project. Do not include it in your pipeline configuration. +# For information on how to set up and use DAST, visit https://docs.gitlab.com/ee/user/application_security/dast/ stages: - build diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 4d0259fe678..51bcbd278d5 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -12,6 +12,7 @@ stages: - test - build - deploy + - cleanup fmt: extends: .terraform:fmt diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 019b970bc30..0b6c10293fc 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -12,6 +12,7 @@ stages: - test - build - deploy + - cleanup fmt: extends: .terraform:fmt diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 9a40a23b276..dd1676f25b6 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -13,7 +13,7 @@ image: variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project - TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend + TF_STATE_NAME: default # The name of the state file used by the GitLab Managed Terraform state backend cache: key: "${TF_ROOT}" diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 4579f31d7ac..9c967d48de1 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -14,7 +14,7 @@ image: variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project - TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend + TF_STATE_NAME: default # The name of the state file used by the GitLab Managed Terraform state backend cache: key: "${TF_ROOT}" @@ -27,12 +27,22 @@ cache: - cd "${TF_ROOT}" - gitlab-terraform fmt allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. .terraform:validate: &terraform_validate stage: validate script: - cd "${TF_ROOT}" - gitlab-terraform validate + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. .terraform:build: &terraform_build stage: build @@ -46,6 +56,11 @@ cache: - ${TF_ROOT}/plan.cache reports: terraform: ${TF_ROOT}/plan.json + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. .terraform:deploy: &terraform_deploy stage: deploy diff --git a/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml b/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml index 8a0913e8f66..47329a602b1 100644 --- a/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml @@ -8,6 +8,7 @@ stages: - deploy:production staging: + environment: staging image: python:2 stage: deploy:staging script: @@ -18,6 +19,7 @@ staging: - $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH production: + environment: production image: python:2 stage: deploy:production script: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index 2349c37c130..c3113ffebf3 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html stages: - build diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index 73ab5fcbe44..c9f0c173692 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html stages: - build diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index 53fabcfc721..bf5cfbb519d 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/load_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/code_quality.html stages: - build @@ -17,7 +17,7 @@ load_performance: variables: K6_IMAGE: loadimpact/k6 K6_VERSION: 0.27.0 - K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js + K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 50ce181095e..8dfb6c38b55 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -89,3 +89,4 @@ deploy_job: dependencies: - build_job - test_job + environment: production diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index cf5f04215ad..8db8ea3a720 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -171,16 +171,6 @@ module Gitlab end end - def strong_memoize_with(name, *args) - container = strong_memoize(name) { {} } - - if container.key?(args) - container[args] - else - container[args] = yield - end - end - def release return unless @pipeline.tag? diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index b6d6e1a3e5f..e9766061072 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -72,7 +72,8 @@ module Gitlab Collection.new(@variables.reject(&block)) end - def expand_value(value, keep_undefined: false, expand_file_vars: true, project: nil) + # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. + def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%) full_match = match[0] @@ -86,19 +87,26 @@ module Gitlab variable = self[variable_name] if variable # VARIABLE_NAME is an existing variable - next variable.value unless variable.file? - - # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 - if project - # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` - # when the variables are sent to Runner. - Gitlab::AppJsonLogger.info( - event: 'file_variable_is_referenced_in_another_variable', - project_id: project.id - ) + if variable.file? + # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 + if project + # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` + # when the variables are sent to Runner. + Gitlab::AppJsonLogger.info(event: 'file_variable_is_referenced_in_another_variable', + project_id: project.id, + variable: variable_name) + end + + expand_file_refs ? variable.value : full_match + elsif variable.raw? + # With `full_match`, we defer the expansion of raw variables to the runner. If we expand them here, + # the runner will not know the expanded value is a raw variable and it tries to expand it again. + # Discussion: https://gitlab.com/gitlab-org/gitlab/-/issues/353991#note_1103274951 + expand_raw_refs ? variable.value : full_match + else + variable.value end - expand_file_vars ? variable.value : full_match elsif keep_undefined full_match # we do not touch the variable definition else @@ -107,7 +115,8 @@ module Gitlab end end - def sort_and_expand_all(keep_undefined: false, expand_file_vars: true, project: nil) + # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. + def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) sorted = Sort.new(self) return self.class.new(self, sorted.errors) unless sorted.valid? @@ -122,7 +131,8 @@ module Gitlab # expand variables as they are added variable = item.to_runner_variable variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, - expand_file_vars: expand_file_vars, + expand_file_refs: expand_file_refs, + expand_raw_refs: expand_raw_refs, project: project) new_collection.append(variable) end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index ea2aa8f2db8..0fcf11121fa 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -21,9 +21,10 @@ module Gitlab @variable.fetch(:value) end - def raw + def raw? @variable.fetch(:raw) end + alias_method :raw, :raw? def file? @variable.fetch(:file) @@ -39,7 +40,7 @@ module Gitlab def depends_on strong_memoize(:depends_on) do - next if raw + next if raw? next unless self.class.possible_var_reference?(value) @@ -48,9 +49,8 @@ module Gitlab end ## - # If `file: true` has been provided we expose it, otherwise we - # don't expose `file` attribute at all (stems from what the runner - # expects). + # If `file: true` or `raw: true` has been provided we expose it, otherwise we + # don't expose `file` and `raw` attributes at all (stems from what the runner expects). # def to_runner_variable @variable.reject do |hash_key, hash_value| diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 5c3864362da..ff255543d3b 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -6,12 +6,17 @@ module Gitlab module Ci class YamlProcessor class Result - attr_reader :errors, :warnings + attr_reader :errors, :warnings, + :root_variables, :root_variables_with_prefill_data, + :stages, :jobs, + :workflow_rules, :workflow_name def initialize(ci_config: nil, errors: [], warnings: []) @ci_config = ci_config @errors = errors || [] @warnings = warnings || [] + + assign_valid_attributes if valid? end def valid? @@ -32,34 +37,10 @@ module Gitlab end end - def workflow_rules - @workflow_rules ||= @ci_config.workflow_rules - end - - def workflow_name - @workflow_name ||= @ci_config.workflow_name&.strip - end - - def root_variables - @root_variables ||= transform_to_array(@ci_config.variables) - end - - def jobs - @jobs ||= @ci_config.normalized_jobs - end - - def stages - @stages ||= @ci_config.stages - end - def included_templates @included_templates ||= @ci_config.included_templates end - def variables_with_data - @ci_config.variables_with_data - end - def yaml_variables_for(job_name) job = jobs[job_name] @@ -82,6 +63,22 @@ module Gitlab private + def assign_valid_attributes + @root_variables = if YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) + transform_to_array(@ci_config.variables_with_data) + else + transform_to_array(@ci_config.variables) + end + + @root_variables_with_prefill_data = @ci_config.variables_with_prefill_data + + @stages = @ci_config.stages + @jobs = @ci_config.normalized_jobs + + @workflow_rules = @ci_config.workflow_rules + @workflow_name = @ci_config.workflow_name&.strip + end + def stage_builds_attributes(stage) jobs.values .select { |job| job[:stage] == stage } @@ -129,14 +126,10 @@ module Gitlab start_in: job[:start_in], trigger: job[:trigger], bridge_needs: job.dig(:needs, :bridge)&.first, - release: release(job) + release: job[:release] }.compact }.compact end - def release(job) - job[:release] - end - def transform_to_array(variables) ::Gitlab::Ci::Variables::Helpers.transform_to_array(variables) end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index be08ada9d2f..b39d2a02f02 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -63,6 +63,15 @@ module Gitlab # # Sidekiq/Puma Single: This is called immediately. # + # - on_worker_stop (on worker process): + # + # Puma Cluster: Called in the worker process + # exactly once after it stops processing requests + # but before it shuts down. + # + # Sidekiq: Called after the scheduler shuts down but + # before the worker finishes ongoing jobs. + # # Blocks will be executed in the order in which they are registered. # class LifecycleEvents @@ -113,6 +122,10 @@ module Gitlab end end + def on_worker_stop(&block) + (@worker_stop_hooks ||= []) << block + end + # # Lifecycle integration methods (called from puma.rb, etc.) # @@ -137,6 +150,10 @@ module Gitlab call(:master_restart_hooks, @master_restart_hooks) end + def do_worker_stop + call(:worker_stop_hooks, @worker_stop_hooks) + end + # DEPRECATED alias_method :do_master_restart, :do_before_master_restart diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 5908de68687..957faf797b5 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -9,6 +9,10 @@ module Gitlab puma_master_max_memory_mb: 950, additional_puma_dev_max_memory_mb: 200) + # We are replacing PWK with Watchdog by using backward compatible RssMemoryLimit monitor by default. + # https://gitlab.com/groups/gitlab-org/-/epics/9119 + return if Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_MEMORY_WATCHDOG_ENABLED', true)) + require 'puma_worker_killer' PumaWorkerKiller.config do |config| diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb index 64950fb4eef..ff20833b5be 100644 --- a/lib/gitlab/config_checker/external_database_checker.rb +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -9,19 +9,23 @@ module Gitlab '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' def check - unsupported_database = Gitlab::Database + unsupported_databases = Gitlab::Database .database_base_models - .map { |_, model| Gitlab::Database::Reflection.new(model) } - .reject(&:postgresql_minimum_supported_version?) + .each_with_object({}) do |(database_name, base_model), databases| + database = Gitlab::Database::Reflection.new(base_model) - unsupported_database.map do |database| + databases[database_name] = database unless database.postgresql_minimum_supported_version? + end + + unsupported_databases.map do |database_name, database| { type: 'warning', - message: _('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \ - '%{pg_version_minimum} is required for this version of GitLab. ' \ + message: _('Database \'%{database_name}\' is using PostgreSQL %{pg_version_current}, ' \ + 'but PostgreSQL %{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ 'see %{pg_requirements_url} for details.') % \ { + database_name: database_name, pg_version_current: database.version, pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, pg_requirements_url: PG_REQUIREMENTS_LINK diff --git a/lib/gitlab/container_repository/tags/cache.rb b/lib/gitlab/container_repository/tags/cache.rb index 47a6e67a5a1..f9de16f002f 100644 --- a/lib/gitlab/container_repository/tags/cache.rb +++ b/lib/gitlab/container_repository/tags/cache.rb @@ -18,7 +18,7 @@ module Gitlab keys = tags.map(&method(:cache_key)) cached_tags_count = 0 - ::Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| tags.zip(redis.mget(keys)).each do |tag, created_at| next unless created_at @@ -45,7 +45,7 @@ module Gitlab now = Time.zone.now - ::Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| # we use a pipeline instead of a MSET because each tag has # a specific ttl redis.pipelined do |pipeline| @@ -66,6 +66,10 @@ module Gitlab def cache_key(tag) "container_repository:{#{@container_repository.id}}:tag:#{tag.name}:created_at" end + + def with_redis(&block) + ::Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index f1faade250e..29e8e631fb7 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -24,7 +24,7 @@ module Gitlab 'frame_src' => ContentSecurityPolicy::Directives.frame_src, 'img_src' => "'self' data: blob: http: https:", 'manifest_src' => "'self'", - 'media_src' => "'self' data:", + 'media_src' => "'self' data: http: https:", 'script_src' => ContentSecurityPolicy::Directives.script_src, 'style_src' => "'self' 'unsafe-inline'", 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 4640f85bb0a..8eda871770b 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -12,7 +12,7 @@ module Gitlab author_url = build_author_url(build.commit, commit) - { + data = { object_kind: 'build', ref: build.ref, @@ -68,6 +68,10 @@ module Gitlab environment: build_environment(build) } + + data[:retries_count] = build.retries_count if Feature.enabled?(:job_webhook_retries_count, project) + + data end private @@ -91,7 +95,7 @@ module Gitlab end def build_environment(build) - return unless build.has_environment? + return unless build.has_environment_keyword? { name: build.expanded_environment_name, diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index a75c7c539ae..939eaa377aa 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -105,6 +105,7 @@ module Gitlab target_project_id: merge_request.target_project_id, state: merge_request.state, merge_status: merge_request.public_merge_status, + detailed_merge_status: detailed_merge_status(merge_request), url: Gitlab::UrlBuilder.build(merge_request) } end @@ -146,7 +147,7 @@ module Gitlab end def environment_hook_attrs(build) - return unless build.has_environment? + return unless build.has_environment_keyword? { name: build.expanded_environment_name, @@ -154,6 +155,10 @@ module Gitlab deployment_tier: build.persisted_environment.try(:tier) } end + + def detailed_merge_status(merge_request) + ::MergeRequests::Mergeability::DetailedMergeStatusService.new(merge_request: merge_request).execute.to_s + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index dd84127459d..04cf056199c 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -56,7 +56,7 @@ module Gitlab # Note that we use ActiveRecord::Base here and not ApplicationRecord. # This is deliberate, as we also use these classes to apply load # balancing to, and the load balancer must be enabled for _all_ models - # that inher from ActiveRecord::Base; not just our own models that + # that inherit from ActiveRecord::Base; not just our own models that # inherit from ApplicationRecord. main: ::ActiveRecord::Base, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil @@ -217,13 +217,13 @@ module Gitlab Rails.application.config.paths['db'].each do |db_path| path = Rails.root.join(db_path, 'post_migrate').to_s - unless Rails.application.config.paths['db/migrate'].include? path - Rails.application.config.paths['db/migrate'] << path + next if Rails.application.config.paths['db/migrate'].include? path - # Rails memoizes migrations at certain points where it won't read the above - # path just yet. As such we must also update the following list of paths. - ActiveRecord::Migrator.migrations_paths << path - end + Rails.application.config.paths['db/migrate'] << path + + # Rails memoizes migrations at certain points where it won't read the above + # path just yet. As such we must also update the following list of paths. + ActiveRecord::Migrator.migrations_paths << path end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 81898a59da7..6b7ff308c7e 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -14,7 +14,8 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError, - ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout].freeze + ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout, + ActiveRecord::QueryCanceled].freeze belongs_to :batched_migration, foreign_key: :batched_background_migration_id has_many :batched_job_transition_logs, foreign_key: :batched_background_migration_job_id @@ -112,7 +113,10 @@ module Gitlab end def can_split?(exception) - attempts >= MAX_ATTEMPTS && TIMEOUT_EXCEPTIONS.include?(exception&.class) && batch_size > sub_batch_size && batch_size > 1 + attempts >= MAX_ATTEMPTS && + exception&.class&.in?(TIMEOUT_EXCEPTIONS) && + batch_size > sub_batch_size && + batch_size > 1 end def split_and_retry! diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 92cafd1d00e..61a660ad14c 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -94,8 +94,21 @@ module Gitlab end def self.active_migration(connection:) + active_migrations_distinct_on_table(connection: connection, limit: 1).first + end + + def self.find_executable(id, connection:) for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection)) - .executable.queue_order.first + .executable.find_by_id(id) + end + + def self.active_migrations_distinct_on_table(connection:, limit:) + distinct_on_table = select('DISTINCT ON (table_name) id') + .for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection)) + .executable + .order(table_name: :asc, id: :asc) + + where(id: distinct_on_table).queue_order.limit(limit) end def self.successful_rows_counts(migrations) diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index c4a9cf8b80f..bf6ebb21f7d 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -40,6 +40,7 @@ atlassian_identities: :gitlab_main audit_events_external_audit_event_destinations: :gitlab_main audit_events: :gitlab_main audit_events_streaming_headers: :gitlab_main +audit_events_streaming_event_type_filters: :gitlab_main authentication_events: :gitlab_main award_emoji: :gitlab_main aws_roles: :gitlab_main @@ -167,6 +168,7 @@ dast_site_profiles_pipelines: :gitlab_main dast_sites: :gitlab_main dast_site_tokens: :gitlab_main dast_site_validations: :gitlab_main +dependency_proxy_blob_states: :gitlab_main dependency_proxy_blobs: :gitlab_main dependency_proxy_group_settings: :gitlab_main dependency_proxy_image_ttl_group_policies: :gitlab_main @@ -206,7 +208,6 @@ events: :gitlab_main evidences: :gitlab_main experiments: :gitlab_main experiment_subjects: :gitlab_main -experiment_users: :gitlab_main external_approval_rules: :gitlab_main external_approval_rules_protected_branches: :gitlab_main external_pull_requests: :gitlab_ci @@ -342,6 +343,7 @@ namespace_limits: :gitlab_main namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main namespace_ci_cd_settings: :gitlab_main +namespace_commit_emails: :gitlab_main namespace_settings: :gitlab_main namespace_details: :gitlab_main namespaces: :gitlab_main @@ -363,6 +365,7 @@ operations_scopes: :gitlab_main operations_strategies: :gitlab_main operations_strategies_user_lists: :gitlab_main operations_user_lists: :gitlab_main +p_ci_builds_metadata: :gitlab_ci packages_build_infos: :gitlab_main packages_cleanup_policies: :gitlab_main packages_composer_cache_files: :gitlab_main @@ -451,6 +454,7 @@ projects: :gitlab_main projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main +project_wiki_repositories: :gitlab_main project_wiki_repository_states: :gitlab_main prometheus_alert_events: :gitlab_main prometheus_alerts: :gitlab_main diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 59b08fac7e9..50472bd5780 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -57,7 +57,8 @@ module Gitlab record_type: 'A', interval: 60, disconnect_timeout: 120, - use_tcp: false + use_tcp: false, + max_replica_pools: nil } end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 0881025b425..cb3a378ad64 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -119,6 +119,13 @@ module Gitlab connection = pool.connection transaction_open = connection.transaction_open? + if attempt && attempt > 1 + ::Gitlab::Database::LoadBalancing::Logger.warn( + event: :read_write_retry, + message: 'A read_write block was retried because of connection error' + ) + end + yield connection rescue StandardError => e # No leaking will happen on the final attempt. Leaks are caused by subsequent retries diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index dfd4892371c..52a9e8798d4 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -48,6 +48,7 @@ module Gitlab # forcefully disconnected. # use_tcp - Use TCP instaed of UDP to look up resources # load_balancer - The load balancer instance to use + # rubocop:disable Metrics/ParameterLists def initialize( load_balancer, nameserver:, @@ -56,7 +57,8 @@ module Gitlab record_type: 'A', interval: 60, disconnect_timeout: 120, - use_tcp: false + use_tcp: false, + max_replica_pools: nil ) @nameserver = nameserver @port = port @@ -66,7 +68,9 @@ module Gitlab @disconnect_timeout = disconnect_timeout @use_tcp = use_tcp @load_balancer = load_balancer + @max_replica_pools = max_replica_pools end + # rubocop:enable Metrics/ParameterLists def start Thread.new do @@ -170,6 +174,8 @@ module Gitlab addresses_from_srv_record(response) end + addresses = sampler.sample(addresses) + raise EmptyDnsResponse if addresses.empty? # Addresses are sorted so we can directly compare the old and new @@ -221,6 +227,11 @@ module Gitlab def addresses_from_a_record(resources) resources.map { |r| Address.new(r.address.to_s) } end + + def sampler + @sampler ||= ::Gitlab::Database::LoadBalancing::ServiceDiscovery::Sampler + .new(max_replica_pools: @max_replica_pools) + end end end end diff --git a/lib/gitlab/database/load_balancing/service_discovery/sampler.rb b/lib/gitlab/database/load_balancing/service_discovery/sampler.rb new file mode 100644 index 00000000000..71870214156 --- /dev/null +++ b/lib/gitlab/database/load_balancing/service_discovery/sampler.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class ServiceDiscovery + class Sampler + def initialize(max_replica_pools:, seed: Random.new_seed) + # seed must be set once and consistent + # for every invocation of #sample on + # the same instance of Sampler + @seed = seed + @max_replica_pools = max_replica_pools + end + + def sample(addresses) + return addresses if @max_replica_pools.nil? || addresses.count <= @max_replica_pools + + ::Gitlab::Database::LoadBalancing::Logger.info( + event: :host_list_limit_exceeded, + message: "Host list length exceeds max_replica_pools so random hosts will be chosen.", + max_replica_pools: @max_replica_pools, + total_host_list_length: addresses.count, + randomization_seed: @seed + ) + + # First sort them in case the ordering from DNS server changes + # then randomly order all addresses using consistent seed so + # this process always gives the same set for this instance of + # Sampler + addresses = addresses.sort + addresses = addresses.shuffle(random: Random.new(@seed)) + + # Group by hostname so that we can sample evenly across hosts + addresses_by_host = addresses.group_by(&:hostname) + + selected_addresses = [] + while selected_addresses.count < @max_replica_pools + # Loop over all hostnames grabbing one address at a time to + # evenly distribute across all hostnames + addresses_by_host.each do |host, addresses| + next if addresses.empty? + + selected_addresses << addresses.pop + + break unless selected_addresses.count < @max_replica_pools + end + end + + selected_addresses + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 3180289ec69..737852d5ccb 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -4,7 +4,7 @@ module Gitlab module Database module LoadBalancing class SidekiqServerMiddleware - JobReplicaNotUpToDate = Class.new(StandardError) + JobReplicaNotUpToDate = Class.new(::Gitlab::SidekiqMiddleware::RetryError) MINIMUM_DELAY_INTERVAL_SECONDS = 0.8 diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index fe75cd763b4..2594ee04b35 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -5,6 +5,11 @@ module Gitlab class LockWritesManager TRIGGER_FUNCTION_NAME = 'gitlab_schema_prevent_write' + # Triggers to block INSERT / UPDATE / DELETE + # Triggers on TRUNCATE are not added to the information_schema.triggers + # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us + EXPECTED_TRIGGER_RECORD_COUNT = 3 + def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) @table_name = table_name @connection = connection @@ -20,7 +25,7 @@ module Gitlab AND trigger_name = '#{write_trigger_name(table_name)}' SQL - connection.select_value(query) == 3 + connection.select_value(query) == EXPECTED_TRIGGER_RECORD_COUNT end def lock_writes diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index df40e3b3868..16416dd2507 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -6,6 +6,10 @@ module Gitlab include Migrations::ReestablishedConnectionStack include Migrations::BackgroundMigrationHelpers include Migrations::BatchedBackgroundMigrationHelpers + include Migrations::LockRetriesHelpers + include Migrations::TimeoutHelpers + include Migrations::ConstraintsHelpers + include Migrations::ExtensionHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers @@ -22,8 +26,6 @@ module Gitlab super(table_name, connection: connection, **kwargs) end - # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS - MAX_IDENTIFIER_NAME_LENGTH = 63 DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze # Adds `created_at` and `updated_at` columns with timezone information. @@ -146,6 +148,12 @@ module Gitlab 'in the body of your migration class' end + if !options.delete(:allow_partition) && partition?(table_name) + raise ArgumentError, 'add_concurrent_index can not be used on a partitioned ' \ + 'table. Please use add_concurrent_partitioned_index on the partitioned table ' \ + 'as we need to create indexes on each partition and an index on the parent table' + end + options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, **options) @@ -202,6 +210,12 @@ module Gitlab 'in the body of your migration class' end + if partition?(table_name) + raise ArgumentError, 'remove_concurrent_index can not be used on a partitioned ' \ + 'table. Please use remove_concurrent_partitioned_index_by_name on the partitioned table ' \ + 'as we need to remove the index on the parent table' + end + options = options.merge({ algorithm: :concurrently }) unless index_exists?(table_name, column_name, **options) @@ -231,6 +245,12 @@ module Gitlab 'in the body of your migration class' end + if partition?(table_name) + raise ArgumentError, 'remove_concurrent_index_by_name can not be used on a partitioned ' \ + 'table. Please use remove_concurrent_partitioned_index_by_name on the partitioned table ' \ + 'as we need to remove the index on the parent table' + end + index_name = index_name[:name] if index_name.is_a?(Hash) raise 'remove_concurrent_index_by_name must get an index name as the second argument' if index_name.blank? @@ -360,97 +380,6 @@ module Gitlab "#{prefix}#{hashed_identifier}" end - # Long-running migrations may take more than the timeout allowed by - # the database. Disable the session's statement timeout to ensure - # migrations don't get killed prematurely. - # - # There are two possible ways to disable the statement timeout: - # - # - Per transaction (this is the preferred and default mode) - # - Per connection (requires a cleanup after the execution) - # - # When using a per connection disable statement, code must be inside - # a block so we can automatically execute `RESET statement_timeout` after block finishes - # otherwise the statement will still be disabled until connection is dropped - # or `RESET statement_timeout` is executed - def disable_statement_timeout - if block_given? - if statement_timeout_disabled? - # Don't do anything if the statement_timeout is already disabled - # Allows for nested calls of disable_statement_timeout without - # resetting the timeout too early (before the outer call ends) - yield - else - begin - execute('SET statement_timeout TO 0') - - yield - ensure - execute('RESET statement_timeout') - end - end - else - unless transaction_open? - raise <<~ERROR - Cannot call disable_statement_timeout() without a transaction open or outside of a transaction block. - If you don't want to use a transaction wrap your code in a block call: - - disable_statement_timeout { # code that requires disabled statement here } - - This will make sure statement_timeout is disabled before and reset after the block execution is finished. - ERROR - end - - execute('SET LOCAL statement_timeout TO 0') - end - end - - # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts. - # The timings can be controlled via the +timing_configuration+ parameter. - # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+. - # - # Note this helper uses subtransactions when run inside an already open transaction. - # - # ==== Examples - # # Invoking without parameters - # with_lock_retries do - # drop_table :my_table - # end - # - # # Invoking with custom +timing_configuration+ - # t = [ - # [1.second, 1.second], - # [2.seconds, 2.seconds] - # ] - # - # with_lock_retries(timing_configuration: t) do - # drop_table :my_table # this will be retried twice - # end - # - # # Disabling the retries using an environment variable - # > export DISABLE_LOCK_RETRIES=true - # - # with_lock_retries do - # drop_table :my_table # one invocation, it will not retry at all - # end - # - # ==== Parameters - # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` - # * +logger+ - [Gitlab::JsonLogger] - # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` - def with_lock_retries(*args, **kwargs, &block) - raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) - merged_args = { - connection: connection, - klass: self.class, - logger: Gitlab::BackgroundMigration::Logger, - allow_savepoints: true - }.merge(kwargs) - - Gitlab::Database::WithLockRetries.new(**merged_args) - .run(raise_on_exhaustion: raise_on_exhaustion, &block) - end - def true_value Database.true_value end @@ -796,6 +725,10 @@ module Gitlab install_rename_triggers(table, old, new) end + def convert_to_type_column(column, from_type, to_type) + "#{column}_convert_#{from_type}_to_#{to_type}" + end + def convert_to_bigint_column(column) "#{column}_convert_to_bigint" end @@ -826,7 +759,22 @@ module Gitlab # columns - The name, or array of names, of the column(s) that we want to convert to bigint. # primary_key - The name of the primary key column (most often :id) def initialize_conversion_of_integer_to_bigint(table, columns, primary_key: :id) - create_temporary_columns_and_triggers(table, columns, primary_key: primary_key, data_type: :bigint) + mappings = Array(columns).map do |c| + { + c => { + from_type: :int, + to_type: :bigint, + default_value: 0 + } + } + end.reduce(&:merge) + + create_temporary_columns_and_triggers( + table, + mappings, + primary_key: primary_key, + old_bigint_column_naming: true + ) end # Reverts `initialize_conversion_of_integer_to_bigint` @@ -849,9 +797,23 @@ module Gitlab # table - The name of the database table containing the columns # columns - The name, or array of names, of the column(s) that we have converted to bigint. # primary_key - The name of the primary key column (most often :id) - def restore_conversion_of_integer_to_bigint(table, columns, primary_key: :id) - create_temporary_columns_and_triggers(table, columns, primary_key: primary_key, data_type: :int) + mappings = Array(columns).map do |c| + { + c => { + from_type: :bigint, + to_type: :int, + default_value: 0 + } + } + end.reduce(&:merge) + + create_temporary_columns_and_triggers( + table, + mappings, + primary_key: primary_key, + old_bigint_column_naming: true + ) end # Backfills the new columns used in an integer-to-bigint conversion using background migrations. @@ -947,43 +909,6 @@ module Gitlab execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end - def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) - Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - - Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information - migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( - Gitlab::Database.gitlab_schemas_for_connection(connection), - job_class_name, table_name, column_name, job_arguments - ) - - configuration = { - job_class_name: job_class_name, - table_name: table_name, - column_name: column_name, - job_arguments: job_arguments - } - - return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil? - - return if migration.finished? - - finalize_batched_background_migration(job_class_name: job_class_name, table_name: table_name, column_name: column_name, job_arguments: job_arguments) if finalize - - unless migration.reload.finished? # rubocop:disable Cop/ActiveRecordAssociationReload - raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ - "but it is '#{migration.status_name}':" \ - "\t#{configuration}" \ - "\n\n" \ - "Finalize it manually by running the following command in a `bash` or `sh` shell:" \ - "\n\n" \ - "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ - "\n\n" \ - "For more information, check the documentation" \ - "\n\n" \ - "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" - end - end - # Returns an Array containing the indexes for the given column def indexes_for(table, column) column = column.to_s @@ -1102,6 +1027,24 @@ module Gitlab rescue ArgumentError end + # Remove any instances of deprecated job classes lingering in queues. + # + # rubocop:disable Cop/SidekiqApiUsage + def sidekiq_remove_jobs(job_klass:) + Sidekiq::Queue.new(job_klass.queue).each do |job| + job.delete if job.klass == job_klass.to_s + end + + Sidekiq::RetrySet.new.each do |retri| + retri.delete if retri.klass == job_klass.to_s + end + + Sidekiq::ScheduledSet.new.each do |scheduled| + scheduled.delete if scheduled.klass == job_klass.to_s + end + end + # rubocop:enable Cop/SidekiqApiUsage + def sidekiq_queue_migrate(queue_from, to:) while sidekiq_queue_length(queue_from) > 0 Sidekiq.redis do |conn| @@ -1194,320 +1137,6 @@ into similar problems in the future (e.g. when new tables are created). execute(sql) end - # Returns the name for a check constraint - # - # type: - # - Any value, as long as it is unique - # - Constraint names are unique per table in Postgres, and, additionally, - # we can have multiple check constraints over a column - # So we use the (table, column, type) triplet as a unique name - # - e.g. we use 'max_length' when adding checks for text limits - # or 'not_null' when adding a NOT NULL constraint - # - def check_constraint_name(table, column, type) - identifier = "#{table}_#{column}_check_#{type}" - # Check concurrent_foreign_key_name() for info on why we use a hash - hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) - - "check_#{hashed_identifier}" - end - - def check_constraint_exists?(table, constraint_name) - # Constraint names are unique per table in Postgres, not per schema - # Two tables can have constraints with the same name, so we filter by - # the table name in addition to using the constraint_name - check_sql = <<~SQL - SELECT COUNT(*) - FROM pg_catalog.pg_constraint con - INNER JOIN pg_catalog.pg_class rel - ON rel.oid = con.conrelid - INNER JOIN pg_catalog.pg_namespace nsp - ON nsp.oid = con.connamespace - WHERE con.contype = 'c' - AND con.conname = #{connection.quote(constraint_name)} - AND nsp.nspname = #{connection.quote(current_schema)} - AND rel.relname = #{connection.quote(table)} - SQL - - connection.select_value(check_sql) > 0 - end - - # Adds a check constraint to a table - # - # This method is the generic helper for adding any check constraint - # More specialized helpers may use it (e.g. add_text_limit or add_not_null) - # - # This method only requires minimal locking: - # - The constraint is added using NOT VALID - # This allows us to add the check constraint without validating it - # - The check will be enforced for new data (inserts) coming in - # - If `validate: true` the constraint is also validated - # Otherwise, validate_check_constraint() can be used at a later stage - # - Check comments on add_concurrent_foreign_key for more info - # - # table - The table the constraint will be added to - # check - The check clause to add - # e.g. 'char_length(name) <= 5' or 'store IS NOT NULL' - # constraint_name - The name of the check constraint (otherwise auto-generated) - # Should be unique per table (not per column) - # validate - Whether to validate the constraint in this call - # - def add_check_constraint(table, check, constraint_name, validate: true) - # Transactions would result in ALTER TABLE locks being held for the - # duration of the transaction, defeating the purpose of this method. - validate_not_in_transaction!(:add_check_constraint) - - validate_check_constraint_name!(constraint_name) - - if check_constraint_exists?(table, constraint_name) - warning_message = <<~MESSAGE - Check constraint was not created because it exists already - (this may be due to an aborted migration or similar) - table: #{table}, check: #{check}, constraint name: #{constraint_name} - MESSAGE - - Gitlab::AppLogger.warn warning_message - else - # Only add the constraint without validating it - # Even though it is fast, ADD CONSTRAINT requires an EXCLUSIVE lock - # Use with_lock_retries to make sure that this operation - # will not timeout on tables accessed by many processes - with_lock_retries do - execute <<-EOF.strip_heredoc - ALTER TABLE #{table} - ADD CONSTRAINT #{constraint_name} - CHECK ( #{check} ) - NOT VALID; - EOF - end - end - - if validate - validate_check_constraint(table, constraint_name) - end - end - - def validate_check_constraint(table, constraint_name) - validate_check_constraint_name!(constraint_name) - - unless check_constraint_exists?(table, constraint_name) - raise missing_schema_object_message(table, "check constraint", constraint_name) - end - - disable_statement_timeout do - # VALIDATE CONSTRAINT only requires a SHARE UPDATE EXCLUSIVE LOCK - # It only conflicts with other validations and creating indexes - execute("ALTER TABLE #{table} VALIDATE CONSTRAINT #{constraint_name};") - end - end - - def remove_check_constraint(table, constraint_name) - # This is technically not necessary, but aligned with add_check_constraint - # and allows us to continue use with_lock_retries here - validate_not_in_transaction!(:remove_check_constraint) - - validate_check_constraint_name!(constraint_name) - - # DROP CONSTRAINT requires an EXCLUSIVE lock - # Use with_lock_retries to make sure that this will not timeout - with_lock_retries do - execute <<-EOF.strip_heredoc - ALTER TABLE #{table} - DROP CONSTRAINT IF EXISTS #{constraint_name} - EOF - end - end - - # Copies all check constraints for the old column to the new column. - # - # table - The table containing the columns. - # old - The old column. - # new - The new column. - # schema - The schema the table is defined for - # If it is not provided, then the current_schema is used - def copy_check_constraints(table, old, new, schema: nil) - if transaction_open? - raise 'copy_check_constraints can not be run inside a transaction' - end - - unless column_exists?(table, old) - raise "Column #{old} does not exist on #{table}" - end - - unless column_exists?(table, new) - raise "Column #{new} does not exist on #{table}" - end - - table_with_schema = schema.present? ? "#{schema}.#{table}" : table - - check_constraints_for(table, old, schema: schema).each do |check_c| - validate = !(check_c["constraint_def"].end_with? "NOT VALID") - - # Normalize: - # - Old constraint definitions: - # '(char_length(entity_path) <= 5500)' - # - Definitionss from pg_get_constraintdef(oid): - # 'CHECK ((char_length(entity_path) <= 5500))' - # - Definitions from pg_get_constraintdef(oid, pretty_bool): - # 'CHECK (char_length(entity_path) <= 5500)' - # - Not valid constraints: 'CHECK (...) NOT VALID' - # to a single format that we can use: - # '(char_length(entity_path) <= 5500)' - check_definition = check_c["constraint_def"] - .sub(/^\s*(CHECK)?\s*\({0,2}/, '(') - .sub(/\){0,2}\s*(NOT VALID)?\s*$/, ')') - - constraint_name = begin - if check_definition == "(#{old} IS NOT NULL)" - not_null_constraint_name(table_with_schema, new) - elsif check_definition.start_with? "(char_length(#{old}) <=" - text_limit_name(table_with_schema, new) - else - check_constraint_name(table_with_schema, new, 'copy_check_constraint') - end - end - - add_check_constraint( - table_with_schema, - check_definition.gsub(old.to_s, new.to_s), - constraint_name, - validate: validate - ) - end - end - - # Migration Helpers for adding limit to text columns - def add_text_limit(table, column, limit, constraint_name: nil, validate: true) - add_check_constraint( - table, - "char_length(#{column}) <= #{limit}", - text_limit_name(table, column, name: constraint_name), - validate: validate - ) - end - - def validate_text_limit(table, column, constraint_name: nil) - validate_check_constraint(table, text_limit_name(table, column, name: constraint_name)) - end - - def remove_text_limit(table, column, constraint_name: nil) - remove_check_constraint(table, text_limit_name(table, column, name: constraint_name)) - end - - def check_text_limit_exists?(table, column, constraint_name: nil) - check_constraint_exists?(table, text_limit_name(table, column, name: constraint_name)) - end - - # Migration Helpers for managing not null constraints - def add_not_null_constraint(table, column, constraint_name: nil, validate: true) - if column_is_nullable?(table, column) - add_check_constraint( - table, - "#{column} IS NOT NULL", - not_null_constraint_name(table, column, name: constraint_name), - validate: validate - ) - else - warning_message = <<~MESSAGE - NOT NULL check constraint was not created: - column #{table}.#{column} is already defined as `NOT NULL` - MESSAGE - - Gitlab::AppLogger.warn warning_message - end - end - - def validate_not_null_constraint(table, column, constraint_name: nil) - validate_check_constraint( - table, - not_null_constraint_name(table, column, name: constraint_name) - ) - end - - def remove_not_null_constraint(table, column, constraint_name: nil) - remove_check_constraint( - table, - not_null_constraint_name(table, column, name: constraint_name) - ) - end - - def check_not_null_constraint_exists?(table, column, constraint_name: nil) - check_constraint_exists?( - table, - not_null_constraint_name(table, column, name: constraint_name) - ) - end - - def create_extension(extension) - execute('CREATE EXTENSION IF NOT EXISTS %s' % extension) - rescue ActiveRecord::StatementInvalid => e - dbname = ApplicationRecord.database.database_name - user = ApplicationRecord.database.username - - warn(<<~MSG) if e.to_s =~ /permission denied/ - GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but - the database user is not allowed to install the extension. - - You can either install the extension manually using a database superuser: - - CREATE EXTENSION IF NOT EXISTS #{extension} - - Or, you can solve this by logging in to the GitLab - database (#{dbname}) using a superuser and running: - - ALTER #{user} WITH SUPERUSER - - This query will grant the user superuser permissions, ensuring any database extensions - can be installed through migrations. - - For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. - MSG - - raise - end - - def drop_extension(extension) - execute('DROP EXTENSION IF EXISTS %s' % extension) - rescue ActiveRecord::StatementInvalid => e - dbname = ApplicationRecord.database.database_name - user = ApplicationRecord.database.username - - warn(<<~MSG) if e.to_s =~ /permission denied/ - This migration attempts to drop the PostgreSQL extension '#{extension}' - installed in database '#{dbname}', but the database user is not allowed - to drop the extension. - - You can either drop the extension manually using a database superuser: - - DROP EXTENSION IF EXISTS #{extension} - - Or, you can solve this by logging in to the GitLab - database (#{dbname}) using a superuser and running: - - ALTER #{user} WITH SUPERUSER - - This query will grant the user superuser permissions, ensuring any database extensions - can be dropped through migrations. - - For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. - MSG - - raise - end - - def rename_constraint(table_name, old_name, new_name) - execute <<~SQL - ALTER TABLE #{quote_table_name(table_name)} - RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} - SQL - end - - def drop_constraint(table_name, constraint_name, cascade: false) - execute <<~SQL - ALTER TABLE #{quote_table_name(table_name)} DROP CONSTRAINT #{quote_column_name(constraint_name)} #{cascade_statement(cascade)} - SQL - end - def add_primary_key_using_index(table_name, pk_name, index_to_use) execute <<~SQL ALTER TABLE #{quote_table_name(table_name)} ADD CONSTRAINT #{quote_table_name(pk_name)} PRIMARY KEY USING INDEX #{quote_table_name(index_to_use)} @@ -1536,17 +1165,20 @@ into similar problems in the future (e.g. when new tables are created). SQL end - private + # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + def create_temporary_columns_and_triggers(table, mappings, primary_key: :id, old_bigint_column_naming: false) + raise ArgumentError, "No mappings for column conversion provided" if mappings.blank? - def multiple_columns(columns, separator: ', ') - Array.wrap(columns).join(separator) - end + unless mappings.values.all? { |values| mapping_has_required_columns?(values) } + raise ArgumentError, "Some mappings don't have required keys provided" + end - def cascade_statement(cascade) - cascade ? 'CASCADE' : '' - end + neutral_values_for_type = { + int: 0, + bigint: 0, + uuid: '00000000-0000-0000-0000-000000000000' + } - def create_temporary_columns_and_triggers(table, columns, primary_key: :id, data_type: :bigint) unless table_exists?(table) raise "Table #{table} does not exist" end @@ -1555,7 +1187,7 @@ into similar problems in the future (e.g. when new tables are created). raise "Column #{primary_key} does not exist on #{table}" end - columns = Array.wrap(columns) + columns = mappings.keys columns.each do |column| next if column_exists?(table, column) @@ -1564,67 +1196,88 @@ into similar problems in the future (e.g. when new tables are created). check_trigger_permissions!(table) - conversions = columns.to_h { |column| [column, convert_to_bigint_column(column)] } + if old_bigint_column_naming + mappings.each do |column, params| + params.merge!( + temporary_column_name: convert_to_bigint_column(column) + ) + end + else + mappings.each do |column, params| + params.merge!( + temporary_column_name: convert_to_type_column(column, params[:from_type], params[:to_type]) + ) + end + end with_lock_retries do - conversions.each do |(source_column, temporary_name)| - column = column_for(table, source_column) + mappings.each do |(column_name, params)| + column = column_for(table, column_name) + temporary_name = params[:temporary_column_name] + data_type = params[:to_type] + default_value = params[:default_value] if (column.name.to_s == primary_key.to_s) || !column.null # If the column to be converted is either a PK or is defined as NOT NULL, # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow # That way, we skip the expensive validation step required to add # a NOT NULL constraint at the end of the process - add_column(table, temporary_name, data_type, default: column.default || 0, null: false) + add_column( + table, + temporary_name, + data_type, + default: column.default || default_value || neutral_values_for_type.fetch(data_type), + null: false + ) else - add_column(table, temporary_name, data_type, default: column.default) + add_column( + table, + temporary_name, + data_type, + default: column.default + ) end end - install_rename_triggers(table, conversions.keys, conversions.values) + old_column_names = mappings.keys + temporary_column_names = mappings.values.map { |v| v[:temporary_column_name] } + install_rename_triggers(table, old_column_names, temporary_column_names) end end + # rubocop:enable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity - def validate_check_constraint_name!(constraint_name) - if constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH - raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters" + def partition?(table_name) + if view_exists?(:postgres_partitions) + Gitlab::Database::PostgresPartition.partition_exists?(table_name) + else + Gitlab::Database::PostgresPartition.legacy_partition_exists?(table_name) end end - # Returns an ActiveRecord::Result containing the check constraints - # defined for the given column. - # - # If the schema is not provided, then the current_schema is used - def check_constraints_for(table, column, schema: nil) - check_sql = <<~SQL - SELECT - ccu.table_schema as schema_name, - ccu.table_name as table_name, - ccu.column_name as column_name, - con.conname as constraint_name, - pg_get_constraintdef(con.oid) as constraint_def - FROM pg_catalog.pg_constraint con - INNER JOIN pg_catalog.pg_class rel - ON rel.oid = con.conrelid - INNER JOIN pg_catalog.pg_namespace nsp - ON nsp.oid = con.connamespace - INNER JOIN information_schema.constraint_column_usage ccu - ON con.conname = ccu.constraint_name - AND nsp.nspname = ccu.constraint_schema - AND rel.relname = ccu.table_name - WHERE nsp.nspname = #{connection.quote(schema.presence || current_schema)} - AND rel.relname = #{connection.quote(table)} - AND ccu.column_name = #{connection.quote(column)} - AND con.contype = 'c' - ORDER BY constraint_name - SQL + private + + def multiple_columns(columns, separator: ', ') + Array.wrap(columns).join(separator) + end + + def cascade_statement(cascade) + cascade ? 'CASCADE' : '' + end - connection.exec_query(check_sql) + def validate_check_constraint_name!(constraint_name) + if constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH + raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters" + end end - def statement_timeout_disabled? - # This is a string of the form "100ms" or "0" when disabled - connection.select_value('SHOW statement_timeout') == "0" + # mappings => {} where keys are column names and values are hashes with the following keys: + # from_type - from which type we're migrating + # to_type - to which type we're migrating + # default_value - custom default value, if not provided will be taken from neutral_values_for_type + def mapping_has_required_columns?(mapping) + %i[from_type to_type].map do |required_key| + mapping.has_key?(required_key) + end.all? end def column_is_nullable?(table, column) @@ -1640,14 +1293,6 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(check_sql) == 'YES' end - def text_limit_name(table, column, name: nil) - name.presence || check_constraint_name(table, column, 'max_length') - end - - def not_null_constraint_name(table, column, name: nil) - name.presence || check_constraint_name(table, column, 'not_null') - end - def missing_schema_object_message(table, type, name) <<~MESSAGE Could not find #{type} "#{name}" on table "#{table}" which was referenced during the migration. @@ -1717,17 +1362,6 @@ into similar problems in the future (e.g. when new tables are created). Must end with `_at`} MESSAGE end - - def validate_not_in_transaction!(method_name, modifier = nil) - return unless transaction_open? - - raise <<~ERROR - #{["`#{method_name}`", modifier].compact.join(' ')} cannot be run inside a transaction. - - You can disable transactions by calling `disable_ddl_transaction!` in the body of - your migration class - ERROR - end end end end diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb index dd426962033..b5b8b58681c 100644 --- a/lib/gitlab/database/migration_helpers/v2.rb +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -205,8 +205,8 @@ module Gitlab raise "Column #{old_column} does not exist on #{table}" end - if column.default - raise "#{calling_operation} does not currently support columns with default values" + if column.default_function + raise "#{calling_operation} does not currently support columns with default functions" end unless column_exists?(table, batch_column_name) @@ -269,17 +269,20 @@ module Gitlab def create_insert_trigger(trigger_name, quoted_table, quoted_old_column, quoted_new_column) function_name = function_name_for_trigger(trigger_name) + column = columns(quoted_table.delete('"').to_sym).find { |column| column.name == quoted_old_column.delete('"') } + quoted_default_value = connection.quote(column.default) + execute(<<~SQL) CREATE OR REPLACE FUNCTION #{function_name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - IF NEW.#{quoted_old_column} IS NULL AND NEW.#{quoted_new_column} IS NOT NULL THEN + IF NEW.#{quoted_old_column} IS NOT DISTINCT FROM #{quoted_default_value} AND NEW.#{quoted_new_column} IS DISTINCT FROM #{quoted_default_value} THEN NEW.#{quoted_old_column} = NEW.#{quoted_new_column}; END IF; - IF NEW.#{quoted_new_column} IS NULL AND NEW.#{quoted_old_column} IS NOT NULL THEN + IF NEW.#{quoted_new_column} IS NOT DISTINCT FROM #{quoted_default_value} AND NEW.#{quoted_old_column} IS DISTINCT FROM #{quoted_default_value} THEN NEW.#{quoted_new_column} = NEW.#{quoted_old_column}; END IF; diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 363fd0598f9..e958ce2aba4 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -196,6 +196,43 @@ module Gitlab :gitlab_main end end + + def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( + Gitlab::Database.gitlab_schemas_for_connection(connection), + job_class_name, table_name, column_name, job_arguments + ) + + configuration = { + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments + } + + return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil? + + return if migration.finished? + + finalize_batched_background_migration(job_class_name: job_class_name, table_name: table_name, column_name: column_name, job_arguments: job_arguments) if finalize + + return if migration.reload.finished? # rubocop:disable Cop/ActiveRecordAssociationReload + + raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ + "but it is '#{migration.status_name}':" \ + "\t#{configuration}" \ + "\n\n" \ + "Finalize it manually by running the following command in a `bash` or `sh` shell:" \ + "\n\n" \ + "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ + "\n\n" \ + "For more information, check the documentation" \ + "\n\n" \ + "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" + end end end end diff --git a/lib/gitlab/database/migrations/constraints_helpers.rb b/lib/gitlab/database/migrations/constraints_helpers.rb new file mode 100644 index 00000000000..7b849e3137a --- /dev/null +++ b/lib/gitlab/database/migrations/constraints_helpers.rb @@ -0,0 +1,337 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module ConstraintsHelpers + include LockRetriesHelpers + include TimeoutHelpers + + # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS + MAX_IDENTIFIER_NAME_LENGTH = 63 + + # Returns the name for a check constraint + # + # type: + # - Any value, as long as it is unique + # - Constraint names are unique per table in Postgres, and, additionally, + # we can have multiple check constraints over a column + # So we use the (table, column, type) triplet as a unique name + # - e.g. we use 'max_length' when adding checks for text limits + # or 'not_null' when adding a NOT NULL constraint + # + def check_constraint_name(table, column, type) + identifier = "#{table}_#{column}_check_#{type}" + # Check concurrent_foreign_key_name() for info on why we use a hash + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + + "check_#{hashed_identifier}" + end + + def check_constraint_exists?(table, constraint_name) + # Constraint names are unique per table in Postgres, not per schema + # Two tables can have constraints with the same name, so we filter by + # the table name in addition to using the constraint_name + + check_sql = <<~SQL + SELECT COUNT(*) + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + WHERE con.contype = 'c' + AND con.conname = #{connection.quote(constraint_name)} + AND nsp.nspname = #{connection.quote(current_schema)} + AND rel.relname = #{connection.quote(table)} + SQL + + connection.select_value(check_sql) > 0 + end + + # Adds a check constraint to a table + # + # This method is the generic helper for adding any check constraint + # More specialized helpers may use it (e.g. add_text_limit or add_not_null) + # + # This method only requires minimal locking: + # - The constraint is added using NOT VALID + # This allows us to add the check constraint without validating it + # - The check will be enforced for new data (inserts) coming in + # - If `validate: true` the constraint is also validated + # Otherwise, validate_check_constraint() can be used at a later stage + # - Check comments on add_concurrent_foreign_key for more info + # + # table - The table the constraint will be added to + # check - The check clause to add + # e.g. 'char_length(name) <= 5' or 'store IS NOT NULL' + # constraint_name - The name of the check constraint (otherwise auto-generated) + # Should be unique per table (not per column) + # validate - Whether to validate the constraint in this call + # + def add_check_constraint(table, check, constraint_name, validate: true) + # Transactions would result in ALTER TABLE locks being held for the + # duration of the transaction, defeating the purpose of this method. + validate_not_in_transaction!(:add_check_constraint) + + validate_check_constraint_name!(constraint_name) + + if check_constraint_exists?(table, constraint_name) + warning_message = <<~MESSAGE + Check constraint was not created because it exists already + (this may be due to an aborted migration or similar) + table: #{table}, check: #{check}, constraint name: #{constraint_name} + MESSAGE + + Gitlab::AppLogger.warn warning_message + else + # Only add the constraint without validating it + # Even though it is fast, ADD CONSTRAINT requires an EXCLUSIVE lock + # Use with_lock_retries to make sure that this operation + # will not timeout on tables accessed by many processes + with_lock_retries do + execute <<~SQL + ALTER TABLE #{table} + ADD CONSTRAINT #{constraint_name} + CHECK ( #{check} ) + NOT VALID; + SQL + end + end + + validate_check_constraint(table, constraint_name) if validate + end + + def validate_check_constraint(table, constraint_name) + validate_check_constraint_name!(constraint_name) + + unless check_constraint_exists?(table, constraint_name) + raise missing_schema_object_message(table, "check constraint", constraint_name) + end + + disable_statement_timeout do + # VALIDATE CONSTRAINT only requires a SHARE UPDATE EXCLUSIVE LOCK + # It only conflicts with other validations and creating indexes + execute("ALTER TABLE #{table} VALIDATE CONSTRAINT #{constraint_name};") + end + end + + def remove_check_constraint(table, constraint_name) + # This is technically not necessary, but aligned with add_check_constraint + # and allows us to continue use with_lock_retries here + validate_not_in_transaction!(:remove_check_constraint) + + validate_check_constraint_name!(constraint_name) + + # DROP CONSTRAINT requires an EXCLUSIVE lock + # Use with_lock_retries to make sure that this will not timeout + with_lock_retries do + execute <<-SQL + ALTER TABLE #{table} + DROP CONSTRAINT IF EXISTS #{constraint_name} + SQL + end + end + + # Copies all check constraints for the old column to the new column. + # + # table - The table containing the columns. + # old - The old column. + # new - The new column. + # schema - The schema the table is defined for + # If it is not provided, then the current_schema is used + def copy_check_constraints(table, old, new, schema: nil) + raise 'copy_check_constraints can not be run inside a transaction' if transaction_open? + + raise "Column #{old} does not exist on #{table}" unless column_exists?(table, old) + + raise "Column #{new} does not exist on #{table}" unless column_exists?(table, new) + + table_with_schema = schema.present? ? "#{schema}.#{table}" : table + + check_constraints_for(table, old, schema: schema).each do |check_c| + validate = !(check_c["constraint_def"].end_with? "NOT VALID") + + # Normalize: + # - Old constraint definitions: + # '(char_length(entity_path) <= 5500)' + # - Definitionss from pg_get_constraintdef(oid): + # 'CHECK ((char_length(entity_path) <= 5500))' + # - Definitions from pg_get_constraintdef(oid, pretty_bool): + # 'CHECK (char_length(entity_path) <= 5500)' + # - Not valid constraints: 'CHECK (...) NOT VALID' + # to a single format that we can use: + # '(char_length(entity_path) <= 5500)' + check_definition = check_c["constraint_def"] + .sub(/^\s*(CHECK)?\s*\({0,2}/, '(') + .sub(/\){0,2}\s*(NOT VALID)?\s*$/, ')') + + constraint_name = if check_definition == "(#{old} IS NOT NULL)" + not_null_constraint_name(table_with_schema, new) + elsif check_definition.start_with? "(char_length(#{old}) <=" + text_limit_name(table_with_schema, new) + else + check_constraint_name(table_with_schema, new, 'copy_check_constraint') + end + + add_check_constraint( + table_with_schema, + check_definition.gsub(old.to_s, new.to_s), + constraint_name, + validate: validate + ) + end + end + + # Migration Helpers for adding limit to text columns + def add_text_limit(table, column, limit, constraint_name: nil, validate: true) + add_check_constraint( + table, + "char_length(#{column}) <= #{limit}", + text_limit_name(table, column, name: constraint_name), + validate: validate + ) + end + + def validate_text_limit(table, column, constraint_name: nil) + validate_check_constraint(table, text_limit_name(table, column, name: constraint_name)) + end + + def remove_text_limit(table, column, constraint_name: nil) + remove_check_constraint(table, text_limit_name(table, column, name: constraint_name)) + end + + def check_text_limit_exists?(table, column, constraint_name: nil) + check_constraint_exists?(table, text_limit_name(table, column, name: constraint_name)) + end + + # Migration Helpers for managing not null constraints + def add_not_null_constraint(table, column, constraint_name: nil, validate: true) + if column_is_nullable?(table, column) + add_check_constraint( + table, + "#{column} IS NOT NULL", + not_null_constraint_name(table, column, name: constraint_name), + validate: validate + ) + else + warning_message = <<~MESSAGE + NOT NULL check constraint was not created: + column #{table}.#{column} is already defined as `NOT NULL` + MESSAGE + + Gitlab::AppLogger.warn warning_message + end + end + + def validate_not_null_constraint(table, column, constraint_name: nil) + validate_check_constraint( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + + def remove_not_null_constraint(table, column, constraint_name: nil) + remove_check_constraint( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + + def check_not_null_constraint_exists?(table, column, constraint_name: nil) + check_constraint_exists?( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + + def rename_constraint(table_name, old_name, new_name) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} + SQL + end + + def drop_constraint(table_name, constraint_name, cascade: false) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} DROP CONSTRAINT #{quote_column_name(constraint_name)} #{cascade_statement(cascade)} + SQL + end + + def validate_check_constraint_name!(constraint_name) + return unless constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH + + raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters" + end + + def text_limit_name(table, column, name: nil) + name.presence || check_constraint_name(table, column, 'max_length') + end + + private + + def validate_not_in_transaction!(method_name, modifier = nil) + return unless transaction_open? + + raise <<~ERROR + #{["`#{method_name}`", modifier].compact.join(' ')} cannot be run inside a transaction. + + You can disable transactions by calling `disable_ddl_transaction!` in the body of + your migration class + ERROR + end + + # Returns an ActiveRecord::Result containing the check constraints + # defined for the given column. + # + # If the schema is not provided, then the current_schema is used + def check_constraints_for(table, column, schema: nil) + check_sql = <<~SQL + SELECT + ccu.table_schema as schema_name, + ccu.table_name as table_name, + ccu.column_name as column_name, + con.conname as constraint_name, + pg_get_constraintdef(con.oid) as constraint_def + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + INNER JOIN information_schema.constraint_column_usage ccu + ON con.conname = ccu.constraint_name + AND nsp.nspname = ccu.constraint_schema + AND rel.relname = ccu.table_name + WHERE nsp.nspname = #{connection.quote(schema.presence || current_schema)} + AND rel.relname = #{connection.quote(table)} + AND ccu.column_name = #{connection.quote(column)} + AND con.contype = 'c' + ORDER BY constraint_name + SQL + + connection.exec_query(check_sql) + end + + def cascade_statement(cascade) + cascade ? 'CASCADE' : '' + end + + def not_null_constraint_name(table, column, name: nil) + name.presence || check_constraint_name(table, column, 'not_null') + end + + def missing_schema_object_message(table, type, name) + <<~MESSAGE + Could not find #{type} "#{name}" on table "#{table}" which was referenced during the migration. + This issue could be caused by the database schema straying from the expected state. + + To resolve this issue, please verify: + 1. all previous migrations have completed + 2. the database objects used in this migration match the Rails definition in schema.rb or structure.sql + + MESSAGE + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/extension_helpers.rb b/lib/gitlab/database/migrations/extension_helpers.rb new file mode 100644 index 00000000000..435e9e0d2dc --- /dev/null +++ b/lib/gitlab/database/migrations/extension_helpers.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module ExtensionHelpers + def create_extension(extension) + execute("CREATE EXTENSION IF NOT EXISTS #{extension}") + rescue ActiveRecord::StatementInvalid => e + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username + + warn(<<~MSG) if e.to_s.include?('permission denied') + GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but + the database user is not allowed to install the extension. + + You can either install the extension manually using a database superuser: + + CREATE EXTENSION IF NOT EXISTS #{extension} + + Or, you can solve this by logging in to the GitLab + database (#{dbname}) using a superuser and running: + + ALTER #{user} WITH SUPERUSER + + This query will grant the user superuser permissions, ensuring any database extensions + can be installed through migrations. + + For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. + MSG + + raise + end + + def drop_extension(extension) + execute("DROP EXTENSION IF EXISTS #{extension}") + rescue ActiveRecord::StatementInvalid => e + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username + + warn(<<~MSG) if e.to_s.include?('permission denied') + This migration attempts to drop the PostgreSQL extension '#{extension}' + installed in database '#{dbname}', but the database user is not allowed + to drop the extension. + + You can either drop the extension manually using a database superuser: + + DROP EXTENSION IF EXISTS #{extension} + + Or, you can solve this by logging in to the GitLab + database (#{dbname}) using a superuser and running: + + ALTER #{user} WITH SUPERUSER + + This query will grant the user superuser permissions, ensuring any database extensions + can be dropped through migrations. + + For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. + MSG + + raise + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/lock_retries_helpers.rb b/lib/gitlab/database/migrations/lock_retries_helpers.rb new file mode 100644 index 00000000000..137ef3ab144 --- /dev/null +++ b/lib/gitlab/database/migrations/lock_retries_helpers.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module LockRetriesHelpers + # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts. + # The timings can be controlled via the +timing_configuration+ parameter. + # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+. + # + # Note this helper uses subtransactions when run inside an already open transaction. + # + # ==== Examples + # # Invoking without parameters + # with_lock_retries do + # drop_table :my_table + # end + # + # # Invoking with custom +timing_configuration+ + # t = [ + # [1.second, 1.second], + # [2.seconds, 2.seconds] + # ] + # + # with_lock_retries(timing_configuration: t) do + # drop_table :my_table # this will be retried twice + # end + # + # # Disabling the retries using an environment variable + # > export DISABLE_LOCK_RETRIES=true + # + # with_lock_retries do + # drop_table :my_table # one invocation, it will not retry at all + # end + # + # ==== Parameters + # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the + # block, sleep time before the next iteration, defaults to + # `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` + # * +logger+ - [Gitlab::JsonLogger] + # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` + def with_lock_retries(*args, **kwargs, &block) + raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) + merged_args = { + connection: connection, + klass: self.class, + logger: Gitlab::BackgroundMigration::Logger, + allow_savepoints: true + }.merge(kwargs) + + Gitlab::Database::WithLockRetries.new(**merged_args) + .run(raise_on_exhaustion: raise_on_exhaustion, &block) + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 85dc6051c7c..27b161419b2 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -7,6 +7,7 @@ module Gitlab BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze METADATA_FILENAME = 'metadata.json' SCHEMA_VERSION = 4 # Version of the output format produced by the runner + POST_MIGRATION_MATCHER = %r{db/post_migrate/}.freeze class << self def up(database:, legacy_mode: false) @@ -116,7 +117,10 @@ module Gitlab verbose_was = ActiveRecord::Migration.verbose ActiveRecord::Migration.verbose = true - sorted_migrations = migrations.sort_by(&:version) + sorted_migrations = migrations.sort_by do |m| + [m.filename.match?(POST_MIGRATION_MATCHER) ? 1 : 0, m.version] + end + sorted_migrations.reverse! if direction == :down instrumentation = Instrumentation.new(result_dir: result_dir) diff --git a/lib/gitlab/database/migrations/timeout_helpers.rb b/lib/gitlab/database/migrations/timeout_helpers.rb new file mode 100644 index 00000000000..423c77452b1 --- /dev/null +++ b/lib/gitlab/database/migrations/timeout_helpers.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module TimeoutHelpers + # Long-running migrations may take more than the timeout allowed by + # the database. Disable the session's statement timeout to ensure + # migrations don't get killed prematurely. + # + # There are two possible ways to disable the statement timeout: + # + # - Per transaction (this is the preferred and default mode) + # - Per connection (requires a cleanup after the execution) + # + # When using a per connection disable statement, code must be inside + # a block so we can automatically execute `RESET statement_timeout` after block finishes + # otherwise the statement will still be disabled until connection is dropped + # or `RESET statement_timeout` is executed + def disable_statement_timeout + if block_given? + if statement_timeout_disabled? + # Don't do anything if the statement_timeout is already disabled + # Allows for nested calls of disable_statement_timeout without + # resetting the timeout too early (before the outer call ends) + yield + else + begin + execute('SET statement_timeout TO 0') + + yield + ensure + execute('RESET statement_timeout') + end + end + else + unless transaction_open? + raise <<~ERROR + Cannot call disable_statement_timeout() without a transaction open or outside of a transaction block. + If you don't want to use a transaction wrap your code in a block call: + + disable_statement_timeout { # code that requires disabled statement here } + + This will make sure statement_timeout is disabled before and reset after the block execution is finished. + ERROR + end + + execute('SET LOCAL statement_timeout TO 0') + end + end + + private + + def statement_timeout_disabled? + # This is a string of the form "100ms" or "0" when disabled + connection.select_value('SHOW statement_timeout') == "0" + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb index 23a8dc0b44f..58447481e60 100644 --- a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb +++ b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb @@ -10,13 +10,17 @@ module Gitlab attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value - def initialize(migration_context:, table_name:, parent_table_name:, partitioning_column:, zero_partition_value:) + def initialize( + migration_context:, table_name:, parent_table_name:, partitioning_column:, + zero_partition_value:, lock_tables: []) + @migration_context = migration_context @connection = migration_context.connection @table_name = table_name @parent_table_name = parent_table_name @partitioning_column = partitioning_column @zero_partition_value = zero_partition_value + @lock_tables = Array.wrap(lock_tables) end def prepare_for_partitioning @@ -35,7 +39,12 @@ module Gitlab create_parent_table attach_foreign_keys_to_parent - migration_context.with_lock_retries(raise_on_exhaustion: true) do + lock_args = { + raise_on_exhaustion: true, + timing_configuration: lock_timing_configuration + } + + migration_context.with_lock_retries(**lock_args) do migration_context.execute(sql_to_convert_table) end end @@ -74,6 +83,7 @@ module Gitlab # but they acquire the same locks so it's much faster to incude them # here. [ + lock_tables_statement, attach_table_to_parent_statement, alter_sequence_statements(old_table: table_name, new_table: parent_table_name), remove_constraint_statement @@ -162,6 +172,16 @@ module Gitlab end end + def lock_tables_statement + return if @lock_tables.empty? + + table_names = @lock_tables.map { |name| quote_table_name(name) }.join(', ') + + <<~SQL + LOCK #{table_names} IN ACCESS EXCLUSIVE MODE + SQL + end + def attach_table_to_parent_statement <<~SQL ALTER TABLE #{quote_table_name(parent_table_name)} @@ -235,6 +255,13 @@ module Gitlab ALTER TABLE #{connection.quote_table_name(table_name)} OWNER TO CURRENT_USER SQL end + + def lock_timing_configuration + iterations = Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION + aggressive_iterations = Array.new(5) { [10.seconds, 1.minute] } + + iterations + aggressive_iterations + end end end end diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb index 5e32ecad4ca..58c0728b614 100644 --- a/lib/gitlab/database/partitioning/detached_partition_dropper.rb +++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb @@ -7,7 +7,7 @@ module Gitlab Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop") Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition| - if partition_attached?(qualify_partition_name(detached_partition.table_name)) + if partition_attached?(detached_partition.fully_qualified_table_name) unmark_partition(detached_partition) else drop_partition(detached_partition) @@ -41,14 +41,14 @@ module Gitlab # Another process may have already dropped the table and deleted this entry next unless try_lock_detached_partition(detached_partition.id) - drop_detached_partition(detached_partition.table_name) + drop_detached_partition(detached_partition) detached_partition.destroy! end end def remove_foreign_keys(detached_partition) - partition_identifier = qualify_partition_name(detached_partition.table_name) + partition_identifier = detached_partition.fully_qualified_table_name # We want to load all of these into memory at once to get a consistent view to loop over, # since we'll be deleting from this list as we go @@ -65,7 +65,7 @@ module Gitlab # It is important to only drop one foreign key per transaction. # Dropping a foreign key takes an ACCESS EXCLUSIVE lock on both tables participating in the foreign key. - partition_identifier = qualify_partition_name(detached_partition.table_name) + partition_identifier = detached_partition.fully_qualified_table_name with_lock_retries do connection.transaction(requires_new: false) do next unless try_lock_detached_partition(detached_partition.id) @@ -83,16 +83,10 @@ module Gitlab end end - def drop_detached_partition(partition_name) - partition_identifier = qualify_partition_name(partition_name) + def drop_detached_partition(detached_partition) + connection.drop_table(detached_partition.fully_qualified_table_name, if_exists: true) - connection.drop_table(partition_identifier, if_exists: true) - - Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: partition_name) - end - - def qualify_partition_name(table_name) - "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{table_name}" + Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: detached_partition.table_name) end def partition_attached?(partition_identifier) diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index 15b542cf089..62f33bb56bc 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -7,6 +7,8 @@ module Gitlab include Gitlab::Database::MigrationHelpers include Gitlab::Database::SchemaHelpers + DuplicatedIndexesError = Class.new(StandardError) + ERROR_SCOPE = 'index' # Concurrently creates a new index on a partitioned table. In concept this works similarly to @@ -38,7 +40,7 @@ module Gitlab partitioned_table.postgres_partitions.order(:name).each do |partition| partition_index_name = generated_index_name(partition.identifier, options[:name]) - partition_options = options.merge(name: partition_index_name) + partition_options = options.merge(name: partition_index_name, allow_partition: true) add_concurrent_index(partition.identifier, column_names, partition_options) end @@ -92,6 +94,42 @@ module Gitlab .map { |_, indexes| indexes.map { |index| index['index_name'] } } end + # Retrieves a hash of index names for a given table and schema, by index + # definition. + # + # Example: + # + # indexes_by_definition_for_table('table_name_goes_here') + # + # Returns: + # + # { + # "CREATE _ btree (created_at)" => "index_on_created_at" + # } + def indexes_by_definition_for_table(table_name, schema_name: connection.current_schema) + duplicate_indexes = find_duplicate_indexes(table_name, schema_name: schema_name) + + unless duplicate_indexes.empty? + raise DuplicatedIndexesError, "#{table_name} has duplicate indexes: #{duplicate_indexes}" + end + + find_indexes(table_name, schema_name: schema_name) + .each_with_object({}) { |row, hash| hash[row['index_id']] = row['index_name'] } + end + + # Renames indexes for a given table and schema, mapping by index + # definition, to a hash of new index names. + # + # Example: + # + # index_names = indexes_by_definition_for_table('source_table_name_goes_here') + # drop_table('source_table_name_goes_here') + # rename_indexes_for_table('destination_table_name_goes_here', index_names) + def rename_indexes_for_table(table_name, new_index_names, schema_name: connection.current_schema) + current_index_names = indexes_by_definition_for_table(table_name, schema_name: schema_name) + rename_indexes(current_index_names, new_index_names, schema_name: schema_name) + end + private def find_indexes(table_name, schema_name: connection.current_schema) @@ -124,6 +162,18 @@ module Gitlab def generated_index_name(partition_name, index_name) object_name("#{partition_name}_#{index_name}", 'index') end + + def rename_indexes(from, to, schema_name: connection.current_schema) + indexes_to_rename = from.select { |index_id, _| to.has_key?(index_id) } + statements = indexes_to_rename.map do |index_id, index_name| + <<~SQL + ALTER INDEX #{connection.quote_table_name("#{schema_name}.#{connection.quote_column_name(index_name)}")} + RENAME TO #{connection.quote_column_name(to[index_id])} + SQL + end + + connection.execute(statements.join(';')) + end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 695a5d7ec77..f9790bf53b9 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -275,7 +275,7 @@ module Gitlab ).revert_preparation_for_partitioning end - def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, lock_tables: []) validate_not_in_transaction!(:convert_table_to_first_list_partition) Gitlab::Database::Partitioning::ConvertTableToFirstListPartition @@ -283,7 +283,8 @@ module Gitlab table_name: table_name, parent_table_name: parent_table_name, partitioning_column: partitioning_column, - zero_partition_value: initial_partitioning_value + zero_partition_value: initial_partitioning_value, + lock_tables: lock_tables ).partition end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index eb080904f73..eda11fd8382 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -19,6 +19,20 @@ module Gitlab scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) } + def self.partition_exists?(table_name) + where("identifier = concat(current_schema(), '.', ?)", table_name).exists? + end + + def self.legacy_partition_exists?(table_name) + result = connection.select_value(<<~SQL) + SELECT true FROM pg_class + WHERE relname = '#{table_name}' + AND relispartition = true; + SQL + + !!result + end + def to_s name end diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 6f64d04270f..1280789b30c 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -86,7 +86,11 @@ module Gitlab analyzers.each do |analyzer| next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) - analyzer.analyze(parsed) + if analyzer.raw? + analyzer.analyze(sql) + else + analyzer.analyze(parsed) + end rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index 9a52a4f6e23..9c2c228f869 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -53,6 +53,10 @@ module Gitlab Thread.current[self.context_key] end + def self.raw? + false + end + def self.enabled? raise NotImplementedError end diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_id_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_id_analyzer.rb new file mode 100644 index 00000000000..47277182d9a --- /dev/null +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_id_analyzer.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + module Ci + # The purpose of this analyzer is to detect queries missing partition_id clause + # when selecting, inserting, updating or deleting data. + class PartitioningIdAnalyzer < Database::QueryAnalyzers::Base + PartitionIdMissingError = Class.new(QueryAnalyzerError) + + ROUTING_TABLES = %w[p_ci_builds_metadata].freeze + + class << self + def enabled? + ::Feature::FlipperFeature.table_exists? && + ::Feature.enabled?(:ci_partitioning_analyze_queries_partition_id_check, type: :ops) + end + + def analyze(parsed) + analyze_partition_id_presence(parsed) + end + + private + + def analyze_partition_id_presence(parsed) + detected = ROUTING_TABLES & (parsed.pg.dml_tables + parsed.pg.select_tables) + return if detected.none? + + if insert_query?(parsed) + return if insert_include_partition_id?(parsed) + else + detected_with_selected_columns = parsed_detected_tables(parsed, detected) + return if partition_id_included?(detected_with_selected_columns) + end + + ::Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + PartitionIdMissingError.new( + "Detected query against a partitioned table without partition id: #{parsed.sql}" + ) + ) + end + + def parsed_detected_tables(parsed, routing_tables) + parsed.pg.filter_columns.each_with_object(Hash.new { |h, k| h[k] = [] }) do |item, hash| + table_name = item[0] || routing_tables[0] + column_name = item[1] + + hash[table_name] << column_name if routing_tables.include?(table_name) + end + end + + def partition_id_included?(result) + return false if result.empty? + + result.all? { |_routing_table, columns| columns.include?('partition_id') } + end + + def insert_query?(parsed) + parsed.sql.start_with?('INSERT') + end + + def insert_include_partition_id?(parsed) + filtered_columns_on_insert(parsed).include?('partition_id') + end + + def filtered_columns_on_insert(parsed) + result = parsed.pg.tree.to_h.dig(:stmts, 0, :stmt, :insert_stmt, :cols).map do |h| + h.dig(:res_target, :name) + end + + result || [] + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb index c2d5dfc1a15..eb55ebc7619 100644 --- a/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb @@ -5,12 +5,10 @@ module Gitlab module QueryAnalyzers module Ci # The purpose of this analyzer is to detect queries not going through a partitioning routing table - class PartitioningAnalyzer < Database::QueryAnalyzers::Base + class PartitioningRoutingAnalyzer < Database::QueryAnalyzers::Base RoutingTableNotUsedError = Class.new(QueryAnalyzerError) - ENABLED_TABLES = %w[ - ci_builds_metadata - ].freeze + ENABLED_TABLES = %w[ci_builds_metadata].freeze class << self def enabled? diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb new file mode 100644 index 00000000000..88fe829c3d2 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/query_recorder.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class QueryRecorder < Base + LOG_FILE = 'rspec/query_recorder.ndjson' + + class << self + def raw? + true + end + + def enabled? + # Only enable QueryRecorder in CI + ENV['CI'].present? + end + + def analyze(sql) + payload = { + sql: sql + } + + log_query(payload) + end + + private + + def log_query(payload) + log_path = Rails.root.join(LOG_FILE) + log_dir = File.dirname(log_path) + + # Create log directory if it does not exist since it is only created + # ahead of time by certain CI jobs + FileUtils.mkdir_p(log_dir) unless Dir.exist?(log_dir) + + log_line = "#{Gitlab::Json.dump(payload)}\n" + + File.write(log_path, log_line, mode: 'a') + end + end + end + end + end +end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index 164520fbab3..8380bf23899 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -14,7 +14,7 @@ module Gitlab end def execute - raise "Cannot truncate legacy tables in single-db setup" unless Gitlab::Database.has_config?(:ci) + raise "Cannot truncate legacy tables in single-db setup" if single_database_setup? raise "database is not supported" unless %w[main ci].include?(database_name) logger&.info "DRY RUN:" if dry_run @@ -91,6 +91,13 @@ module Gitlab end end end + + def single_database_setup? + return true unless Gitlab::Database.has_config?(:ci) + + ci_base_model = Gitlab::Database.database_base_models[:ci] + !!Gitlab::Database.db_config_share_with(ci_base_model.connection_db_config) + end end end end diff --git a/lib/gitlab/database/type/symbolized_jsonb.rb b/lib/gitlab/database/type/symbolized_jsonb.rb new file mode 100644 index 00000000000..5bec738ec9c --- /dev/null +++ b/lib/gitlab/database/type/symbolized_jsonb.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' Jsonb data type to deserialize it into symbolized Hash. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, :sym_jsonb + # end + class SymbolizedJsonb < ::ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Jsonb + def type + :sym_jsonb + end + + def deserialize(value) + data = super + return unless data + + ::Gitlab::Utils.deep_symbolized_access(data) + end + end + end + end +end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index 57d354eb907..be500171bef 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -98,7 +98,7 @@ module Gitlab if environment.save success(result) else - log_error("Could not create environment for the Self monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages }) + log_error("Could not create environment for the Self-monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages }) error(_('Could not create environment')) end end diff --git a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb index 998977b4000..d5bed94d735 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb @@ -23,7 +23,7 @@ module Gitlab def validate_self_monitoring_project_exists(result) unless project_created? || self_monitoring_project_id.present? - return error(_('Self monitoring project does not exist')) + return error(_('Self-monitoring project does not exist')) end success(result) diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 5583c896803..d5c0b187f92 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,10 +44,6 @@ module Gitlab add_blobs_to_batch_loader end - def use_semantic_ipynb_diff? - strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project) } - end - def has_renderable? rendered&.has_renderable? end @@ -372,7 +368,7 @@ module Gitlab end def rendered - return unless use_semantic_ipynb_diff? && ipynb? && modified_file? && !collapsed? && !too_large? + return unless ipynb? && modified_file? && !collapsed? && !too_large? strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 924de132840..ae55dae1201 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -46,7 +46,9 @@ module Gitlab # This is either the new path, otherwise the old path for the diff_file def diff_file_paths - diff_files.map(&:file_path) + diffs.map do |diff| + diff.new_path.presence || diff.old_path + end end # This is both the new and old paths for the diff_file diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index d6f5e45c034..5128b09aef4 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -62,7 +62,7 @@ module Gitlab end def clear - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.del(key) end end @@ -124,7 +124,7 @@ module Gitlab # ...it will write/update a Gitlab::Redis hash (HSET) # def write_to_redis_hash(hash) - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.pipelined do |pipeline| hash.each do |diff_file_id, highlighted_diff_lines_hash| pipeline.hset( @@ -132,7 +132,7 @@ module Gitlab diff_file_id, gzip_compress(highlighted_diff_lines_hash.to_json) ) - rescue Encoding::UndefinedConversionError + rescue Encoding::UndefinedConversionError, EncodingError, JSON::GeneratorError nil end @@ -189,7 +189,7 @@ module Gitlab results = [] cache_key = key # Moving out redis calls for feature flags out of redis.pipelined - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.pipelined do |pipeline| results = pipeline.hmget(cache_key, file_paths) pipeline.expire(key, EXPIRATION) @@ -223,6 +223,10 @@ module Gitlab ::Gitlab::Metrics::WebTransaction.current end + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end + def record_hit_ratio(results) current_transaction&.increment(:gitlab_redis_diff_caching_requests_total) current_transaction&.increment(:gitlab_redis_diff_caching_hits_total) if results.any?(&:present?) diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 3337aeb9262..14cb773251b 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -14,12 +14,14 @@ module Gitlab # # mapping - Write multiple cache values at once def write_multiple(mapping) - Redis::Cache.with do |redis| - redis.multi do |multi| - mapping.each do |raw_key, value| - key = cache_key_for(raw_key) + with_redis do |redis| + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.multi do |multi| + mapping.each do |raw_key, value| + key = cache_key_for(raw_key) - multi.set(key, gzip_compress(value.to_json), ex: EXPIRATION) + multi.set(key, gzip_compress(value.to_json), ex: EXPIRATION) + end end end end @@ -37,7 +39,7 @@ module Gitlab keys = raw_keys.map { |id| cache_key_for(id) } content = - Redis::Cache.with do |redis| + with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do redis.mget(keys) end @@ -62,7 +64,7 @@ module Gitlab keys = raw_keys.map { |id| cache_key_for(id) } - Redis::Cache.with do |redis| + with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do redis.del(keys) end @@ -78,6 +80,10 @@ module Gitlab def cache_key_prefix "#{Redis::Cache::CACHE_NAMESPACE}:#{VERSION}:discussion-highlight" end + + def with_redis(&block) + Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb index f9e6d4076f3..7bb9ac2ffdb 100644 --- a/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb +++ b/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb @@ -12,8 +12,6 @@ module Gitlab SALT = '' def self.transform_secret(plain_secret) - return plain_secret unless Feature.enabled?(:hash_oauth_tokens) - Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) end diff --git a/lib/gitlab/email/common.rb b/lib/gitlab/email/common.rb new file mode 100644 index 00000000000..afee8d9cd3d --- /dev/null +++ b/lib/gitlab/email/common.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Email + # Contains common methods which must be present in all email classes + module Common + UNSUBSCRIBE_SUFFIX = '-unsubscribe' + UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe' + WILDCARD_PLACEHOLDER = '%{key}' + + # This can be overridden for a custom config + def config + raise NotImplementedError + end + + def incoming_email_config + Gitlab.config.incoming_email + end + + def enabled? + !!config&.enabled && config.address.present? + end + + def supports_wildcard? + config_address = incoming_email_config.address + + config_address.present? && config_address.include?(WILDCARD_PLACEHOLDER) + end + + def supports_issue_creation? + enabled? && supports_wildcard? + end + + def reply_address(key) + incoming_email_config.address.sub(WILDCARD_PLACEHOLDER, key) + end + + # example: incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com + def unsubscribe_address(key) + incoming_email_config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") + end + + def key_from_address(address, wildcard_address: nil) + raise NotImplementedError + end + + def key_from_fallback_message_id(mail_id) + message_id_regexp = /\Areply-(.+)@#{Gitlab.config.gitlab.host}\z/ + + mail_id[message_id_regexp, 1] + end + + def scan_fallback_references(references) + # It's looking for each <...> + references.scan(/(?!<)[^<>]+(?=>)/) + end + end + end +end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 434893eab82..e21a88c4e0d 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -73,7 +73,7 @@ module Gitlab end def can_handle_legacy_format? - project_path && !incoming_email_token.include?('+') && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY) + project_path && !incoming_email_token.include?('+') && !mail_key.include?(Gitlab::Email::Common::UNSUBSCRIBE_SUFFIX_LEGACY) end end end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 528857aff14..a4e526d9a24 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -12,8 +12,8 @@ module Gitlab delegate :project, to: :sent_notification, allow_nil: true HANDLER_REGEX_FOR = -> (suffix) { /\A(?<reply_token>\w+)#{Regexp.escape(suffix)}\z/ }.freeze - HANDLER_REGEX = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX).freeze - HANDLER_REGEX_LEGACY = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY).freeze + HANDLER_REGEX = HANDLER_REGEX_FOR.call(Gitlab::Email::Common::UNSUBSCRIBE_SUFFIX).freeze + HANDLER_REGEX_LEGACY = HANDLER_REGEX_FOR.call(Gitlab::Email::Common::UNSUBSCRIBE_SUFFIX_LEGACY).freeze def initialize(mail, mail_key) super(mail, mail_key) diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ba84be6e8ca..1e03f5d17ee 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -10,6 +10,14 @@ module Gitlab RECEIVED_HEADER_REGEX = /for\s+\<(.+)\>/.freeze + # Errors that are purely from users and not anything we can control + USER_ERRORS = [ + Gitlab::Email::AutoGeneratedEmailError, Gitlab::Email::ProjectNotFound, Gitlab::Email::EmptyEmailError, + Gitlab::Email::UserNotFoundError, Gitlab::Email::UserBlockedError, Gitlab::Email::UserNotAuthorizedError, + Gitlab::Email::NoteableNotFoundError, Gitlab::Email::InvalidAttachment, Gitlab::Email::InvalidRecordError, + Gitlab::Email::EmailTooLarge + ].freeze + def initialize(raw) @raw = raw end @@ -24,6 +32,9 @@ module Gitlab handler.execute.tap do Gitlab::Metrics::BackgroundTransaction.current&.add_event(handler.metrics_event, handler.metrics_params) end + rescue *USER_ERRORS => e + # do not send a metric event since these are purely user errors that we can't control + raise e rescue StandardError => e Gitlab::Metrics::BackgroundTransaction.current&.add_event('email_receiver_error', error: e.class.name) raise e diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb index 3c6ed696b9d..b1a9603d3a5 100644 --- a/lib/gitlab/environment.rb +++ b/lib/gitlab/environment.rb @@ -5,9 +5,5 @@ module Gitlab def self.hostname @hostname ||= ENV['HOSTNAME'] || Socket.gethostname end - - def self.qa_user_agent - ENV['GITLAB_QA_USER_AGENT'] - end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 83920182da4..582c3380869 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -131,6 +131,9 @@ module Gitlab end def before_send(event, hint) + # Don't report Sidekiq retry errors to Sentry + return if hint[:exception].is_a?(Gitlab::SidekiqMiddleware::RetryError) + inject_context_for_exception(event, hint[:exception]) custom_fingerprinting(event, hint[:exception]) diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 437d577e70e..bc97c88ce85 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -15,10 +15,12 @@ module Gitlab def touch(*keys, only_if_missing: false) etags = keys.map { generate_etag } - Gitlab::Redis::SharedState.with do |redis| - redis.pipelined do |pipeline| - keys.each_with_index do |key, i| - pipeline.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Gitlab::Redis::SharedState.with do |redis| + redis.pipelined do |pipeline| + keys.each_with_index do |key, i| + pipeline.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) + end end end end diff --git a/lib/gitlab/experimentation/group_types.rb b/lib/gitlab/experimentation/group_types.rb deleted file mode 100644 index 8e8f7284b99..00000000000 --- a/lib/gitlab/experimentation/group_types.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Experimentation - module GroupTypes - GROUP_CONTROL = :control - GROUP_EXPERIMENTAL = :experimental - end - end -end diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb index c06711d16f8..2ba1a363421 100644 --- a/lib/gitlab/external_authorization/cache.rb +++ b/lib/gitlab/external_authorization/cache.rb @@ -11,7 +11,7 @@ module Gitlab end def load - @access, @reason, @refreshed_at = ::Gitlab::Redis::Cache.with do |redis| + @access, @reason, @refreshed_at = with_redis do |redis| redis.hmget(cache_key, :access, :reason, :refreshed_at) end @@ -19,7 +19,7 @@ module Gitlab end def store(new_access, new_reason, new_refreshed_at) - ::Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.pipelined do |pipeline| pipeline.mapped_hmset( cache_key, @@ -58,6 +58,10 @@ module Gitlab def cache_key "external_authorization:user-#{@user.id}:label-#{@label}" end + + def with_redis(&block) + ::Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/feature_categories.rb b/lib/gitlab/feature_categories.rb index d06f3b14fed..17586a94d7e 100644 --- a/lib/gitlab/feature_categories.rb +++ b/lib/gitlab/feature_categories.rb @@ -31,6 +31,14 @@ module Gitlab category end + def get!(feature_category) + return feature_category if valid?(feature_category) + + raise "Unknown feature category: #{feature_category}" if Gitlab.dev_or_test_env? + + FEATURE_CATEGORY_DEFAULT + end + def valid?(category) categories.include?(category.to_s) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9bbe17dcad1..b8f4ff0e9c4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,7 +45,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :storage, :gl_repository, :gl_project_path + attr_reader :storage, :gl_repository, :gl_project_path, :container # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration @@ -56,11 +56,12 @@ module Gitlab # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. - def initialize(storage, relative_path, gl_repository, gl_project_path) + def initialize(storage, relative_path, gl_repository, gl_project_path, container: nil) @storage = storage @relative_path = relative_path @gl_repository = gl_repository @gl_project_path = gl_project_path + @container = container @name = @relative_path.split("/").last end @@ -69,6 +70,11 @@ module Gitlab "<#{self.class.name}: #{self.gl_project_path}>" end + # Support Feature Flag Repository actor + def flipper_id + "Repository:#{@relative_path}" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end @@ -534,9 +540,9 @@ module Gitlab # Returns matching refs for OID # # Limit of 0 means there is no limit. - def refs_by_oid(oid:, limit: 0) + def refs_by_oid(oid:, limit: 0, ref_patterns: nil) wrapped_gitaly_errors do - gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit) + gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit, ref_patterns: ref_patterns) end rescue CommandError, TypeError, NoRepository nil @@ -1054,19 +1060,19 @@ module Gitlab end end - def search_files_by_name(query, ref) + def search_files_by_name(query, ref, limit: 0, offset: 0) safe_query = query.sub(%r{^/*}, "") ref ||= root_ref return [] if empty? || safe_query.blank? - gitaly_repository_client.search_files_by_name(ref, safe_query).map do |file| + gitaly_repository_client.search_files_by_name(ref, safe_query, limit: limit, offset: offset).map do |file| Gitlab::EncodingHelper.encode_utf8(file) end end - def search_files_by_regexp(filter, ref = 'HEAD') - gitaly_repository_client.search_files_by_regexp(ref, filter).map do |file| + def search_files_by_regexp(filter, ref = 'HEAD', limit: 0, offset: 0) + gitaly_repository_client.search_files_by_regexp(ref, filter, limit: limit, offset: offset).map do |file| Gitlab::EncodingHelper.encode_utf8(file) end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 1330b06bf9c..f4d4cebc096 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -13,6 +13,7 @@ module Gitlab # # Returns true for a valid reference name, false otherwise def validate(ref_name) + return false if ref_name.to_s.empty? # #blank? raises an ArgumentError for invalid encodings return false if ref_name.start_with?(*(EXPANDED_PREFIXES + DISALLOWED_PREFIXES)) return false if ref_name == 'HEAD' @@ -24,6 +25,7 @@ module Gitlab end def validate_merge_request_branch(ref_name) + return false if ref_name.to_s.empty? return false if ref_name.start_with?(*DISALLOWED_PREFIXES) expanded_name = if ref_name.start_with?(*EXPANDED_PREFIXES) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 996534f4194..735c7fcf80c 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -204,8 +204,9 @@ module Gitlab metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id metadata['gitaly-session-id'] = session_id metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) + metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil) metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) - metadata.merge!(Feature::Gitaly.server_feature_flags) + metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors)) metadata.merge!(route_to_primary) deadline_info = request_deadline(timeout) @@ -293,7 +294,7 @@ module Gitlab # check if the limit is being exceeded while testing in those environments # In that case we can use a feature flag to indicate that we do want to # enforce request limits. - return true if Feature::Gitaly.enabled?('enforce_requests_limits') + return true if Feature::Gitaly.enabled_for_any?(:gitaly_enforce_requests_limits) !Rails.env.production? end @@ -502,5 +503,24 @@ module Gitlab end private_class_method :max_stacks + + def self.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block) + feature_flag_actors[:repository] = repository + feature_flag_actors[:user] = user + feature_flag_actors[:project] = project + feature_flag_actors[:group] = group + + yield + ensure + feature_flag_actors.clear + end + + def self.feature_flag_actors + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {} + else + Thread.current[:gitaly_feature_flag_actors] ||= {} + end + end end end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 3b08a833aeb..6d87c3329d7 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -4,9 +4,12 @@ module Gitlab module GitalyClient class BlobService include Gitlab::EncodingHelper + include WithFeatureFlagActors def initialize(repository) @gitaly_repo = repository.gitaly_repository + + self.repository_actor = repository end def get_blob(oid:, limit:) @@ -15,7 +18,7 @@ module Gitlab oid: oid, limit: limit ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@gitaly_repo.storage_name, :blob_service, :get_blob, request, timeout: GitalyClient.fast_timeout) consume_blob_response(response) end @@ -35,7 +38,7 @@ module Gitlab GitalyClient.medium_timeout end - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_blobs, request, timeout: timeout) + response = gitaly_client_call(@gitaly_repo.storage_name, :blob_service, :list_blobs, request, timeout: timeout) GitalyClient::BlobsStitcher.new(GitalyClient::ListBlobsAdapter.new(response)) end @@ -47,7 +50,7 @@ module Gitlab blob_ids: blob_ids ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end @@ -64,7 +67,7 @@ module Gitlab limit: limit ) - response = GitalyClient.call( + response = gitaly_client_call( @gitaly_repo.storage_name, :blob_service, :get_blobs, @@ -87,7 +90,7 @@ module Gitlab limit: limit ) - response = GitalyClient.call( + response = gitaly_client_call( @gitaly_repo.storage_name, :blob_service, :get_blobs, @@ -107,7 +110,7 @@ module Gitlab GitalyClient.medium_timeout end - response = GitalyClient.call( + response = gitaly_client_call( @gitaly_repo.storage_name, :blob_service, rpc, @@ -123,7 +126,7 @@ module Gitlab revisions: [encode_binary("--all")] ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_lfs_pointers, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@gitaly_repo.storage_name, :blob_service, :list_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb index 649aaa46362..3c2c41a244e 100644 --- a/lib/gitlab/gitaly_client/cleanup_service.rb +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -3,6 +3,8 @@ module Gitlab module GitalyClient class CleanupService + include WithFeatureFlagActors + attr_reader :repository, :gitaly_repo, :storage # 'repository' is a Gitlab::Git::Repository @@ -10,10 +12,12 @@ module Gitlab @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def apply_bfg_object_map_stream(io, &blk) - response = GitalyClient.call( + response = gitaly_client_call( storage, :cleanup_service, :apply_bfg_object_map_stream, diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 312d1dddff1..6bcf4802fbe 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -4,12 +4,15 @@ module Gitlab module GitalyClient class CommitService include Gitlab::EncodingHelper + include WithFeatureFlagActors TREE_ENTRIES_DEFAULT_LIMIT = 100_000 def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository + + self.repository_actor = repository end def ls_files(revision) @@ -18,7 +21,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map { |d| EncodingHelper.encode!(d.dup) } end @@ -31,7 +34,7 @@ module Gitlab child_id: child_id ) - GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value + gitaly_client_call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value end def diff(from, to, options = {}) @@ -74,7 +77,7 @@ module Gitlab def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout) response.flat_map { |msg| msg.deltas } end @@ -93,7 +96,7 @@ module Gitlab limit: limit.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) entry = nil data = [] @@ -127,7 +130,7 @@ module Gitlab ) request.sort = Gitaly::GetTreeEntriesRequest::SortBy::TREES_FIRST if pagination_params - response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) cursor = nil @@ -163,7 +166,7 @@ module Gitlab request.path = encode_binary(options[:path]) if options[:path].present? request.max_count = options[:max_count] if options[:max_count].present? - GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count + gitaly_client_call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end def diverging_commit_count(from, to, max_count:) @@ -173,7 +176,7 @@ module Gitlab to: encode_binary(to), max_count: max_count ) - response = GitalyClient.call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) [response.left_count, response.right_count] end @@ -187,7 +190,7 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) response.each_with_object({}) do |gitaly_response, hsh| gitaly_response.commits.each do |commit_for_tree| @@ -204,7 +207,7 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit + gitaly_commit = gitaly_client_call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit return unless gitaly_commit Gitlab::Git::Commit.new(@repository, gitaly_commit) @@ -217,7 +220,7 @@ module Gitlab right_commit_id: right_commit_sha ) - response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) response.flat_map { |rsp| rsp.stats.to_a } end @@ -227,7 +230,7 @@ module Gitlab commits: commits ) - response = GitalyClient.call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map do |path| Gitlab::Git::ChangedPath.new( @@ -247,7 +250,7 @@ module Gitlab ) request.order = opts[:order].upcase if opts[:order].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -268,7 +271,7 @@ module Gitlab request.before = GitalyClient.timestamp(params[:before]) if params[:before] request.after = GitalyClient.timestamp(params[:after]) if params[:after] - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -290,7 +293,7 @@ module Gitlab repository: quarantined_repo ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) quarantined_commits = consume_commits_response(response) quarantined_commit_ids = quarantined_commits.map(&:id) @@ -328,7 +331,7 @@ module Gitlab request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids) - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) rescue GRPC::NotFound # If no repository is found, happens mainly during testing [] @@ -345,13 +348,13 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end def languages(ref = nil) request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') - response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout) response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } end @@ -364,7 +367,7 @@ module Gitlab range: (encode_binary(range) if range) ) - response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce([]) { |memo, msg| memo << msg.data }.join end @@ -400,7 +403,7 @@ module Gitlab repository: @gitaly_repo, revision: encode_binary(revision) ) - GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) + gitaly_client_call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end def find_commits(options) @@ -424,7 +427,7 @@ module Gitlab request.paths = encode_repeated(Array(options[:path])) if options[:path].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -443,7 +446,7 @@ module Gitlab end end - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :commit_service, :check_objects_exist, enum, timeout: GitalyClient.medium_timeout ) @@ -470,7 +473,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout) response.flat_map do |msg| msg.shas.map { |sha| EncodingHelper.encode!(sha) } end @@ -478,7 +481,7 @@ module Gitlab def get_commit_signatures(commit_ids) request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } current_commit_id = nil @@ -497,7 +500,7 @@ module Gitlab def get_commit_messages(commit_ids) request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout) messages = Hash.new { |h, k| h[k] = +''.b } current_commit_id = nil @@ -515,7 +518,7 @@ module Gitlab request = Gitaly::ListCommitsByRefNameRequest .new(repository: @gitaly_repo, ref_names: refs.map { |ref| encode_binary(ref) }) - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_ref_name, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits_by_ref_name, request, timeout: GitalyClient.medium_timeout) commit_refs = response.flat_map do |message| message.commit_refs.map do |commit_ref| @@ -540,7 +543,7 @@ module Gitlab request_params.merge!(Gitlab::Git::DiffCollection.limits(options)) request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) GitalyClient::DiffStitcher.new(response) end @@ -577,7 +580,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) response.commit end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 982454b117e..38f648ccc31 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -4,6 +4,7 @@ module Gitlab module GitalyClient class ConflictsService include Gitlab::EncodingHelper + include WithFeatureFlagActors MAX_MSG_SIZE = 128.kilobytes.freeze @@ -12,6 +13,8 @@ module Gitlab @repository = repository @our_commit_oid = our_commit_oid @their_commit_oid = their_commit_oid + + self.repository_actor = repository end def list_conflict_files(allow_tree_conflicts: false) @@ -21,7 +24,7 @@ module Gitlab their_commit_oid: @their_commit_oid, allow_tree_conflicts: allow_tree_conflicts ) - response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout) GitalyClient::ConflictFilesStitcher.new(response, @gitaly_repo) end @@ -50,7 +53,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.long_timeout) if response.resolution_error.present? raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index 786ef0ebebe..e07bf3fbccc 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -3,6 +3,8 @@ module Gitlab module GitalyClient class ObjectPoolService + include WithFeatureFlagActors + attr_reader :object_pool, :storage def initialize(object_pool) @@ -15,8 +17,10 @@ module Gitlab object_pool: object_pool, origin: repository.gitaly_repository) - GitalyClient.call(storage, :object_pool_service, :create_object_pool, - request, timeout: GitalyClient.medium_timeout) + GitalyClient.with_feature_flag_actors(**gitaly_feature_flag_actors(repository)) do + GitalyClient.call(storage, :object_pool_service, :create_object_pool, + request, timeout: GitalyClient.medium_timeout) + end end def delete @@ -32,8 +36,10 @@ module Gitlab repository: repository.gitaly_repository ) - GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, - request, timeout: GitalyClient.fast_timeout) + GitalyClient.with_feature_flag_actors(**gitaly_feature_flag_actors(repository)) do + GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, + request, timeout: GitalyClient.fast_timeout) + end end def fetch(repository) @@ -42,8 +48,10 @@ module Gitlab origin: repository.gitaly_repository ) - GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, - request, timeout: GitalyClient.long_timeout) + GitalyClient.with_feature_flag_actors(**gitaly_feature_flag_actors(repository)) do + GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, + request, timeout: GitalyClient.long_timeout) + end end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 7835fb32f59..2312def5efc 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -4,12 +4,15 @@ module Gitlab module GitalyClient class OperationService include Gitlab::EncodingHelper + include WithFeatureFlagActors MAX_MSG_SIZE = 128.kilobytes.freeze def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository + + self.repository_actor = repository end def rm_tag(tag_name, user) @@ -19,7 +22,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -36,7 +39,7 @@ module Gitlab timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error elsif response.exists @@ -73,7 +76,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, start_point: encode_binary(start_point) ) - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_create_branch, request, timeout: GitalyClient.long_timeout) if response.pre_receive_error.present? @@ -110,7 +113,7 @@ module Gitlab oldrev: encode_binary(oldrev) ) - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_update_branch, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence @@ -125,7 +128,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_delete_branch, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence @@ -156,7 +159,7 @@ module Gitlab timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_merge_to_ref, request, timeout: GitalyClient.long_timeout) response.commit_id @@ -164,7 +167,7 @@ module Gitlab def user_merge_branch(user, source_sha, target_branch, message) request_enum = QueueEnumerator.new - response_enum = GitalyClient.call( + response_enum = gitaly_client_call( @repository.storage, :operation_service, :user_merge_branch, @@ -225,7 +228,7 @@ module Gitlab branch: encode_binary(target_branch) ) - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :user_ff_branch, @@ -268,7 +271,7 @@ module Gitlab request_enum = QueueEnumerator.new rebase_sha = nil - response_enum = GitalyClient.call( + response_enum = gitaly_client_call( @repository.storage, :operation_service, :user_rebase_confirmable, @@ -334,7 +337,7 @@ module Gitlab timestamp: Google::Protobuf::Timestamp.new(seconds: time.to_i) ) - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :user_squash, @@ -376,7 +379,7 @@ module Gitlab timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :user_update_submodule, @@ -422,7 +425,7 @@ module Gitlab end end - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :user_commit_files, req_enum, timeout: GitalyClient.long_timeout, remote_storage: start_repository&.storage) @@ -435,9 +438,25 @@ module Gitlab end Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :custom_hook + raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), + fallback_message: e.details) + when :index_update + raise Gitlab::Git::Index::IndexError, index_error_message(detailed_error.index_update) + else + raise e + end end - # rubocop:enable Metrics/ParameterLists + # rubocop:enable Metrics/ParameterLists def user_commit_patches(user, branch_name, patches) header = Gitaly::UserApplyPatchRequest::Header.new( repository: @gitaly_repo, @@ -457,7 +476,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_apply_patch, chunks, timeout: GitalyClient.long_timeout) Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) @@ -493,7 +512,7 @@ module Gitlab dry_run: dry_run ) - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :"user_#{rpc}", @@ -575,6 +594,27 @@ module Gitlab custom_hook_output = custom_hook_error.stderr.presence || custom_hook_error.stdout EncodingHelper.encode!(custom_hook_output) end + + def index_error_message(index_error) + encoded_path = EncodingHelper.encode!(index_error.path) + + case index_error.error_type + when :ERROR_TYPE_EMPTY_PATH + "Received empty path" + when :ERROR_TYPE_INVALID_PATH + "Invalid path: #{encoded_path}" + when :ERROR_TYPE_DIRECTORY_EXISTS + "Directory already exists: #{encoded_path}" + when :ERROR_TYPE_DIRECTORY_TRAVERSAL + "Directory traversal in path escapes repository: #{encoded_path}" + when :ERROR_TYPE_FILE_EXISTS + "File already exists: #{encoded_path}" + when :ERROR_TYPE_FILE_NOT_FOUND + "File not found: #{encoded_path}" + else + "Unknown error performing git operation" + end + end end end end diff --git a/lib/gitlab/gitaly_client/praefect_info_service.rb b/lib/gitlab/gitaly_client/praefect_info_service.rb index 127f8cfbdf6..b565898acf8 100644 --- a/lib/gitlab/gitaly_client/praefect_info_service.rb +++ b/lib/gitlab/gitaly_client/praefect_info_service.rb @@ -3,16 +3,20 @@ module Gitlab module GitalyClient class PraefectInfoService + include WithFeatureFlagActors + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def replicas request = Gitaly::RepositoryReplicasRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :praefect_info_service, :repository_replicas, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :praefect_info_service, :repository_replicas, request, timeout: GitalyClient.fast_timeout) end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index d2b702f3a6d..de76ade76cb 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -4,6 +4,7 @@ module Gitlab module GitalyClient class RefService include Gitlab::EncodingHelper + include WithFeatureFlagActors TAGS_SORT_KEY = { 'name' => Gitaly::FindAllTagsRequest::SortBy::Key::REFNAME, @@ -21,17 +22,19 @@ module Gitlab @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def branches request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) consume_find_all_branches_response(response) end def remote_branches(remote_name) request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name) - response = GitalyClient.call(@storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout) consume_find_all_remote_branches_response(remote_name, response) end @@ -41,25 +44,25 @@ module Gitlab merged_only: true, merged_branches: branch_names.map { |s| encode_binary(s) } ) - response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) consume_find_all_branches_response(response) end def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout) Gitlab::Git.branch_name(response.name) end def branch_names request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout) consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } end def tag_names request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout) consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } end @@ -74,7 +77,7 @@ module Gitlab def local_branches(sort_by: nil, pagination_params: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo, pagination_params: pagination_params) request.sort_by = sort_local_branches_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout) consume_find_local_branches_response(response) end @@ -82,13 +85,13 @@ module Gitlab request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo, pagination_params: pagination_params) request.sort_by = sort_tags_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout) consume_tags_response(response) end def ref_exists?(ref_name) request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name)) - response = GitalyClient.call(@storage, :ref_service, :ref_exists, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :ref_exists, request, timeout: GitalyClient.fast_timeout) response.value rescue GRPC::InvalidArgument => e raise ArgumentError, e.message @@ -100,7 +103,7 @@ module Gitlab name: encode_binary(branch_name) ) - response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :ref_service, :find_branch, request, timeout: GitalyClient.medium_timeout) branch = response.branch return unless branch @@ -116,7 +119,7 @@ module Gitlab tag_name: encode_binary(tag_name) ) - response = GitalyClient.call(@repository.storage, :ref_service, :find_tag, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :ref_service, :find_tag, request, timeout: GitalyClient.medium_timeout) tag = response.tag return unless tag @@ -140,7 +143,7 @@ module Gitlab except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? rescue GRPC::BadStatus => e @@ -164,7 +167,7 @@ module Gitlab limit: limit ) - response = GitalyClient.call(@storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout) consume_ref_contains_sha_response(response, :tag_names) end @@ -176,7 +179,7 @@ module Gitlab limit: limit ) - response = GitalyClient.call(@storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout) consume_ref_contains_sha_response(response, :branch_names) end @@ -185,7 +188,7 @@ module Gitlab messages = Hash.new { |h, k| h[k] = +''.b } current_tag_id = nil - response = GitalyClient.call(@storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout) response.each do |rpc_message| current_tag_id = rpc_message.tag_id if rpc_message.tag_id.present? @@ -197,7 +200,7 @@ module Gitlab def get_tag_signatures(tag_ids) request = Gitaly::GetTagSignaturesRequest.new(repository: @gitaly_repo, tag_revisions: tag_ids) - response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_signatures, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :ref_service, :get_tag_signatures, request, timeout: GitalyClient.fast_timeout) signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } current_tag_id = nil @@ -222,20 +225,20 @@ module Gitlab patterns: patterns ) - response = GitalyClient.call(@storage, :ref_service, :list_refs, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :list_refs, request, timeout: GitalyClient.fast_timeout) consume_list_refs_response(response) end def pack_refs request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout) end - def find_refs_by_oid(oid:, limit:) - request = Gitaly::FindRefsByOIDRequest.new(repository: @gitaly_repo, sort_field: :refname, oid: oid, limit: limit) + def find_refs_by_oid(oid:, limit:, ref_patterns: nil) + request = Gitaly::FindRefsByOIDRequest.new(repository: @gitaly_repo, sort_field: :refname, oid: oid, limit: limit, ref_patterns: ref_patterns) - response = GitalyClient.call(@storage, :ref_service, :find_refs_by_oid, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_refs_by_oid, request, timeout: GitalyClient.medium_timeout) response&.refs&.to_a end diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 535b987f91c..9647cfad76e 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -4,6 +4,7 @@ module Gitlab module GitalyClient class RemoteService include Gitlab::EncodingHelper + include WithFeatureFlagActors MAX_MSG_SIZE = 128.kilobytes.freeze @@ -24,6 +25,8 @@ module Gitlab @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def find_remote_root_ref(remote_url, authorization) @@ -31,7 +34,7 @@ module Gitlab remote_url: remote_url, http_authorization_header: authorization) - response = GitalyClient.call(@storage, :remote_service, + response = gitaly_client_call(@storage, :remote_service, :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout) encode_utf8(response.ref) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index f11437552e1..e6565bd33c2 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -4,6 +4,7 @@ module Gitlab module GitalyClient class RepositoryService include Gitlab::EncodingHelper + include WithFeatureFlagActors MAX_MSG_SIZE = 128.kilobytes @@ -11,57 +12,59 @@ module Gitlab @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def exists? request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :repository_exists, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :repository_exists, request, timeout: GitalyClient.fast_timeout) response.exists end def optimize_repository request = Gitaly::OptimizeRepositoryRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :optimize_repository, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :optimize_repository, request, timeout: GitalyClient.long_timeout) end def prune_unreachable_objects request = Gitaly::PruneUnreachableObjectsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :prune_unreachable_objects, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :prune_unreachable_objects, request, timeout: GitalyClient.long_timeout) end def garbage_collect(create_bitmap, prune:) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune) - GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) end def repack_full(create_bitmap) request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) - GitalyClient.call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout) end def repack_incremental request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout) end def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.long_timeout) response.size end def get_object_directory_size request = Gitaly::GetObjectDirectorySizeRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :get_object_directory_size, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :repository_service, :get_object_directory_size, request, timeout: GitalyClient.medium_timeout) response.size end def apply_gitattributes(revision) request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: encode_binary(revision)) - GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :apply_gitattributes, request, timeout: GitalyClient.fast_timeout) rescue GRPC::InvalidArgument => ex raise Gitlab::Git::Repository::InvalidRef, ex end @@ -69,7 +72,7 @@ module Gitlab def info_attributes request = Gitaly::GetInfoAttributesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :get_info_attributes, request, timeout: GitalyClient.fast_timeout) response.each_with_object([]) do |message, attributes| attributes << message.attributes end.join @@ -103,18 +106,18 @@ module Gitlab end end - GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout) end # rubocop: enable Metrics/ParameterLists def create_repository(default_branch = nil) request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: default_branch) - GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end def has_local_branches? request = Gitaly::HasLocalBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :has_local_branches, request, timeout: GitalyClient.fast_timeout) response.value end @@ -125,7 +128,7 @@ module Gitlab revisions: revisions.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :find_merge_base, request, timeout: GitalyClient.fast_timeout) response.base.presence end @@ -135,7 +138,7 @@ module Gitlab source_repository: source_repository.gitaly_repository ) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :create_fork, @@ -153,7 +156,7 @@ module Gitlab mirror: mirror ) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :create_repository_from_url, @@ -170,7 +173,7 @@ module Gitlab target_ref: local_ref.b ) - response = GitalyClient.call( + response = gitaly_client_call( @storage, :repository_service, :fetch_source_branch, @@ -184,7 +187,7 @@ module Gitlab def fsck request = Gitaly::FsckRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout) if response.error.empty? ["", 0] @@ -236,7 +239,7 @@ module Gitlab http_auth: http_auth ) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :create_repository_from_snapshot, @@ -253,11 +256,11 @@ module Gitlab ) request.old_revision = old_ref.b unless old_ref.nil? - GitalyClient.call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout) end def set_full_path(path) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :set_full_path, @@ -272,7 +275,7 @@ module Gitlab end def full_path - response = GitalyClient.call( + response = gitaly_client_call( @storage, :repository_service, :full_path, @@ -286,12 +289,12 @@ module Gitlab def find_license request = Gitaly::FindLicenseRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.medium_timeout) + gitaly_client_call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.medium_timeout) end def calculate_checksum request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout) response.checksum.presence rescue GRPC::DataLoss => e raise Gitlab::Git::Repository::InvalidRepository, e @@ -300,23 +303,23 @@ module Gitlab def raw_changes_between(from, to) request = Gitaly::GetRawChangesRequest.new(repository: @gitaly_repo, from_revision: from, to_revision: to) - GitalyClient.call(@storage, :repository_service, :get_raw_changes, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :get_raw_changes, request, timeout: GitalyClient.fast_timeout) end - def search_files_by_name(ref, query) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query) - GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) + def search_files_by_name(ref, query, limit: 0, offset: 0) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query, limit: limit, offset: offset) + gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def search_files_by_content(ref, query, options = {}) request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) - response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) + response = gitaly_client_call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) search_results_from_response(response, options) end - def search_files_by_regexp(ref, filter) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter) - GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) + def search_files_by_regexp(ref, filter, limit: 0, offset: 0) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter, limit: limit, offset: offset) + gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def disconnect_alternates @@ -324,19 +327,19 @@ module Gitlab repository: @gitaly_repo ) - GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout) end def rename(relative_path) request = Gitaly::RenameRepositoryRequest.new(repository: @gitaly_repo, relative_path: relative_path) - GitalyClient.call(@storage, :repository_service, :rename_repository, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :rename_repository, request, timeout: GitalyClient.fast_timeout) end def remove request = Gitaly::RemoveRepositoryRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :remove_repository, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :remove_repository, request, timeout: GitalyClient.long_timeout) end def replicate(source_repository) @@ -345,7 +348,7 @@ module Gitlab source: source_repository.gitaly_repository ) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :replicate_repository, @@ -371,11 +374,11 @@ module Gitlab current_match << message.match_data - if message.end_of_match - matches << current_match - current_match = +"" - matches_count += 1 - end + next unless message.end_of_match + + matches << current_match + current_match = +"" + matches_count += 1 end matches @@ -383,7 +386,7 @@ module Gitlab def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) request = request_class.new(repository: @gitaly_repo) - response = GitalyClient.call( + response = gitaly_client_call( @storage, :repository_service, rpc_name, @@ -416,7 +419,7 @@ module Gitlab end end - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, rpc_name, diff --git a/lib/gitlab/gitaly_client/with_feature_flag_actors.rb b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb new file mode 100644 index 00000000000..92fc524b724 --- /dev/null +++ b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + # This module is responsible for collecting feature flag actors in Gitaly Client. Unlike normal feature flags used + # in Gitlab development, feature flags passed to Gitaly are pre-evaluated at Rails side before being passed to + # Gitaly. As a result, we need to collect all possible actors for the evaluation before issue any RPC. At this + # layer, the only parameter we have is raw repository. We need to infer other actors from the repository. Adding + # extra SQL queries before any RPC are not good for the performance. We applied some quirky optimizations here to + # avoid issuing SQL queries. However, in some less common code paths, a couple of queries are expected. + module WithFeatureFlagActors + include Gitlab::Utils::StrongMemoize + + attr_accessor :repository_actor + + # gitaly_client_call performs Gitaly calls including collected feature flag actors. The actors are retrieved + # from repository actor and memoized. The service must set `self.repository_actor = a_repository` beforehand. + def gitaly_client_call(*args, **kargs) + return GitalyClient.call(*args, **kargs) unless actors_aware_gitaly_calls? + + unless repository_actor + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + Feature::InvalidFeatureFlagError.new("gitaly_client_call called without setting repository_actor") + ) + end + + GitalyClient.with_feature_flag_actors( + repository: repository_actor, + user: user_actor, + project: project_actor, + group: group_actor + ) do + GitalyClient.call(*args, **kargs) + end + end + + # gitaly_feature_flag_actors returns a hash of actors implied from input repository. If actors_aware_gitaly_calls + # flag is not on, this method returns an empty hash. + def gitaly_feature_flag_actors(repository) + return {} unless actors_aware_gitaly_calls? + + container = find_repository_container(repository) + { + repository: repository, + user: Feature::Gitaly.user_actor, + project: Feature::Gitaly.project_actor(container), + group: Feature::Gitaly.group_actor(container) + } + end + + # Use actor here means the user who originally perform the action. It is collected from ApplicationContext. As + # this information is widely propagated in all entry points, User actor should be available everywhere, even in + # background jobs. + def user_actor + strong_memoize(:user_actor) do + Feature::Gitaly.user_actor + end + end + + # TODO: replace this project actor by Repo actor + def project_actor + strong_memoize(:project_actor) do + Feature::Gitaly.project_actor(repository_container) + end + end + + def group_actor + strong_memoize(:group_actor) do + Feature::Gitaly.group_actor(repository_container) + end + end + + private + + def repository_container + strong_memoize(:repository_container) do + find_repository_container(repository_actor) + end + end + + def find_repository_container(repository) + return if repository&.gl_repository.blank? + + if repository.container.nil? + begin + identifier = Gitlab::GlRepository::Identifier.parse(repository.gl_repository) + identifier.container + rescue Gitlab::GlRepository::Identifier::InvalidIdentifier + nil + end + else + repository.container + end + end + + def actors_aware_gitaly_calls? + Feature.enabled?(:actors_aware_gitaly_calls) + end + end + end +end + +Gitlab::GitalyClient::WithFeatureFlagActors.prepend_mod_with('Gitlab::GitalyClient::WithFeatureFlagActors') diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 0f89a7b6575..d6060141bce 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -76,6 +76,10 @@ module Gitlab each_object(:pull_request_reviews, repo_name, iid) end + def pull_request_review_requests(repo_name, iid) + with_rate_limit { octokit.pull_request_review_requests(repo_name, iid).to_h } + end + def repos(options = {}) octokit.repos(nil, options).map(&:to_h) end diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb index b75d41f40de..bcf9cd94ad9 100644 --- a/lib/gitlab/github_import/importer/events/changed_assignee.rb +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -39,12 +39,10 @@ module Gitlab def parse_body(issue_event, assignee_id) assignee = User.find(assignee_id).to_reference - Gitlab::I18n.with_default_locale do - if issue_event.event == "unassigned" - "unassigned #{assignee}" - else - "assigned to #{assignee}" - end + if issue_event.event == 'unassigned' + "#{SystemNotes::IssuablesService.issuable_events[:unassigned]} #{assignee}" + else + "#{SystemNotes::IssuablesService.issuable_events[:assigned]} #{assignee}" end end end diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb index 83130d18db9..553ef0886e8 100644 --- a/lib/gitlab/github_import/importer/events/changed_label.rb +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -13,6 +13,7 @@ module Gitlab def create_event(issue_event) attrs = { + importing: true, user_id: author_id(issue_event), label_id: label_finder.id_for(issue_event.label_title), action: action(issue_event.event), diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb index 21075e21e1d..801a0840c52 100644 --- a/lib/gitlab/github_import/importer/protected_branch_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -37,18 +37,36 @@ module Gitlab name: protected_branch.id, push_access_levels_attributes: [{ access_level: push_access_level }], merge_access_levels_attributes: [{ access_level: merge_access_level }], - allow_force_push: allow_force_push? + allow_force_push: allow_force_push?, + code_owner_approval_required: code_owner_approval_required? } end def allow_force_push? - if ProtectedBranch.protected?(project, protected_branch.id) - ProtectedBranch.allow_force_push?(project, protected_branch.id) && protected_branch.allow_force_pushes + return false unless protected_branch.allow_force_pushes + + if protected_on_gitlab? + ProtectedBranch.allow_force_push?(project, protected_branch.id) + elsif default_branch? + !default_branch_protection.any? else - protected_branch.allow_force_pushes + true end end + def code_owner_approval_required? + return false unless project.licensed_feature_available?(:code_owner_approval_required) + + return protected_branch.require_code_owner_reviews unless protected_on_gitlab? + + # Gets the strictest require_code_owner rule between GitHub and GitLab + protected_branch.require_code_owner_reviews || + ProtectedBranch.branch_requires_code_owner_approval?( + project, + protected_branch.id + ) + end + def default_branch? protected_branch.id == project.default_branch end diff --git a/lib/gitlab/github_import/importer/protected_branches_importer.rb b/lib/gitlab/github_import/importer/protected_branches_importer.rb index 4372477f55d..ff425528aec 100644 --- a/lib/gitlab/github_import/importer/protected_branches_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branches_importer.rb @@ -13,13 +13,15 @@ module Gitlab protected_branches = client.branches(repo).select { |branch| branch.dig(:protection, :enabled) } protected_branches.each do |protected_branch| + next if already_imported?(protected_branch) + object = client.branch_protection(repo, protected_branch[:name]) - next if object.nil? || already_imported?(object) + next if object.nil? yield object Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - mark_as_imported(object) + mark_as_imported(protected_branch) end end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index dd5b7c93ced..b11af90aa6f 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -18,6 +18,7 @@ module Gitlab if gitlab_user_id add_review_note!(gitlab_user_id) add_approval!(gitlab_user_id) + add_reviewer!(gitlab_user_id) else add_complementary_review_note!(project.creator_id) end @@ -95,6 +96,24 @@ module Gitlab end end + def add_reviewer!(user_id) + return if review_re_requested?(user_id) + + ::MergeRequestReviewer.create!( + merge_request_id: merge_request.id, + user_id: user_id, + state: ::MergeRequestReviewer.states['reviewed'], + created_at: submitted_at + ) + end + + # rubocop:disable CodeReuse/ActiveRecord + def review_re_requested?(user_id) + # records that were imported on previous stage with "unreviewed" status + MergeRequestReviewer.where(merge_request_id: merge_request.id, user_id: user_id).exists? + end + # rubocop:enable CodeReuse/ActiveRecord + def add_approval_system_note!(user_id) attributes = note_attributes( user_id, diff --git a/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb new file mode 100644 index 00000000000..bb51d856d9b --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module PullRequests + class ReviewRequestImporter + def initialize(review_request, project, client) + @review_request = review_request + @user_finder = UserFinder.new(project, client) + @issue_finder = IssuableFinder.new(project, client) + end + + def execute + MergeRequestReviewer.bulk_insert!(build_reviewers) + end + + private + + attr_reader :review_request, :user_finder + + def build_reviewers + reviewer_ids = review_request.users.map { |user| user_finder.user_id_for(user) }.compact + + reviewer_ids.map do |reviewer_id| + MergeRequestReviewer.new( + merge_request_id: review_request.merge_request_id, + user_id: reviewer_id, + state: MergeRequestReviewer.states['unreviewed'], + created_at: Time.zone.now + ) + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb new file mode 100644 index 00000000000..c5d8da3be1c --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module PullRequests + class ReviewRequestsImporter + include ParallelScheduling + + BATCH_SIZE = 100 + + private + + def each_object_to_import(&block) + merge_request_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| + batch.each do |merge_request| + repo = project.import_source + + review_requests = client.pull_request_review_requests(repo, merge_request.iid) + review_requests[:merge_request_id] = merge_request.id + yield review_requests + + mark_merge_request_imported(merge_request) + end + end + end + + def importer_class + ReviewRequestImporter + end + + def representation_class + Gitlab::GithubImport::Representation::PullRequests::ReviewRequests + end + + def sidekiq_worker_class + Gitlab::GithubImport::PullRequests::ImportReviewRequestWorker + end + + def collection_method + :pull_request_review_requests + end + + # rubocop:disable CodeReuse/ActiveRecord + def merge_request_collection + project.merge_requests + .where.not(iid: already_imported_merge_requests) + .select(:id, :iid) + end + # rubocop:enable CodeReuse/ActiveRecord + + def merge_request_imported_cache_key + "github-importer/pull_requests/#{collection_method}/already-imported/#{project.id}" + end + + def already_imported_merge_requests + Gitlab::Cache::Import::Caching.values_from_set(merge_request_imported_cache_key) + end + + def mark_merge_request_imported(merge_request) + Gitlab::Cache::Import::Caching.set_add( + merge_request_imported_cache_key, + merge_request.iid + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 16541c90002..62863ba67fd 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -38,7 +38,7 @@ module Gitlab # deliberate. If we were to update this column after the fetch we may # miss out on changes pushed during the fetch or between the fetch and # updating the timestamp. - project.touch(:last_repository_updated_at) # rubocop: disable Rails/SkipsModelValidations + project.touch(:last_repository_updated_at) project.repository.fetch_remote(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: true) diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 708768a60cf..d7fe01e90f8 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -80,7 +80,7 @@ module Gitlab end def update_clone_time - project.touch(:last_repository_updated_at) # rubocop: disable Rails/SkipsModelValidations + project.touch(:last_repository_updated_at) end private diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb index 07a607ae70d..d2a52b64bbf 100644 --- a/lib/gitlab/github_import/representation/protected_branch.rb +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :attributes expose_attribute :id, :allow_force_pushes, :required_conversation_resolution, :required_signatures, - :required_pull_request_reviews + :required_pull_request_reviews, :require_code_owner_reviews # Builds a Branch Protection info from a GitHub API response. # Resource structure details: @@ -24,7 +24,9 @@ module Gitlab allow_force_pushes: branch_protection.dig(:allow_force_pushes, :enabled), required_conversation_resolution: branch_protection.dig(:required_conversation_resolution, :enabled), required_signatures: branch_protection.dig(:required_signatures, :enabled), - required_pull_request_reviews: branch_protection[:required_pull_request_reviews].present? + required_pull_request_reviews: branch_protection[:required_pull_request_reviews].present?, + require_code_owner_reviews: branch_protection.dig(:required_pull_request_reviews, + :require_code_owner_reviews).present? } new(hash) diff --git a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb new file mode 100644 index 00000000000..692004c4460 --- /dev/null +++ b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module PullRequests + class ReviewRequests + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :merge_request_id, :users + + class << self + # Builds a list of requested reviewers from a GitHub API response. + # + # review_requests - An instance of `Hash` containing the review requests details. + def from_api_response(review_requests, _additional_data = {}) + review_requests = Representation.symbolize_hash(review_requests) + users = review_requests[:users].map do |user_data| + Representation::User.from_api_response(user_data) + end + + new( + merge_request_id: review_requests[:merge_request_id], + users: users + ) + end + alias_method :from_json_hash, :from_api_response + end + + # attributes - A Hash containing the review details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { merge_request_id: merge_request_id } + end + end + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index bdb7484f3d6..ecb57bfc1a2 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -18,8 +18,16 @@ module Gitlab gon.markdown_automatic_lists = current_user&.markdown_automatic_lists if Gitlab.config.sentry.enabled - gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn - gon.sentry_environment = Gitlab.config.sentry.environment + gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn + gon.sentry_environment = Gitlab.config.sentry.environment + end + + # Support for Sentry setup via configuration files will be removed in 16.0 + # in favor of Gitlab::CurrentSettings. + if Feature.enabled?(:enable_new_sentry_clientside_integration, + current_user) && Gitlab::CurrentSettings.sentry_enabled + gon.sentry_dsn = Gitlab::CurrentSettings.sentry_clientside_dsn + gon.sentry_environment = Gitlab::CurrentSettings.sentry_environment end gon.recaptcha_api_server_url = ::Recaptcha.configuration.api_server_url @@ -58,6 +66,7 @@ module Gitlab push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:integration_slack_app_notifications) + push_frontend_feature_flag(:vue_group_select) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/grape_logging/loggers/filter_parameters.rb b/lib/gitlab/grape_logging/loggers/filter_parameters.rb new file mode 100644 index 00000000000..ae9df203544 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/filter_parameters.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module GrapeLogging + module Loggers + # In the CI variables APIs, the POST or PUT parameters will always be + # literally 'key' and 'value'. Rails' default filters_parameters will + # always incorrectly mask the value of param 'key' when it should mask the + # value of the param 'value'. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/353857 + class FilterParameters < ::GrapeLogging::Loggers::FilterParameters + private + + def safe_parameters(request) + loggable_params = super + settings = request.env[Grape::Env::API_ENDPOINT]&.route&.settings + + return loggable_params unless settings&.key?(:log_safety) + + settings[:log_safety][:safe].each do |key| + loggable_params[key] = request.params[key] if loggable_params.key?(key) + end + + settings[:log_safety][:unsafe].each do |key| + loggable_params[key] = @replacement if loggable_params.key?(key) + end + + loggable_params + end + end + end + end +end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index b71abe5c052..1a85c57e6b1 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -2,9 +2,9 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, language: nil, plain: false) + def self.highlight(blob_name, blob_content, language: nil, plain: false, context: {}) new(blob_name, blob_content, language: language) - .highlight(blob_content, continue: false, plain: plain) + .highlight(blob_content, continue: false, plain: plain, context: context) end def self.too_large?(size) diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 65c623c5d7d..96128f432c5 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -66,12 +66,19 @@ module Gitlab labels: merge_request.labels_hook_attrs, state: merge_request.state, # This key is deprecated blocking_discussions_resolved: merge_request.mergeable_discussions_state?, - first_contribution: merge_request.first_contribution? + first_contribution: merge_request.first_contribution?, + detailed_merge_status: detailed_merge_status } merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) .merge!(attrs) end + + private + + def detailed_merge_status + ::MergeRequests::Mergeability::DetailedMergeStatusService.new(merge_request: merge_request).execute.to_s + end end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index a2d06b7f5b3..a42cac61a55 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 37, + 'da_DK' => 36, 'de' => 17, 'en' => 100, 'eo' => 0, - 'es' => 36, + 'es' => 35, 'fil_PH' => 0, - 'fr' => 72, + 'fr' => 85, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 31, - 'ko' => 20, + 'ja' => 30, + 'ko' => 21, 'nb_NO' => 25, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 57, - 'ro_RO' => 99, - 'ru' => 26, + 'pt_BR' => 58, + 'ro_RO' => 98, + 'ru' => 25, 'si_LK' => 11, 'tr_TR' => 11, - 'uk' => 49, + 'uk' => 52, 'zh_CN' => 98, 'zh_HK' => 1, - 'zh_TW' => 99 + 'zh_TW' => 100 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index d5f94ad04f1..08d44184bb6 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -5,10 +5,11 @@ module Gitlab module Identifier def identify(identifier) - if identifier =~ /\Auser-\d+\Z/ + case identifier + when /\Auser-\d+\Z/ # git push over http identify_using_user(identifier) - elsif identifier =~ /\Akey-\d+\Z/ + when /\Akey-\d+\Z/ # git push over ssh identify_using_ssh_key(identifier) end diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index f6f65f85599..8c7a6c13246 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -85,11 +85,11 @@ module Gitlab while stack.any? model_name, relations = stack.pop - if relations.is_a?(Hash) - add_permitted_attributes(model_name, relations.keys) + next unless relations.is_a?(Hash) - stack.concat(relations.to_a) - end + add_permitted_attributes(model_name, relations.keys) + + stack.concat(relations.to_a) end @permitted_attributes diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index 3c473449ec0..ed3858d0bf4 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -81,11 +81,11 @@ module Gitlab subrelation = relation_object.public_send(definition) association = relation_object.class.reflect_on_association(definition) - if association&.collection? && subrelation.size > MIN_RECORDS_SIZE - collection_subrelations[definition] = subrelation.records + next unless association&.collection? && subrelation.size > MIN_RECORDS_SIZE - subrelation.clear - end + collection_subrelations[definition] = subrelation.records + + subrelation.clear end end end diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index c98dcf7b848..aa66fe8a5ae 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -87,7 +87,6 @@ module Gitlab def validate_archive_path Gitlab::Utils.check_path_traversal!(@archive_path) - raise(ServiceError, 'Archive path is not a string') unless @archive_path.is_a?(String) raise(ServiceError, 'Archive path is a symlink') if File.lstat(@archive_path).symlink? raise(ServiceError, 'Archive path is not a file') unless File.file?(@archive_path) end diff --git a/lib/gitlab/import_export/project/exported_relations_merger.rb b/lib/gitlab/import_export/project/exported_relations_merger.rb new file mode 100644 index 00000000000..dda3d00d608 --- /dev/null +++ b/lib/gitlab/import_export/project/exported_relations_merger.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class ExportedRelationsMerger + include Gitlab::ImportExport::CommandLineUtil + + def initialize(export_job:, shared:) + @export_job = export_job + @shared = shared + end + + def save + Dir.mktmpdir do |dirpath| + export_job.relation_exports.each do |relation_export| + relation = relation_export.relation + upload = relation_export.upload + filename = upload.export_file.filename + + tar_gz_full_path = File.join(dirpath, filename) + decompress_path = File.join(dirpath, relation) + Gitlab::Utils.check_path_traversal!(tar_gz_full_path) + Gitlab::Utils.check_path_traversal!(decompress_path) + + # Download tar.gz + download_or_copy_upload( + upload.export_file, tar_gz_full_path, size_limit: relation_export.upload.export_file.size + ) + + # Decompress tar.gz + mkdir_p(decompress_path) + untar_zxf(dir: decompress_path, archive: tar_gz_full_path) + File.delete(tar_gz_full_path) + + # Merge decompressed files into export_path + RecursiveMergeFolders.merge(decompress_path, shared.export_path) + FileUtils.rm_r(decompress_path) + rescue StandardError => e + shared.error(e) + false + end + end + + shared.errors.empty? + end + + private + + attr_reader :shared, :export_job + + delegate :project, to: :export_job + end + end + end +end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index fb44aaf094e..2d9c8d1108e 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -302,6 +302,7 @@ included_attributes: - :environments_access_level - :feature_flags_access_level - :releases_access_level + - :infrastructure_access_level prometheus_metrics: - :created_at - :updated_at @@ -585,7 +586,7 @@ included_attributes: - :target_sha pipeline_metadata: - :project_id - - :title + - :name stages: - :name - :status @@ -717,6 +718,7 @@ included_attributes: - :environments_access_level - :feature_flags_access_level - :releases_access_level + - :infrastructure_access_level - :allow_merge_on_skipped_pipeline - :auto_devops_deploy_strategy - :auto_devops_enabled diff --git a/lib/gitlab/import_export/project/relation_saver.rb b/lib/gitlab/import_export/project/relation_saver.rb index 8e91adac196..967239e17c1 100644 --- a/lib/gitlab/import_export/project/relation_saver.rb +++ b/lib/gitlab/import_export/project/relation_saver.rb @@ -32,7 +32,7 @@ module Gitlab project, reader.project_tree, json_writer, - exportable_path: 'project', + exportable_path: 'tree/project', current_user: nil ) end diff --git a/lib/gitlab/import_export/recursive_merge_folders.rb b/lib/gitlab/import_export/recursive_merge_folders.rb new file mode 100644 index 00000000000..982358699bd --- /dev/null +++ b/lib/gitlab/import_export/recursive_merge_folders.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true +# +# This class is used by Import/Export to move files and folders from a source folders into a target folders +# that can already have the same folders in it, resolving in a merged folder. +# +# Example: +# +# source path +# |-- tree +# | |-- project +# | |-- labels.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image1.png +# | |-- folder2 +# | | |-- image2.png +# +# target path +# |-- tree +# | |-- project +# | |-- issues.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image3.png +# | |-- folder3 +# | | |-- image4.png +# +# target path after merge +# |-- tree +# | |-- project +# | | |-- issues.ndjson +# | | |-- labels.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image1.png +# | | |-- image3.png +# | |-- folder2 +# | | |-- image2.png +# | |-- folder3 +# | | |-- image4.png + +module Gitlab + module ImportExport + class RecursiveMergeFolders + DEFAULT_DIR_MODE = 0o700 + + def self.merge(source_path, target_path) + Gitlab::Utils.check_path_traversal!(source_path) + Gitlab::Utils.check_path_traversal!(target_path) + Gitlab::Utils.check_allowed_absolute_path!(source_path, [Dir.tmpdir]) + + recursive_merge(source_path, target_path) + end + + def self.recursive_merge(source_path, target_path) + Dir.children(source_path).each do |child| + source_child = File.join(source_path, child) + target_child = File.join(target_path, child) + + next if File.lstat(source_child).symlink? + + if File.directory?(source_child) + FileUtils.mkdir_p(target_child, mode: DEFAULT_DIR_MODE) unless File.exist?(target_child) + recursive_merge(source_child, target_child) + else + FileUtils.mv(source_child, target_child) + end + end + end + + private_class_method :recursive_merge + end + end +end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index d55906083ff..d34c19bc9fc 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -2,30 +2,11 @@ module Gitlab module IncomingEmail - UNSUBSCRIBE_SUFFIX = '-unsubscribe' - UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe' - WILDCARD_PLACEHOLDER = '%{key}' - class << self - def enabled? - config.enabled && config.address.present? - end + include Gitlab::Email::Common - def supports_wildcard? - config.address.present? && config.address.include?(WILDCARD_PLACEHOLDER) - end - - def supports_issue_creation? - enabled? && supports_wildcard? - end - - def reply_address(key) - config.address.sub(WILDCARD_PLACEHOLDER, key) - end - - # example: incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com - def unsubscribe_address(key) - config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") + def config + incoming_email_config end def key_from_address(address, wildcard_address: nil) @@ -39,21 +20,6 @@ module Gitlab match[1] end - def key_from_fallback_message_id(mail_id) - message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/ - - mail_id[message_id_regexp, 1] - end - - def scan_fallback_references(references) - # It's looking for each <...> - references.scan(/(?!<)[^<>]+(?=>)/) - end - - def config - Gitlab.config.incoming_email - end - private def address_regex(wildcard_address) diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 0bd10597f24..268c6cdf459 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -66,8 +66,8 @@ module Gitlab query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end - def redis_cluster_validate!(command) - ::Gitlab::Instrumentation::RedisClusterValidator.validate!(command) if @redis_cluster_validation + def redis_cluster_validate!(commands) + ::Gitlab::Instrumentation::RedisClusterValidator.validate!(commands) if @redis_cluster_validation end def enable_redis_cluster_validation diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 005751fb0db..36d3e088956 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -10,57 +10,189 @@ module Gitlab # # Gitlab::Redis::Cache # .with { |redis| redis.call('COMMAND') } - # .select { |command| command[3] != command[4] } - # .map { |command| [command[0].upcase, { first: command[3], last: command[4], step: command[5] }] } + # .select { |cmd| cmd[3] != 0 } + # .map { |cmd| [ + # cmd[0].upcase, + # { first: cmd[3], last: cmd[4], step: cmd[5], single_key: cmd[3] == cmd[4] } + # ] + # } # .sort_by(&:first) # .to_h - # - MULTI_KEY_COMMANDS = { - "BITOP" => { first: 2, last: -1, step: 1 }, - "BLPOP" => { first: 1, last: -2, step: 1 }, - "BRPOP" => { first: 1, last: -2, step: 1 }, - "BRPOPLPUSH" => { first: 1, last: 2, step: 1 }, - "BZPOPMAX" => { first: 1, last: -2, step: 1 }, - "BZPOPMIN" => { first: 1, last: -2, step: 1 }, - "DEL" => { first: 1, last: -1, step: 1 }, - "EXISTS" => { first: 1, last: -1, step: 1 }, - "MGET" => { first: 1, last: -1, step: 1 }, - "MSET" => { first: 1, last: -1, step: 2 }, - "MSETNX" => { first: 1, last: -1, step: 2 }, - "PFCOUNT" => { first: 1, last: -1, step: 1 }, - "PFMERGE" => { first: 1, last: -1, step: 1 }, - "RENAME" => { first: 1, last: 2, step: 1 }, - "RENAMENX" => { first: 1, last: 2, step: 1 }, - "RPOPLPUSH" => { first: 1, last: 2, step: 1 }, - "SDIFF" => { first: 1, last: -1, step: 1 }, - "SDIFFSTORE" => { first: 1, last: -1, step: 1 }, - "SINTER" => { first: 1, last: -1, step: 1 }, - "SINTERSTORE" => { first: 1, last: -1, step: 1 }, - "SMOVE" => { first: 1, last: 2, step: 1 }, - "SUNION" => { first: 1, last: -1, step: 1 }, - "SUNIONSTORE" => { first: 1, last: -1, step: 1 }, - "UNLINK" => { first: 1, last: -1, step: 1 }, - "WATCH" => { first: 1, last: -1, step: 1 } + REDIS_COMMANDS = { + "APPEND" => { first: 1, last: 1, step: 1, single_key: true }, + "BITCOUNT" => { first: 1, last: 1, step: 1, single_key: true }, + "BITFIELD" => { first: 1, last: 1, step: 1, single_key: true }, + "BITFIELD_RO" => { first: 1, last: 1, step: 1, single_key: true }, + "BITOP" => { first: 2, last: -1, step: 1, single_key: false }, + "BITPOS" => { first: 1, last: 1, step: 1, single_key: true }, + "BLMOVE" => { first: 1, last: 2, step: 1, single_key: false }, + "BLPOP" => { first: 1, last: -2, step: 1, single_key: false }, + "BRPOP" => { first: 1, last: -2, step: 1, single_key: false }, + "BRPOPLPUSH" => { first: 1, last: 2, step: 1, single_key: false }, + "BZPOPMAX" => { first: 1, last: -2, step: 1, single_key: false }, + "BZPOPMIN" => { first: 1, last: -2, step: 1, single_key: false }, + "COPY" => { first: 1, last: 2, step: 1, single_key: false }, + "DECR" => { first: 1, last: 1, step: 1, single_key: true }, + "DECRBY" => { first: 1, last: 1, step: 1, single_key: true }, + "DEL" => { first: 1, last: -1, step: 1, single_key: false }, + "DUMP" => { first: 1, last: 1, step: 1, single_key: true }, + "EXISTS" => { first: 1, last: -1, step: 1, single_key: false }, + "EXPIRE" => { first: 1, last: 1, step: 1, single_key: true }, + "EXPIREAT" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOADD" => { first: 1, last: 1, step: 1, single_key: true }, + "GEODIST" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOHASH" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOPOS" => { first: 1, last: 1, step: 1, single_key: true }, + "GEORADIUS" => { first: 1, last: 1, step: 1, single_key: true }, + "GEORADIUSBYMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "GEORADIUSBYMEMBER_RO" => { first: 1, last: 1, step: 1, single_key: true }, + "GEORADIUS_RO" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOSEARCH" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOSEARCHSTORE" => { first: 1, last: 2, step: 1, single_key: false }, + "GET" => { first: 1, last: 1, step: 1, single_key: true }, + "GETBIT" => { first: 1, last: 1, step: 1, single_key: true }, + "GETDEL" => { first: 1, last: 1, step: 1, single_key: true }, + "GETEX" => { first: 1, last: 1, step: 1, single_key: true }, + "GETRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "GETSET" => { first: 1, last: 1, step: 1, single_key: true }, + "HDEL" => { first: 1, last: 1, step: 1, single_key: true }, + "HEXISTS" => { first: 1, last: 1, step: 1, single_key: true }, + "HGET" => { first: 1, last: 1, step: 1, single_key: true }, + "HGETALL" => { first: 1, last: 1, step: 1, single_key: true }, + "HINCRBY" => { first: 1, last: 1, step: 1, single_key: true }, + "HINCRBYFLOAT" => { first: 1, last: 1, step: 1, single_key: true }, + "HKEYS" => { first: 1, last: 1, step: 1, single_key: true }, + "HLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "HMGET" => { first: 1, last: 1, step: 1, single_key: true }, + "HMSET" => { first: 1, last: 1, step: 1, single_key: true }, + "HRANDFIELD" => { first: 1, last: 1, step: 1, single_key: true }, + "HSCAN" => { first: 1, last: 1, step: 1, single_key: true }, + "HSET" => { first: 1, last: 1, step: 1, single_key: true }, + "HSETNX" => { first: 1, last: 1, step: 1, single_key: true }, + "HSTRLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "HVALS" => { first: 1, last: 1, step: 1, single_key: true }, + "INCR" => { first: 1, last: 1, step: 1, single_key: true }, + "INCRBY" => { first: 1, last: 1, step: 1, single_key: true }, + "INCRBYFLOAT" => { first: 1, last: 1, step: 1, single_key: true }, + "LINDEX" => { first: 1, last: 1, step: 1, single_key: true }, + "LINSERT" => { first: 1, last: 1, step: 1, single_key: true }, + "LLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "LMOVE" => { first: 1, last: 2, step: 1, single_key: false }, + "LPOP" => { first: 1, last: 1, step: 1, single_key: true }, + "LPOS" => { first: 1, last: 1, step: 1, single_key: true }, + "LPUSH" => { first: 1, last: 1, step: 1, single_key: true }, + "LPUSHX" => { first: 1, last: 1, step: 1, single_key: true }, + "LRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "LREM" => { first: 1, last: 1, step: 1, single_key: true }, + "LSET" => { first: 1, last: 1, step: 1, single_key: true }, + "LTRIM" => { first: 1, last: 1, step: 1, single_key: true }, + "MGET" => { first: 1, last: -1, step: 1, single_key: false }, + "MIGRATE" => { first: 3, last: 3, step: 1, single_key: true }, + "MOVE" => { first: 1, last: 1, step: 1, single_key: true }, + "MSET" => { first: 1, last: -1, step: 2, single_key: false }, + "MSETNX" => { first: 1, last: -1, step: 2, single_key: false }, + "OBJECT" => { first: 2, last: 2, step: 1, single_key: true }, + "PERSIST" => { first: 1, last: 1, step: 1, single_key: true }, + "PEXPIRE" => { first: 1, last: 1, step: 1, single_key: true }, + "PEXPIREAT" => { first: 1, last: 1, step: 1, single_key: true }, + "PFADD" => { first: 1, last: 1, step: 1, single_key: true }, + "PFCOUNT" => { first: 1, last: -1, step: 1, single_key: false }, + "PFDEBUG" => { first: 2, last: 2, step: 1, single_key: true }, + "PFMERGE" => { first: 1, last: -1, step: 1, single_key: false }, + "PSETEX" => { first: 1, last: 1, step: 1, single_key: true }, + "PTTL" => { first: 1, last: 1, step: 1, single_key: true }, + "RENAME" => { first: 1, last: 2, step: 1, single_key: false }, + "RENAMENX" => { first: 1, last: 2, step: 1, single_key: false }, + "RESTORE" => { first: 1, last: 1, step: 1, single_key: true }, + "RESTORE-ASKING" => { first: 1, last: 1, step: 1, single_key: true }, + "RPOP" => { first: 1, last: 1, step: 1, single_key: true }, + "RPOPLPUSH" => { first: 1, last: 2, step: 1, single_key: false }, + "RPUSH" => { first: 1, last: 1, step: 1, single_key: true }, + "RPUSHX" => { first: 1, last: 1, step: 1, single_key: true }, + "SADD" => { first: 1, last: 1, step: 1, single_key: true }, + "SCARD" => { first: 1, last: 1, step: 1, single_key: true }, + "SDIFF" => { first: 1, last: -1, step: 1, single_key: false }, + "SDIFFSTORE" => { first: 1, last: -1, step: 1, single_key: false }, + "SET" => { first: 1, last: 1, step: 1, single_key: true }, + "SETBIT" => { first: 1, last: 1, step: 1, single_key: true }, + "SETEX" => { first: 1, last: 1, step: 1, single_key: true }, + "SETNX" => { first: 1, last: 1, step: 1, single_key: true }, + "SETRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "SINTER" => { first: 1, last: -1, step: 1, single_key: false }, + "SINTERSTORE" => { first: 1, last: -1, step: 1, single_key: false }, + "SISMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "SMEMBERS" => { first: 1, last: 1, step: 1, single_key: true }, + "SMISMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "SMOVE" => { first: 1, last: 2, step: 1, single_key: false }, + "SORT" => { first: 1, last: 1, step: 1, single_key: true }, + "SPOP" => { first: 1, last: 1, step: 1, single_key: true }, + "SRANDMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "SREM" => { first: 1, last: 1, step: 1, single_key: true }, + "SSCAN" => { first: 1, last: 1, step: 1, single_key: true }, + "STRLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "SUBSTR" => { first: 1, last: 1, step: 1, single_key: true }, + "SUNION" => { first: 1, last: -1, step: 1, single_key: false }, + "SUNIONSTORE" => { first: 1, last: -1, step: 1, single_key: false }, + "TOUCH" => { first: 1, last: -1, step: 1, single_key: false }, + "TTL" => { first: 1, last: 1, step: 1, single_key: true }, + "TYPE" => { first: 1, last: 1, step: 1, single_key: true }, + "UNLINK" => { first: 1, last: -1, step: 1, single_key: false }, + "WATCH" => { first: 1, last: -1, step: 1, single_key: false }, + "XACK" => { first: 1, last: 1, step: 1, single_key: true }, + "XADD" => { first: 1, last: 1, step: 1, single_key: true }, + "XAUTOCLAIM" => { first: 1, last: 1, step: 1, single_key: true }, + "XCLAIM" => { first: 1, last: 1, step: 1, single_key: true }, + "XDEL" => { first: 1, last: 1, step: 1, single_key: true }, + "XGROUP" => { first: 2, last: 2, step: 1, single_key: true }, + "XINFO" => { first: 2, last: 2, step: 1, single_key: true }, + "XLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "XPENDING" => { first: 1, last: 1, step: 1, single_key: true }, + "XRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "XREVRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "XSETID" => { first: 1, last: 1, step: 1, single_key: true }, + "XTRIM" => { first: 1, last: 1, step: 1, single_key: true }, + "ZADD" => { first: 1, last: 1, step: 1, single_key: true }, + "ZCARD" => { first: 1, last: 1, step: 1, single_key: true }, + "ZCOUNT" => { first: 1, last: 1, step: 1, single_key: true }, + "ZDIFFSTORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZINCRBY" => { first: 1, last: 1, step: 1, single_key: true }, + "ZINTERSTORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZLEXCOUNT" => { first: 1, last: 1, step: 1, single_key: true }, + "ZMSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZPOPMAX" => { first: 1, last: 1, step: 1, single_key: true }, + "ZPOPMIN" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANDMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANGEBYLEX" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANGEBYSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANGESTORE" => { first: 1, last: 2, step: 1, single_key: false }, + "ZRANK" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREM" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREMRANGEBYLEX" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREMRANGEBYRANK" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREMRANGEBYSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREVRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREVRANGEBYLEX" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREVRANGEBYSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREVRANK" => { first: 1, last: 1, step: 1, single_key: true }, + "ZSCAN" => { first: 1, last: 1, step: 1, single_key: true }, + "ZSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZUNIONSTORE" => { first: 1, last: 1, step: 1, single_key: true } }.freeze CrossSlotError = Class.new(StandardError) class << self - def validate!(command) + def validate!(commands) return unless Rails.env.development? || Rails.env.test? return if allow_cross_slot_commands? + return if commands.empty? - command_name = command.first.to_s.upcase - argument_positions = MULTI_KEY_COMMANDS[command_name] - - return unless argument_positions - - arguments = command.flatten[argument_positions[:first]..argument_positions[:last]] - - key_slots = arguments.each_slice(argument_positions[:step]).map do |args| - key_slot(args.first) - end + # early exit for single-command (non-pipelined) if it is a single-key-command + command_name = commands.size > 1 ? "PIPELINE/MULTI" : commands.first.first.to_s.upcase + return if commands.size == 1 && REDIS_COMMANDS.dig(command_name, :single_key) + key_slots = commands.map { |command| key_slots(command) }.flatten if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord raise CrossSlotError, "Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end @@ -78,6 +210,17 @@ module Gitlab private + def key_slots(command) + argument_positions = REDIS_COMMANDS[command.first.to_s.upcase] + + return [] unless argument_positions + + arguments = command.flatten[argument_positions[:first]..argument_positions[:last]] + arguments.each_slice(argument_positions[:step]).map do |args| + key_slot(args.first) + end + end + def allow_cross_slot_commands? Thread.current[:allow_cross_slot_commands].to_i > 0 end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 7e2acb91b94..f19279df2fe 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -5,14 +5,6 @@ module Gitlab module RedisInterceptor APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze - class MysteryRedisDurationError < StandardError - attr_reader :backtrace - - def initialize(backtrace) - @backtrace = backtrace - end - end - def call(command) instrument_call([command]) do super @@ -41,8 +33,7 @@ module Gitlab def instrument_call(commands) start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request(commands.size) - - commands.each { |c| instrumentation_class.redis_cluster_validate!(c) } + instrumentation_class.redis_cluster_validate!(commands) yield rescue ::Redis::BaseError => ex diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb index abb50281f7a..36346564b39 100644 --- a/lib/gitlab/issues/rebalancing/state.rb +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -25,7 +25,7 @@ module Gitlab redis.multi do |multi| # we trigger re-balance for namespaces(groups) or specific user project value = "#{rebalanced_container_type}/#{rebalanced_container_id}" - multi.sadd(CONCURRENT_RUNNING_REBALANCES_KEY, value) + multi.sadd?(CONCURRENT_RUNNING_REBALANCES_KEY, value) multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) end end @@ -99,11 +99,13 @@ module Gitlab def refresh_keys_expiration with_redis do |redis| - redis.multi do |multi| - multi.expire(issue_ids_key, REDIS_EXPIRY_TIME) - multi.expire(current_index_key, REDIS_EXPIRY_TIME) - multi.expire(current_project_key, REDIS_EXPIRY_TIME) - multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.multi do |multi| + multi.expire(issue_ids_key, REDIS_EXPIRY_TIME) + multi.expire(current_index_key, REDIS_EXPIRY_TIME) + multi.expire(current_project_key, REDIS_EXPIRY_TIME) + multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) + end end end end @@ -112,12 +114,14 @@ module Gitlab value = "#{rebalanced_container_type}/#{rebalanced_container_id}" with_redis do |redis| - redis.multi do |multi| - multi.del(issue_ids_key) - multi.del(current_index_key) - multi.del(current_project_key) - multi.srem(CONCURRENT_RUNNING_REBALANCES_KEY, value) - multi.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.multi do |multi| + multi.del(issue_ids_key) + multi.del(current_index_key) + multi.del(current_project_key) + multi.srem?(CONCURRENT_RUNNING_REBALANCES_KEY, value) + multi.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour) + end end end end @@ -136,9 +140,10 @@ module Gitlab current_rebalancing_containers.each do |string| container_type, container_id = string.split('/', 2).map(&:to_i) - if container_type == NAMESPACE + case container_type + when NAMESPACE namespace_ids << container_id - elsif container_type == PROJECT + when PROJECT project_ids << container_id end end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 823d6202b1e..8332e4f6d56 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -41,6 +41,11 @@ module Gitlab # as the underlying implementation of this varies wildly based on # the adapter in use. # + # This method does, in some situations, differ in the data it returns + # compared to .generate. Counter-intuitively, this is closest in + # terms of response to JSON.generate and to the default ActiveSupport + # .to_json method. + # # @param object [Object] the object to convert to JSON # @return [String] def dump(object) @@ -162,23 +167,11 @@ module Gitlab # @return [Boolean, String, Array, Hash, Object] # @raise [JSON::ParserError] def handle_legacy_mode!(data) - return data unless feature_table_exists? + return data unless Feature.feature_flags_available? return data unless Feature.enabled?(:json_wrapper_legacy_mode) raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) } end - - # There are a variety of database errors possible when checking the feature - # flags at the wrong time during boot, e.g. during migrations. We don't care - # about these errors, we just need to ensure that we skip feature detection - # if they will fail. - # - # @return [Boolean] - def feature_table_exists? - Feature::FlipperFeature.table_exists? - rescue StandardError - false - end end # GrapeFormatter is a JSON formatter for the Grape API. @@ -263,5 +256,19 @@ module Gitlab buffer.string end end + + class RailsEncoder < ActiveSupport::JSON::Encoding::JSONGemEncoder + # Rails doesn't provide a way of changing the JSON adapter for + # render calls in controllers, so here we're overriding the parent + # class method to use our generator, and it's monkey-patched in + # config/initializers/active_support_json.rb + def stringify(jsonified) + Gitlab::Json.dump(jsonified) + rescue EncodingError => ex + # Raise the same error as the default implementation if we encounter + # an error. These are usually related to invalid UTF-8 errors. + raise JSON::GeneratorError, ex + end + end end end diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb index d0dcd232ecc..7dbedef44ee 100644 --- a/lib/gitlab/json_logger.rb +++ b/lib/gitlab/json_logger.rb @@ -1,31 +1,52 @@ # frozen_string_literal: true +require 'labkit/logging' module Gitlab - class JsonLogger < ::Gitlab::Logger - def self.file_name_noext - raise NotImplementedError - end + class JsonLogger < ::Labkit::Logging::JsonLogger + class << self + def file_name_noext + raise NotImplementedError, "JsonLogger implementations must provide file_name_noext implementation" + end + + def file_name + file_name_noext + ".log" + end + + def debug(message) + build.debug(message) + end + + def error(message) + build.error(message) + end + + def warn(message) + build.warn(message) + end - def format_message(severity, timestamp, progname, message) - data = default_attributes - data[:severity] = severity - data[:time] = timestamp.utc.iso8601(3) - data[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id + def info(message) + build.info(message) + end - case message - when String - data[:message] = message - when Hash - data.merge!(message) + def build + Gitlab::SafeRequestStore[cache_key] ||= + new(full_log_path, level: log_level) end - Gitlab::Json.dump(data) + "\n" + def cache_key + "logger:" + full_log_path.to_s + end + + def full_log_path + Rails.root.join("log", file_name) + end end - protected + private - def default_attributes - {} + # Override Labkit's default impl, which uses the default Ruby platform json module. + def dump_json(data) + Gitlab::Json.dump(data) end end end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index bf7b7f2d089..a1e290a54e6 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -34,7 +34,7 @@ module Gitlab end def version_info - Gitlab::VersionInfo.parse(version) + Gitlab::VersionInfo.parse(version, parse_suffix: true) end # Return GitLab KAS external_url diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb index 6799be8e279..5fa77c1f1ba 100644 --- a/lib/gitlab/kroki.rb +++ b/lib/gitlab/kroki.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'asciidoctor/extensions/asciidoctor_kroki/version' require 'asciidoctor/extensions/asciidoctor_kroki/extension' module Gitlab diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb index 6fe9bb10cdf..3747431c6a7 100644 --- a/lib/gitlab/manifest_import/metadata.rb +++ b/lib/gitlab/manifest_import/metadata.rb @@ -14,9 +14,11 @@ module Gitlab def save(repositories, group_id) Gitlab::Redis::SharedState.with do |redis| - redis.multi do |multi| - multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) - multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.multi do |multi| + multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + end end end end diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index aab58bfa211..1997ebb952b 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -45,6 +45,18 @@ module Gitlab def db_config_name ::Gitlab::Database.db_config_name(marginalia_adapter) end + + def console_hostname + return unless ::Gitlab::Runtime.console? + + @cached_console_hostname ||= Socket.gethostname # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def console_username + return unless ::Gitlab::Runtime.console? + + ENV['SUDO_USER'] || ENV['USER'] + end end end end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index 752ab153f37..8cab069e1bf 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -10,9 +10,11 @@ module Gitlab results = {} Gitlab::Redis::Cache.with do |r| - r.pipelined do |pipeline| - subjects.each do |subject| - results[subject.cache_key] = new(subject).read(pipeline) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + r.pipelined do |pipeline| + subjects.each do |subject| + results[subject.cache_key] = new(subject).read(pipeline) + end end end end @@ -28,7 +30,7 @@ module Gitlab def save(updates) @loaded = false - Gitlab::Redis::Cache.with do |r| + with_redis do |r| r.mapped_hmset(markdown_cache_key, updates) r.expire(markdown_cache_key, EXPIRES_IN) end @@ -40,7 +42,7 @@ module Gitlab if pipeline pipeline.mapped_hmget(markdown_cache_key, *fields) else - Gitlab::Redis::Cache.with do |r| + with_redis do |r| r.mapped_hmget(markdown_cache_key, *fields) end end @@ -64,6 +66,10 @@ module Gitlab "markdown_cache:#{@subject.cache_key}" end + + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 7007fdfe386..19dfc640b5d 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -54,6 +54,17 @@ module Gitlab init_prometheus_metrics end + ## + # Configuration for Watchdog, use like: + # + # watchdog.configure do |config| + # config.handler = Gitlab::Memory::Watchdog::TermProcessHandler + # config.sleep_time_seconds = 60 + # config.logger = Gitlab::AppLogger + # config.monitors do |stack| + # stack.push MyMonitorClass, args*, max_strikes:, kwargs**, &block + # end + # end def configure yield @configuration end @@ -125,7 +136,7 @@ module Gitlab end def process_rss_bytes - Gitlab::Metrics::System.memory_usage_rss + Gitlab::Metrics::System.memory_usage_rss[:total] end def worker_id diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb index 2d84b083f55..793f75adf59 100644 --- a/lib/gitlab/memory/watchdog/configuration.rb +++ b/lib/gitlab/memory/watchdog/configuration.rb @@ -9,7 +9,7 @@ module Gitlab @monitors = [] end - def use(monitor_class, *args, **kwargs, &block) + def push(monitor_class, *args, **kwargs, &block) remove(monitor_class) @monitors.push(build_monitor_state(monitor_class, *args, **kwargs, &block)) end @@ -39,11 +39,12 @@ module Gitlab DEFAULT_SLEEP_TIME_SECONDS = 60 - attr_reader :monitors attr_writer :logger, :handler, :sleep_time_seconds - def initialize - @monitors = MonitorStack.new + def monitors + @monitor_stack ||= MonitorStack.new + yield @monitor_stack if block_given? + @monitor_stack end def handler diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb new file mode 100644 index 00000000000..6d6f97dc8ba --- /dev/null +++ b/lib/gitlab/memory/watchdog/configurator.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class Configurator + class << self + def configure_for_puma + lambda do |config| + sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', 60).to_i + config.logger = Gitlab::AppLogger + config.handler = Gitlab::Memory::Watchdog::PumaHandler.new + config.sleep_time_seconds = sleep_time_seconds + config.monitors(&configure_monitors_for_puma) + end + end + + def configure_for_sidekiq + lambda do |config| + sleep_time_seconds = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max + config.logger = Sidekiq.logger + config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new + config.sleep_time_seconds = sleep_time_seconds + config.monitors(&configure_monitors_for_sidekiq) + end + end + + private + + def configure_monitors_for_puma + lambda do |stack| + max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', 5).to_i + + if Gitlab::Utils.to_boolean(ENV['DISABLE_PUMA_WORKER_KILLER']) + max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', 0.5).to_f + max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', 3.0).to_f + + # stack.push MonitorClass, args*, max_strikes:, kwargs**, &block + stack.push Gitlab::Memory::Watchdog::Monitor::HeapFragmentation, + max_heap_fragmentation: max_heap_frag, + max_strikes: max_strikes + + stack.push Gitlab::Memory::Watchdog::Monitor::UniqueMemoryGrowth, + max_mem_growth: max_mem_growth, + max_strikes: max_strikes + else + memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', 1200).to_i + + stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, + memory_limit: memory_limit, + max_strikes: max_strikes + end + end + end + + def configure_monitors_for_sidekiq + # NOP - At the moment we don't run watchdog for Sidekiq + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb index 7748c19c6d8..8f230980eac 100644 --- a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb +++ b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb @@ -22,7 +22,7 @@ module Gitlab def call heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation - return { threshold_violated: false, payload: {} } unless heap_fragmentation > max_heap_fragmentation + return { threshold_violated: false, payload: {} } if heap_fragmentation <= max_heap_fragmentation { threshold_violated: true, payload: payload(heap_fragmentation) } end diff --git a/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb new file mode 100644 index 00000000000..3e7de024630 --- /dev/null +++ b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Monitor + class RssMemoryLimit + attr_reader :memory_limit + + def initialize(memory_limit:) + @memory_limit = memory_limit + end + + def call + worker_rss = Gitlab::Metrics::System.memory_usage_rss[:total] + + return { threshold_violated: false, payload: {} } if worker_rss <= memory_limit + + { threshold_violated: true, payload: payload(worker_rss, memory_limit) } + end + + private + + def payload(worker_rss, memory_limit) + { + message: 'rss memory limit exceeded', + memwd_rss_bytes: worker_rss, + memwd_max_rss_bytes: memory_limit + } + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb b/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb index 2a1512c4cff..ce3477e6227 100644 --- a/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb +++ b/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb @@ -16,7 +16,7 @@ module Gitlab reference_uss = reference_mem[:uss] memory_limit = max_mem_growth * reference_uss - return { threshold_violated: false, payload: {} } unless worker_uss > memory_limit + return { threshold_violated: false, payload: {} } if worker_uss <= memory_limit { threshold_violated: true, payload: payload(worker_uss, reference_uss, memory_limit) } end diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb index a25156661af..fae4b721e1a 100644 --- a/lib/gitlab/merge_requests/mergeability/check_result.rb +++ b/lib/gitlab/merge_requests/mergeability/check_result.rb @@ -22,8 +22,8 @@ module Gitlab def self.from_hash(data) new( - status: data.fetch('status').to_sym, - payload: data.fetch('payload')) + status: data.fetch(:status).to_sym, + payload: data.fetch(:payload)) end def initialize(status:, payload: {}) diff --git a/lib/gitlab/merge_requests/mergeability/redis_interface.rb b/lib/gitlab/merge_requests/mergeability/redis_interface.rb index b0e739f91ff..1129fa639d8 100644 --- a/lib/gitlab/merge_requests/mergeability/redis_interface.rb +++ b/lib/gitlab/merge_requests/mergeability/redis_interface.rb @@ -7,16 +7,20 @@ module Gitlab VERSION = 1 def save_check(merge_check:, result_hash:) - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.set(merge_check.cache_key + ":#{VERSION}", result_hash.to_json, ex: EXPIRATION) end end def retrieve_check(merge_check:) - Gitlab::Redis::Cache.with do |redis| - Gitlab::Json.parse(redis.get(merge_check.cache_key + ":#{VERSION}")) + with_redis do |redis| + Gitlab::Json.parse(redis.get(merge_check.cache_key + ":#{VERSION}"), symbolize_keys: true) end end + + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index c8591a81a05..a4964ae0ebc 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -78,7 +78,7 @@ module Gitlab end def predefined_dashboard_services_for(project) - # Only list the self monitoring dashboard on the self monitoring project, + # Only list the self-monitoring dashboard on the self-monitoring project, # since it is the only dashboard (at time of writing) that shows data # about GitLab itself. if project.self_monitoring? diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index 3400a6c78ef..200c6eb4043 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -5,12 +5,12 @@ module Gitlab module GlobalSearchSlis class << self # The following targets are the 99.95th percentile of code searches - # gathered on 24-08-2022 + # gathered on 25-10-2022 # from https://log.gprd.gitlab.net/goto/0c89cd80-23af-11ed-8656-f5f2137823ba (internal only) - BASIC_CONTENT_TARGET_S = 7.031 - BASIC_CODE_TARGET_S = 21.903 - ADVANCED_CONTENT_TARGET_S = 4.865 - ADVANCED_CODE_TARGET_S = 13.546 + BASIC_CONTENT_TARGET_S = 8.812 + BASIC_CODE_TARGET_S = 27.538 + ADVANCED_CONTENT_TARGET_S = 2.452 + ADVANCED_CODE_TARGET_S = 15.52 def initialize_slis! Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) diff --git a/lib/gitlab/metrics/loose_foreign_keys_slis.rb b/lib/gitlab/metrics/loose_foreign_keys_slis.rb new file mode 100644 index 00000000000..5d8245aa609 --- /dev/null +++ b/lib/gitlab/metrics/loose_foreign_keys_slis.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module LooseForeignKeysSlis + class << self + def initialize_slis! + Gitlab::Metrics::Sli::Apdex.initialize_sli(:loose_foreign_key_clean_ups, possible_labels) + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:loose_foreign_key_clean_ups, possible_labels) + end + + def record_apdex(success:, db_config_name:) + Gitlab::Metrics::Sli::Apdex[:loose_foreign_key_clean_ups].increment( + labels: labels(db_config_name), + success: success + ) + end + + def record_error_rate(error:, db_config_name:) + Gitlab::Metrics::Sli::ErrorRate[:loose_foreign_key_clean_ups].increment( + labels: labels(db_config_name), + error: error + ) + end + + private + + def possible_labels + ::Gitlab::Database.db_config_names.map do |db_config_name| + { + db_config_name: db_config_name, + feature_category: :database + } + end + end + + def labels(db_config_name) + { + db_config_name: db_config_name, + feature_category: :database + } + end + end + end + end +end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index c6b0a0c5e76..f39ec9cc8ab 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -39,7 +39,6 @@ module Gitlab docstring 'Method calls real duration' label_keys label_keys buckets [0.01, 0.05, 0.1, 0.5, 1] - with_feature :prometheus_metrics_method_instrumentation end end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index b2a9de21145..e62a62a935e 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -46,11 +46,11 @@ module Gitlab # 2. Don't sample data at the same interval two times in a row. def sleep_interval while step = @interval_steps.sample - if step != @last_step - @last_step = step + next if step == @last_step - return @interval + @last_step - end + @last_step = step + + return @interval + @last_step end end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4fe338ffc7f..5a7ca6b6c04 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -35,6 +35,8 @@ module Gitlab process_cpu_seconds_total: ::Gitlab::Metrics.gauge(metric_name(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(metric_name(:process, :max_fds), 'Process max fds'), process_resident_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), + process_resident_anon_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_anon_memory_bytes), 'Anonymous memory used (RSS)', labels), + process_resident_file_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_file_memory_bytes), 'File backed memory used (RSS)', labels), process_unique_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :unique_memory_bytes), 'Memory used (USS)', labels), process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), @@ -95,7 +97,10 @@ module Gitlab end def set_memory_usage_metrics - metrics[:process_resident_memory_bytes].set(labels, System.memory_usage_rss) + rss = System.memory_usage_rss + metrics[:process_resident_memory_bytes].set(labels, rss[:total]) + metrics[:process_resident_anon_memory_bytes].set(labels, rss[:anon]) + metrics[:process_resident_file_memory_bytes].set(labels, rss[:file]) if Gitlab::Utils.to_boolean(ENV['enable_memory_uss_pss'] || '1') memory_uss_pss = System.memory_usage_uss_pss diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index affadc4274c..9b0ae84dec2 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -18,7 +18,9 @@ module Gitlab PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/.freeze PSS_PATTERN = /^Pss:\s+(?<value>\d+)/.freeze - RSS_PATTERN = /VmRSS:\s+(?<value>\d+)/.freeze + RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/.freeze + RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/.freeze + RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/.freeze MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/.freeze @@ -27,7 +29,7 @@ module Gitlab { version: RUBY_DESCRIPTION, gc_stat: GC.stat, - memory_rss: memory_usage_rss, + memory_rss: memory_usage_rss[:total], memory_uss: proportional_mem[:uss], memory_pss: proportional_mem[:pss], time_cputime: cpu_time, @@ -38,7 +40,21 @@ module Gitlab # Returns the given process' RSS (resident set size) in bytes. def memory_usage_rss(pid: 'self') - sum_matches(PROC_STATUS_PATH % pid, rss: RSS_PATTERN)[:rss].kilobytes + results = { total: 0, anon: 0, file: 0 } + + safe_yield_procfile(PROC_STATUS_PATH % pid) do |io| + io.each_line do |line| + if (value = parse_metric_value(line, RSS_TOTAL_PATTERN)) > 0 + results[:total] = value.kilobytes + elsif (value = parse_metric_value(line, RSS_ANON_PATTERN)) > 0 + results[:anon] = value.kilobytes + elsif (value = parse_metric_value(line, RSS_FILE_PATTERN)) > 0 + results[:file] = value.kilobytes + end + end + end + + results end # Returns the given process' USS/PSS (unique/proportional set size) in bytes. @@ -115,9 +131,7 @@ module Gitlab safe_yield_procfile(proc_file) do |io| io.each_line do |line| patterns.each do |metric, pattern| - match = line.match(pattern) - value = match&.named_captures&.fetch('value', 0) - results[metric] += value.to_i + results[metric] += parse_metric_value(line, pattern) end end end @@ -125,6 +139,13 @@ module Gitlab results end + def parse_metric_value(line, pattern) + match = line.match(pattern) + return 0 unless match + + match.named_captures.fetch('value', 0).to_i + end + def proc_stat_entries safe_yield_procfile(PROC_STAT_PATH) do |io| io.read.split(' ') diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb index a8e25708107..8cb2729ff61 100644 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -42,13 +42,10 @@ module Gitlab def build menu = @menu_builder.build - hide_menu_text = Feature.enabled?(:new_navbar_layout) - menu.merge({ views: @views, shortcuts: @shortcuts, - menuTitle: (_('Menu') unless hide_menu_text), - menuTooltip: (_('Main menu') if hide_menu_text) + menuTooltip: _('Main menu') }.compact) end end diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb new file mode 100644 index 00000000000..8dde60a73be --- /dev/null +++ b/lib/gitlab/observability.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Observability + module_function + + def observability_url + return ENV['OVERRIDE_OBSERVABILITY_URL'] if ENV['OVERRIDE_OBSERVABILITY_URL'] + # TODO Make observability URL configurable https://gitlab.com/gitlab-org/opstrace/opstrace-ui/-/issues/80 + return 'https://observe.staging.gitlab.com' if Gitlab.staging? + + 'https://observe.gitlab.com' + end + end +end diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb index a3c0fdcf467..a92860f7eb8 100644 --- a/lib/gitlab/octokit/middleware.rb +++ b/lib/gitlab/octokit/middleware.rb @@ -8,7 +8,11 @@ module Gitlab end def call(env) - Gitlab::UrlBlocker.validate!(env[:url], allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests?) + Gitlab::UrlBlocker.validate!(env[:url], + schemes: %w[http https], + allow_localhost: allow_local_requests?, + allow_local_network: allow_local_requests? + ) @app.call(env) end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index d4de2791195..6235874132f 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -35,11 +35,12 @@ module Gitlab def keyset_pagination_enabled?(finder) return false unless params[:pagination] == "keyset" - if finder.is_a?(BranchesFinder) + case finder + when BranchesFinder Feature.enabled?(:branch_list_keyset_pagination, project) - elsif finder.is_a?(TagsFinder) + when TagsFinder true - elsif finder.is_a?(::Repositories::TreeFinder) + when ::Repositories::TreeFinder Feature.enabled?(:repository_tree_gitaly_pagination, project) else false @@ -49,11 +50,12 @@ module Gitlab def paginate_first_page?(finder) return false unless params[:page].blank? || params[:page].to_i == 1 - if finder.is_a?(BranchesFinder) + case finder + when BranchesFinder Feature.enabled?(:branch_list_keyset_pagination, project) - elsif finder.is_a?(TagsFinder) + when TagsFinder true - elsif finder.is_a?(::Repositories::TreeFinder) + when ::Repositories::TreeFinder Feature.enabled?(:repository_tree_gitaly_pagination, project) else false diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb index 51f38c1da58..4f79a3593f4 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -39,15 +39,15 @@ module Gitlab def verify_order_by_attributes_on_model!(model, order_by_columns) order_by_columns.map(&:column).each do |column| - unless model.columns_hash[column.attribute_name.to_s] - text = <<~TEXT + next if model.columns_hash[column.attribute_name.to_s] + + text = <<~TEXT The "RecordLoaderStrategy" does not support the following ORDER BY column because it's not available on the \"#{model.table_name}\" table: #{column.attribute_name} Omit the "finder_query" parameter to use the "OrderValuesLoaderStrategy". - TEXT - raise text - end + TEXT + raise text end end end diff --git a/lib/gitlab/pagination_delegate.rb b/lib/gitlab/pagination_delegate.rb new file mode 100644 index 00000000000..05aaff5bbfc --- /dev/null +++ b/lib/gitlab/pagination_delegate.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + class PaginationDelegate # rubocop:disable Gitlab/NamespacedClass + DEFAULT_PER_PAGE = Kaminari.config.default_per_page + MAX_PER_PAGE = Kaminari.config.max_per_page + + def initialize(page:, per_page:, count:, options: {}) + @count = count + @options = { default_per_page: DEFAULT_PER_PAGE, + max_per_page: MAX_PER_PAGE }.merge(options) + + @per_page = sanitize_per_page(per_page) + @page = sanitize_page(page) + end + + def total_count + @count + end + + def total_pages + (total_count.to_f / @per_page).ceil + end + + def next_page + current_page + 1 unless last_page? + end + + def prev_page + current_page - 1 unless first_page? + end + + def current_page + @page + end + + def limit_value + @per_page + end + + def first_page? + current_page == 1 + end + + def last_page? + current_page >= total_pages + end + + def offset + (current_page - 1) * limit_value + end + + private + + def sanitize_per_page(per_page) + return @options[:default_per_page] unless per_page && per_page > 0 + + [@options[:max_per_page], per_page].min + end + + def sanitize_page(page) + return 1 unless page && page > 1 + + [total_pages, page].min + end + end +end diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb new file mode 100644 index 00000000000..c9eae2f899f --- /dev/null +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Patch to address https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1932 +# It restores the behavior of `poll_internal_average` to the one from Sidekiq 6.5.7 +# when the cron poll interval is not configured. +# (see https://github.com/mperham/sidekiq/blob/v6.5.7/lib/sidekiq/scheduled.rb#L176-L178) +require 'sidekiq/version' +require 'sidekiq/cron/version' + +if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.7') + raise 'New version of sidekiq detected, please remove or update this patch' +end + +if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.8.0') + raise 'New version of sidekiq-cron detected, please remove or update this patch' +end + +module Gitlab + module Patch + module SidekiqCronPoller + def poll_interval_average(count) + Gitlab.config.cron_jobs.poll_interval || @config[:poll_interval_average] || scaled_poll_interval(count) # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb index fbc77113875..79c00a48336 100644 --- a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -19,7 +19,7 @@ module Gitlab def enqueue_stats_job(request_id) return unless Feature.enabled?(:performance_bar_stats, type: :ops) - @client.sadd(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) + @client.sadd?(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) return unless uuid = Gitlab::ExclusiveLease.new( GitlabPerformanceBarStatsWorker::LEASE_KEY, diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index fb9447f9665..8cc96970ebd 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -117,7 +117,7 @@ module Gitlab end def blobs(limit: count_limit) - return [] unless Ability.allowed?(@current_user, :download_code, @project) + return [] unless Ability.allowed?(@current_user, :read_code, @project) @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query, content_match_cutoff: limit) end @@ -153,7 +153,7 @@ module Gitlab end def find_commits(query, limit:) - return [] unless Ability.allowed?(@current_user, :download_code, @project) + return [] unless Ability.allowed?(@current_user, :read_code, @project) commits = find_commits_by_message(query, limit: limit) commit_by_sha = find_commit_by_sha(query) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 6673940ccf3..51a5bedc44b 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -28,6 +28,14 @@ module Gitlab "#{preview}.git" end + def project_host + return unless preview + + uri = URI.parse(preview) + uri.path = "" + uri.to_s + end + def project_path URI.parse(preview).path.delete_prefix('/') end diff --git a/lib/gitlab/qa.rb b/lib/gitlab/qa.rb new file mode 100644 index 00000000000..c47a8982901 --- /dev/null +++ b/lib/gitlab/qa.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Qa + def self.user_agent + ENV['GITLAB_QA_USER_AGENT'] + end + + def self.request?(request_user_agent) + return false unless Gitlab.com? + return false unless request_user_agent.present? + return false unless user_agent.present? + + ActiveSupport::SecurityUtils.secure_compare(request_user_agent, user_agent) + end + end +end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 46c0a0ddf7a..498da38e268 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -68,14 +68,14 @@ module Gitlab GEO_NODES_LOAD = 'SELECT 1 AS one FROM "geo_nodes" LIMIT 1' LICENSES_LOAD = 'SELECT "licenses".* FROM "licenses" ORDER BY "licenses"."id"' - ATTR_INTROSPECTION = %r/SELECT .*\ba.attname\b.* (FROM|JOIN) pg_attribute a/m.freeze + SCHEMA_INTROSPECTION = %r/SELECT.*(FROM|JOIN) (pg_attribute|pg_class)/m.freeze # queries can be safely ignored if they are amoritized in regular usage # (i.e. only requested occasionally and otherwise cached). def ignorable?(sql) return true if sql&.include?(GEO_NODES_LOAD) return true if sql&.include?(LICENSES_LOAD) - return true if ATTR_INTROSPECTION =~ sql + return true if SCHEMA_INTROSPECTION.match?(sql) false end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 3b85d6952a1..0b37c80dc5f 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -12,16 +12,13 @@ module Gitlab included do # Issue, MergeRequest, Epic: quick actions definitions desc do - _('Close this %{quick_action_target}') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Close this %{quick_action_target}') % { quick_action_target: target_issuable_name } end explanation do - _('Closes this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Closes this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end execution_message do - _('Closed this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Closed this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end types ::Issuable condition do @@ -35,15 +32,15 @@ module Gitlab desc do _('Reopen this %{quick_action_target}') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + { quick_action_target: target_issuable_name } end explanation do _('Reopens this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + { quick_action_target: target_issuable_name } end execution_message do _('Reopened this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + { quick_action_target: target_issuable_name } end types ::Issuable condition do @@ -170,12 +167,10 @@ module Gitlab desc { _('Subscribe') } explanation do - _('Subscribes to this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Subscribes to this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end execution_message do - _('Subscribed to this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Subscribed to this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end types ::Issuable condition do @@ -188,12 +183,10 @@ module Gitlab desc { _('Unsubscribe') } explanation do - _('Unsubscribes from this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Unsubscribes from this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end execution_message do - _('Unsubscribed from this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Unsubscribed from this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end types ::Issuable condition do @@ -266,6 +259,16 @@ module Gitlab end end + desc { _("Make %{type} confidential") % { type: target_issuable_name } } + explanation { _("Makes this %{type} confidential.") % { type: target_issuable_name } } + types ::Issuable + condition { quick_action_target.supports_confidentiality? && can_make_confidential? } + command :confidential do + @updates[:confidential] = true + + @execution_message[:confidential] = confidential_execution_message + end + private def find_severity(severity_param) @@ -315,6 +318,29 @@ module Gitlab _('Removed all labels.') end end + + def target_issuable_name + quick_action_target.to_ability_name.humanize(capitalize: false) + end + + def can_make_confidential? + confidentiality_not_supported = quick_action_target.respond_to?(:issue_type_supports?) && + !quick_action_target.issue_type_supports?(:confidentiality) + + return false if confidentiality_not_supported + + !quick_action_target.confidential? && current_user.can?(:set_confidentiality, quick_action_target) + end + + def confidential_execution_message + confidential_error_message.presence || _("Made this %{type} confidential.") % { type: target_issuable_name } + end + + def confidential_error_message + return unless quick_action_target.respond_to?(:confidentiality_errors) + + quick_action_target.confidentiality_errors.join("\n") + end end end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 4883c649a62..e74c58e45b1 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -161,23 +161,6 @@ module Gitlab @execution_message[:move] = message end - desc { _('Make issue confidential') } - explanation do - _('Makes this issue confidential.') - end - execution_message do - _('Made this issue confidential.') - end - types Issue - condition do - quick_action_target.issue_type_supports?(:confidentiality) && - !quick_action_target.confidential? && - current_user.can?(:set_confidentiality, quick_action_target) - end - command :confidential do - @updates[:confidential] = true - end - desc { _('Create a merge request') } explanation do |branch_name = nil| if branch_name diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index a0faf8dd460..8b1ff5d298a 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -161,7 +161,7 @@ module Gitlab parse_params do |raw_duration| Gitlab::TimeTrackingFormatter.parse(raw_duration) end - command :estimate do |time_estimate| + command :estimate, :estimate_time do |time_estimate| if time_estimate @updates[:time_estimate] = time_estimate end @@ -184,7 +184,7 @@ module Gitlab parse_params do |raw_time_date| Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute end - command :spend, :spent do |time_spent, time_spent_date| + command :spend, :spent, :spend_time do |time_spent, time_spent_date| if time_spent @updates[:spend_time] = { duration: time_spent, @@ -202,7 +202,7 @@ module Gitlab quick_action_target.persisted? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end - command :remove_estimate do + command :remove_estimate, :remove_time_estimate do @updates[:time_estimate] = 0 end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index a7c36786d2d..12cb1fc6153 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -52,6 +52,7 @@ module Gitlab del flushdb rpush + eval ).freeze PIPELINED_COMMANDS = %i( diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index f914123a94d..c5798bec0d7 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -42,10 +42,10 @@ module Gitlab @references[type] ||= references(type) end - if %w(mentioned_user mentioned_group mentioned_project).include?(type.to_s) - define_method("#{type}_ids") do - @references[type] ||= references(type, ids_only: true) - end + next unless %w(mentioned_user mentioned_group mentioned_project).include?(type.to_s) + + define_method("#{type}_ids") do + @references[type] ||= references(type, ids_only: true) end end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index 258c904290d..d5e80053772 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -10,6 +10,14 @@ module Gitlab class Controller < ActionController::Base protect_from_forgery with: :exception, prepend: true + def initialize + super + + # Squelch noisy and unnecessary "Can't verify CSRF token authenticity." messages. + # X-Csrf-Token is only one authentication mechanism for API helpers. + self.logger = ActiveSupport::Logger.new(File::NULL) + end + def index head :ok end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 6d95cb9a87b..7e9fb82fb8b 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -42,7 +42,7 @@ module Gitlab end def sidekiq? - !!(defined?(::Sidekiq) && Sidekiq.server?) + !!(defined?(::Sidekiq) && Sidekiq.try(:server?)) end def rake? @@ -94,7 +94,7 @@ module Gitlab # # These threads execute Sidekiq client middleware when jobs # are enqueued and those can access DB / Redis. - threads += Sidekiq.options[:concurrency] + 2 + threads += Sidekiq[:concurrency] + 2 end if puma? diff --git a/lib/gitlab/search/recent_items.rb b/lib/gitlab/search/recent_items.rb index 593148025e1..7a16b5dfc87 100644 --- a/lib/gitlab/search/recent_items.rb +++ b/lib/gitlab/search/recent_items.rb @@ -33,7 +33,7 @@ module Gitlab end def search(term) - finder.new(user, search: term, in: 'title') + finder.new(user, search: term, in: 'title', skip_full_text_search_project_condition: true) .execute .limit(SEARCH_LIMIT).reorder(nil).id_in_ordered(latest_ids) # rubocop: disable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/service_desk_email.rb b/lib/gitlab/service_desk_email.rb index 14f07140825..bc49efafdda 100644 --- a/lib/gitlab/service_desk_email.rb +++ b/lib/gitlab/service_desk_email.rb @@ -3,8 +3,10 @@ module Gitlab module ServiceDeskEmail class << self - def enabled? - !!config&.enabled && config&.address.present? + include Gitlab::Email::Common + + def config + Gitlab.config.service_desk_email end def key_from_address(address) @@ -14,20 +16,10 @@ module Gitlab Gitlab::IncomingEmail.key_from_address(address, wildcard_address: wildcard_address) end - def config - Gitlab.config.service_desk_email - end - def address_for_key(key) return if config.address.blank? - config.address.sub(Gitlab::IncomingEmail::WILDCARD_PLACEHOLDER, key) - end - - def key_from_fallback_message_id(mail_id) - message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/ - - mail_id[message_id_regexp, 1] + config.address.sub(WILDCARD_PLACEHOLDER, key) end end end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index c7818cb3418..3d2ff5a68d2 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -34,7 +34,7 @@ module Gitlab def write(key, value) with do |redis| redis.pipelined do |pipeline| - pipeline.sadd(cache_key(key), value) + pipeline.sadd?(cache_key(key), value) pipeline.expire(cache_key(key), expires_in) end diff --git a/lib/gitlab/shard_health_cache.rb b/lib/gitlab/shard_health_cache.rb index eeb0cc75ef9..a34e4e9c8d1 100644 --- a/lib/gitlab/shard_health_cache.rb +++ b/lib/gitlab/shard_health_cache.rb @@ -7,17 +7,17 @@ module Gitlab # Clears the Redis set storing the list of healthy shards def self.clear - Gitlab::Redis::Cache.with { |redis| redis.del(HEALTHY_SHARDS_KEY) } + with_redis { |redis| redis.del(HEALTHY_SHARDS_KEY) } end # Updates the list of healthy shards using a Redis set # # shards - An array of shard names to store def self.update(shards) - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.multi do |m| m.del(HEALTHY_SHARDS_KEY) - shards.each { |shard_name| m.sadd(HEALTHY_SHARDS_KEY, shard_name) } + m.sadd(HEALTHY_SHARDS_KEY, shards) unless shards.blank? m.expire(HEALTHY_SHARDS_KEY, HEALTHY_SHARDS_TIMEOUT) end end @@ -25,19 +25,23 @@ module Gitlab # Returns an array of strings of healthy shards def self.cached_healthy_shards - Gitlab::Redis::Cache.with { |redis| redis.smembers(HEALTHY_SHARDS_KEY) } + with_redis { |redis| redis.smembers(HEALTHY_SHARDS_KEY) } end # Checks whether the given shard name is in the list of healthy shards. # # shard_name - The string to check def self.healthy_shard?(shard_name) - Gitlab::Redis::Cache.with { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) } + with_redis { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) } end # Returns the number of healthy shards in the Redis set def self.healthy_shard_count - Gitlab::Redis::Cache.with { |redis| redis.scard(HEALTHY_SHARDS_KEY) } + with_redis { |redis| redis.scard(HEALTHY_SHARDS_KEY) } + end + + def self.with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index b167afe589a..bc59d4ce943 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -14,6 +14,11 @@ module Gitlab class Shell Error = Class.new(StandardError) + PERMITTED_ACTIONS = %w[ + mv_repository remove_repository add_namespace rm_namespace mv_namespace + repository_exists? + ].freeze + class << self # Retrieve GitLab Shell secret token # diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 3e7bdfbe89a..7e2a934b3dd 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -162,7 +162,7 @@ module Gitlab # the current Sidekiq process def current_worker_queue_mappings worker_queue_mappings - .select { |worker, queue| Sidekiq.options[:queues].include?(queue) } + .select { |worker, queue| Sidekiq[:queues].include?(queue) } .to_h end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index b8f86b92844..d5227e7a007 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -118,9 +118,9 @@ module Gitlab return unless enabled? # Tell sidekiq to restart itself - # Keep extra safe to wait `Sidekiq.options[:timeout] + 2` seconds before SIGKILL + # Keep extra safe to wait `Sidekiq[:timeout] + 2` seconds before SIGKILL refresh_state(:shutting_down) - signal_and_wait(Sidekiq.options[:timeout] + 2, 'SIGTERM', 'gracefully shut down') + signal_and_wait(Sidekiq[:timeout] + 2, 'SIGTERM', 'gracefully shut down') return unless enabled? # Ideally we should never reach this condition @@ -221,7 +221,7 @@ module Gitlab end def get_rss_kb - Gitlab::Metrics::System.memory_usage_rss / 1.kilobytes + Gitlab::Metrics::System.memory_usage_rss[:total] / 1.kilobytes end def get_soft_limit_rss_kb diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index fe5213fc5d7..2c506786d83 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -3,8 +3,10 @@ module Gitlab module SidekiqMiddleware class ArgumentsLogger + include Sidekiq::ServerMiddleware + def call(worker, job, queue) - Sidekiq.logger.info "arguments: #{Gitlab::Json.dump(job['args'])}" + logger.info "arguments: #{Gitlab::Json.dump(job['args'])}" yield end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index d42bd672bac..357e9d41187 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'digest' +require 'msgpack' module Gitlab module SidekiqMiddleware @@ -20,23 +21,8 @@ module Gitlab include Gitlab::Utils::StrongMemoize DEFAULT_DUPLICATE_KEY_TTL = 6.hours - WAL_LOCATION_TTL = 60.seconds - MAX_REDIS_RETRIES = 5 DEFAULT_STRATEGY = :until_executing STRATEGY_NONE = :none - DEDUPLICATED_FLAG_VALUE = 1 - - LUA_SET_WAL_SCRIPT = <<~EOS - local key, wal, offset, ttl = KEYS[1], ARGV[1], tonumber(ARGV[2]), ARGV[3] - local existing_offset = redis.call("LINDEX", key, -1) - if existing_offset == false then - redis.call("RPUSH", key, wal, offset) - redis.call("EXPIRE", key, ttl) - elseif offset > tonumber(existing_offset) then - redis.call("LSET", key, 0, wal) - redis.call("LSET", key, -1, offset) - end - EOS attr_reader :existing_jid @@ -60,66 +46,76 @@ module Gitlab # This method will return the jid that was set in redis def check!(expiry = duplicate_key_ttl) - read_jid = nil - read_wal_locations = {} - - with_redis do |redis| - redis.multi do |multi| - multi.set(idempotency_key, jid, ex: expiry, nx: true) - read_wal_locations = check_existing_wal_locations!(multi, expiry) - read_jid = multi.get(idempotency_key) - end + my_cookie = { + 'jid' => jid, + 'offsets' => {}, + 'wal_locations' => {}, + 'existing_wal_locations' => job_wal_locations + } + + # There are 3 possible scenarios. In order of decreasing likelyhood: + # 1. SET NX succeeds. + # 2. SET NX fails, GET succeeds. + # 3. SET NX fails, the key expires and GET fails. In this case we must retry. + actual_cookie = {} + while actual_cookie.empty? + set_succeeded = with_redis { |r| r.set(cookie_key, my_cookie.to_msgpack, nx: true, ex: expiry) } + actual_cookie = set_succeeded ? my_cookie : get_cookie end job['idempotency_key'] = idempotency_key - # We need to fetch values since the read_wal_locations and read_jid were obtained inside transaction, under redis.multi command. - self.existing_wal_locations = read_wal_locations.transform_values(&:value) - self.existing_jid = read_jid.value + self.existing_wal_locations = actual_cookie['existing_wal_locations'] + self.existing_jid = actual_cookie['jid'] end def update_latest_wal_location! return unless job_wal_locations.present? - with_redis do |redis| - redis.multi do |multi| - job_wal_locations.each do |connection_name, location| - multi.eval( - LUA_SET_WAL_SCRIPT, - keys: [wal_location_key(connection_name)], - argv: [location, pg_wal_lsn_diff(connection_name).to_i, WAL_LOCATION_TTL] - ) - end - end + argv = [] + job_wal_locations.each do |connection_name, location| + argv += [connection_name, pg_wal_lsn_diff(connection_name), location] end + + with_redis { |r| r.eval(UPDATE_WAL_COOKIE_SCRIPT, keys: [cookie_key], argv: argv) } end + # Generally speaking, updating a Redis key by deserializing and + # serializing it on the Redis server is bad for performance. However in + # the case of DuplicateJobs we know that key updates are rare, and the + # most common operations are setting, getting and deleting the key. The + # aim of this design is to make the common operations as fast as + # possible. + UPDATE_WAL_COOKIE_SCRIPT = <<~LUA + local cookie_msgpack = redis.call("get", KEYS[1]) + if not cookie_msgpack then + return + end + local cookie = cmsgpack.unpack(cookie_msgpack) + + for i = 1, #ARGV, 3 do + local connection = ARGV[i] + local current_offset = cookie.offsets[connection] + local new_offset = tonumber(ARGV[i+1]) + if not current_offset or current_offset < new_offset then + cookie.offsets[connection] = new_offset + cookie.wal_locations[connection] = ARGV[i+2] + end + end + + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1])) + LUA + def latest_wal_locations return {} unless job_wal_locations.present? strong_memoize(:latest_wal_locations) do - read_wal_locations = {} - - with_redis do |redis| - redis.multi do |multi| - job_wal_locations.keys.each do |connection_name| - read_wal_locations[connection_name] = multi.lindex(wal_location_key(connection_name), 0) - end - end - end - read_wal_locations.transform_values(&:value).compact + get_cookie.fetch('wal_locations', {}) end end def delete! - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - with_redis do |redis| - redis.multi do |multi| - multi.del(idempotency_key, deduplicated_flag_key) - delete_wal_locations!(multi) - end - end - end + with_redis { |redis| redis.del(cookie_key) } end def reschedule @@ -141,17 +137,21 @@ module Gitlab def set_deduplicated_flag!(expiry = duplicate_key_ttl) return unless reschedulable? - with_redis do |redis| - redis.set(deduplicated_flag_key, DEDUPLICATED_FLAG_VALUE, ex: expiry, nx: true) - end + with_redis { |redis| redis.eval(DEDUPLICATED_SCRIPT, keys: [cookie_key]) } end - def should_reschedule? - return false unless reschedulable? - - with_redis do |redis| - redis.get(deduplicated_flag_key).present? + DEDUPLICATED_SCRIPT = <<~LUA + local cookie_msgpack = redis.call("get", KEYS[1]) + if not cookie_msgpack then + return end + local cookie = cmsgpack.unpack(cookie_msgpack) + cookie.deduplicated = "1" + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1])) + LUA + + def should_reschedule? + reschedulable? && get_cookie['deduplicated'].present? end def scheduled_at @@ -186,31 +186,12 @@ module Gitlab @worker_klass ||= worker_class_name.to_s.safe_constantize end - def delete_wal_locations!(redis) - job_wal_locations.keys.each do |connection_name| - redis.del(wal_location_key(connection_name)) - redis.del(existing_wal_location_key(connection_name)) - end - end - - def check_existing_wal_locations!(redis, expiry) - read_wal_locations = {} - - job_wal_locations.each do |connection_name, location| - key = existing_wal_location_key(connection_name) - redis.set(key, location, ex: expiry, nx: true) - read_wal_locations[connection_name] = redis.get(key) - end - - read_wal_locations - end - def job_wal_locations job['wal_locations'] || {} end def pg_wal_lsn_diff(connection_name) - model = Gitlab::Database.database_base_models[connection_name] + model = Gitlab::Database.database_base_models[connection_name.to_sym] model.connection.load_balancer.wal_diff( job_wal_locations[connection_name], @@ -238,22 +219,18 @@ module Gitlab job['jid'] end - def existing_wal_location_key(connection_name) - "#{idempotency_key}:#{connection_name}:existing_wal_location" + def cookie_key + "#{idempotency_key}:cookie:v2" end - def wal_location_key(connection_name) - "#{idempotency_key}:#{connection_name}:wal_location" + def get_cookie + with_redis { |redis| MessagePack.unpack(redis.get(cookie_key) || "\x80") } end def idempotency_key @idempotency_key ||= job['idempotency_key'] || "#{namespace}:#{idempotency_hash}" end - def deduplicated_flag_key - "#{idempotency_key}:deduplicate_flag" - end - def idempotency_hash Digest::SHA256.hexdigest(idempotency_string) end diff --git a/lib/gitlab/sidekiq_middleware/retry_error.rb b/lib/gitlab/sidekiq_middleware/retry_error.rb new file mode 100644 index 00000000000..372213a8e6a --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/retry_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + # Sidekiq retry error that won't be reported to Sentry + # Use it when a job retry is an expected behavior + RetryError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 3dd5355d3a3..e36f61be3b3 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -43,7 +43,7 @@ module Gitlab def initialize_process_metrics metrics = self.metrics - metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + metrics[:sidekiq_concurrency].set({}, Sidekiq[:concurrency].to_i) return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize) diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb b/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb index bce295d8ba5..f7e0553e536 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb @@ -33,7 +33,7 @@ module Gitlab validate_args!(job) job.except!(ORIGINAL_SIZE_KEY, COMPRESSED_KEY) - job['args'] = Sidekiq.load_json(Zlib::Inflate.inflate(Base64.strict_decode64(job['args'].first))) + job['args'] = Gitlab::Json.load(Zlib::Inflate.inflate(Base64.strict_decode64(job['args'].first))) rescue Zlib::Error raise PayloadDecompressionError, 'Fail to decompress Sidekiq job payload' end diff --git a/lib/gitlab/sidekiq_migrate_jobs.rb b/lib/gitlab/sidekiq_migrate_jobs.rb index 62d62bf82c4..2467dd7ca43 100644 --- a/lib/gitlab/sidekiq_migrate_jobs.rb +++ b/lib/gitlab/sidekiq_migrate_jobs.rb @@ -3,16 +3,18 @@ module Gitlab class SidekiqMigrateJobs LOG_FREQUENCY = 1_000 + LOG_FREQUENCY_QUEUES = 10 - attr_reader :sidekiq_set, :logger + attr_reader :logger, :mappings - def initialize(sidekiq_set, logger: nil) - @sidekiq_set = sidekiq_set + # mappings is a hash of WorkerClassName => target_queue_name + def initialize(mappings, logger: nil) + @mappings = mappings @logger = logger end - # mappings is a hash of WorkerClassName => target_queue_name - def execute(mappings) + # Migrate jobs in SortedSets, i.e. scheduled and retry sets. + def migrate_set(sidekiq_set) source_queues_regex = Regexp.union(mappings.keys) cursor = 0 scanned = 0 @@ -33,7 +35,7 @@ module Gitlab next unless job.match?(source_queues_regex) - job_hash = Sidekiq.load_json(job) + job_hash = Gitlab::Json.load(job) destination_queue = mappings[job_hash['class']] next unless mappings.has_key?(job_hash['class']) @@ -41,7 +43,7 @@ module Gitlab job_hash['queue'] = destination_queue - migrated += migrate_job(job, score, job_hash) + migrated += migrate_job_in_set(sidekiq_set, job, score, job_hash) end end while cursor.to_i != 0 @@ -53,14 +55,54 @@ module Gitlab } end + # Migrates jobs from queues that are outside the mappings + def migrate_queues + routing_rules_queues = mappings.values.uniq + logger&.info("List of queues based on routing rules: #{routing_rules_queues}") + Sidekiq.redis do |conn| + # Redis 6 supports conn.scan_each(match: "queue:*", type: 'list') + conn.scan_each(match: "queue:*") do |key| + # Redis 5 compatibility + next unless conn.type(key) == 'list' + + queue_from = key.split(':', 2).last + next if routing_rules_queues.include?(queue_from) + + logger&.info("Migrating #{queue_from} queue") + + migrated = 0 + while queue_length(queue_from) > 0 + begin + if migrated >= 0 && migrated % LOG_FREQUENCY_QUEUES == 0 + logger&.info("Migrating from #{queue_from}. Total: #{queue_length(queue_from)}. Migrated: #{migrated}.") + end + + job = conn.rpop "queue:#{queue_from}" + job_hash = Gitlab::Json.load(job) + next unless mappings.has_key?(job_hash['class']) + + destination_queue = mappings[job_hash['class']] + job_hash['queue'] = destination_queue + conn.lpush("queue:#{destination_queue}", Gitlab::Json.dump(job_hash)) + migrated += 1 + rescue JSON::ParserError + logger&.error("Unmarshal JSON payload from SidekiqMigrateJobs failed. Job: #{job}") + next + end + end + logger&.info("Finished migrating #{queue_from} queue") + end + end + end + private - def migrate_job(job, score, job_hash) + def migrate_job_in_set(sidekiq_set, job, score, job_hash) Sidekiq.redis do |connection| removed = connection.zrem(sidekiq_set, job) if removed - connection.zadd(sidekiq_set, score, Sidekiq.dump_json(job_hash)) + connection.zadd(sidekiq_set, score, Gitlab::Json.dump(job_hash)) 1 else @@ -68,5 +110,11 @@ module Gitlab end end end + + def queue_length(queue_name) + Sidekiq.redis do |conn| + conn.llen("queue:#{queue_name}") + end + end end end diff --git a/lib/gitlab/slash_commands/application_help.rb b/lib/gitlab/slash_commands/application_help.rb index 1a92346be15..bfdb65a816d 100644 --- a/lib/gitlab/slash_commands/application_help.rb +++ b/lib/gitlab/slash_commands/application_help.rb @@ -3,14 +3,9 @@ module Gitlab module SlashCommands class ApplicationHelp < BaseCommand - def initialize(project, params) - @project = project - @params = params - end - def execute Gitlab::SlashCommands::Presenters::Help - .new(project, commands) + .new(project, commands, params) .present(trigger, params[:text]) end @@ -21,7 +16,11 @@ module Gitlab end def commands - Gitlab::SlashCommands::Command.commands + Gitlab::SlashCommands::Command.new( + project, + chat_name, + params + ).commands end end end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 239479f99d2..265eda46489 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -3,8 +3,8 @@ module Gitlab module SlashCommands class Command < BaseCommand - def self.commands - [ + def commands + commands = [ Gitlab::SlashCommands::IssueShow, Gitlab::SlashCommands::IssueNew, Gitlab::SlashCommands::IssueSearch, @@ -14,6 +14,12 @@ module Gitlab Gitlab::SlashCommands::Deploy, Gitlab::SlashCommands::Run ] + + if Feature.enabled?(:incident_declare_slash_command, current_user) + commands << Gitlab::SlashCommands::IncidentManagement::IncidentNew + end + + commands end def execute @@ -44,7 +50,7 @@ module Gitlab private def available_commands - self.class.commands.keep_if do |klass| + commands.keep_if do |klass| klass.available?(project) end end diff --git a/lib/gitlab/slash_commands/incident_management/incident_command.rb b/lib/gitlab/slash_commands/incident_management/incident_command.rb new file mode 100644 index 00000000000..3fa08621777 --- /dev/null +++ b/lib/gitlab/slash_commands/incident_management/incident_command.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module IncidentManagement + class IncidentCommand < BaseCommand + def self.available?(project) + true + end + + def collection + IssuesFinder.new(current_user, project_id: project.id, issue_types: :incident).execute + end + end + end + end +end + +Gitlab::SlashCommands::IncidentManagement::IncidentCommand.prepend_mod diff --git a/lib/gitlab/slash_commands/incident_management/incident_new.rb b/lib/gitlab/slash_commands/incident_management/incident_new.rb new file mode 100644 index 00000000000..722fcff151d --- /dev/null +++ b/lib/gitlab/slash_commands/incident_management/incident_new.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module IncidentManagement + class IncidentNew < IncidentCommand + def self.help_message + 'incident declare' + end + + def self.allowed?(project, user) + Feature.enabled?(:incident_declare_slash_command, user) && can?(user, :create_incident, project) + end + + def self.match(text) + text == 'incident declare' + end + + private + + def presenter + Gitlab::SlashCommands::Presenters::IncidentManagement::IncidentNew.new + end + end + end + end +end + +Gitlab::SlashCommands::IncidentManagement::IncidentNew.prepend_mod diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 71bc0dc0123..61b36308d20 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -4,9 +4,10 @@ module Gitlab module SlashCommands module Presenters class Help < Presenters::Base - def initialize(project, commands) + def initialize(project, commands, params = {}) @project = project @commands = commands + @params = params end def present(trigger, text) @@ -66,7 +67,13 @@ module Gitlab def full_commands_message(trigger) list = @commands - .map { |command| "#{trigger} #{command.help_message}" } + .map do |command| + if command < Gitlab::SlashCommands::IncidentManagement::IncidentCommand + "#{@params[:command]} #{command.help_message}" + else + "#{trigger} #{command.help_message}" + end + end .join("\n") <<~MESSAGE diff --git a/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb b/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb new file mode 100644 index 00000000000..5030c8282db --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module Presenters + module IncidentManagement + class IncidentNew < Presenters::Base + def present(message) + ephemeral_response(text: message) + end + end + end + end + end +end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index ca7ae429986..d13ccde8576 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -6,7 +6,7 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/.freeze + REGEX_QUOTED_TERM = /(?<=\A| )"[^"]+"(?= |\z)/.freeze class_methods do def fuzzy_search(query, columns, use_minimum_char_limit: true) @@ -40,12 +40,14 @@ module Gitlab # lower_exact_match - When set to `true` we'll fall back to using # `LOWER(column) = query` instead of using `ILIKE`. def fuzzy_arel_match(column, query, lower_exact_match: false, use_minimum_char_limit: true) + return unless query.is_a?(String) + query = query.squish return unless query.present? arel_column = column.is_a?(Arel::Attributes::Attribute) ? column : arel_table[column] - words = select_fuzzy_words(query, use_minimum_char_limit: use_minimum_char_limit) + words = select_fuzzy_terms(query, use_minimum_char_limit: use_minimum_char_limit) if words.any? words.map { |word| arel_column.matches(to_pattern(word, use_minimum_char_limit: use_minimum_char_limit)) }.reduce(:and) @@ -62,19 +64,21 @@ module Gitlab end end - def select_fuzzy_words(query, use_minimum_char_limit: true) - quoted_words = query.scan(REGEX_QUOTED_WORD) - - query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } - - words = query.split - - quoted_words.map! { |quoted_word| quoted_word[1..-2] } + def select_fuzzy_terms(query, use_minimum_char_limit: true) + terms = Gitlab::SQL::Pattern.split_query_to_search_terms(query) + terms.select { |term| partial_matching?(term, use_minimum_char_limit: use_minimum_char_limit) } + end + end - words.concat(quoted_words) + def self.split_query_to_search_terms(query) + quoted_terms = [] - words.select { |word| partial_matching?(word, use_minimum_char_limit: use_minimum_char_limit) } + query = query.gsub(REGEX_QUOTED_TERM) do |quoted_term| + quoted_terms << quoted_term + "" end + + query.split + quoted_terms.map { |quoted_term| quoted_term[1..-2] } end end end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 31e11f73fe7..ededc3db18e 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -47,6 +47,8 @@ module Gitlab { key: key, name: name, content: content } end + alias_method :as_json, :to_json + def <=>(other) name <=> other.name end diff --git a/lib/gitlab/tracking/helpers/weak_password_error_event.rb b/lib/gitlab/tracking/helpers/weak_password_error_event.rb new file mode 100644 index 00000000000..beb6119e3f7 --- /dev/null +++ b/lib/gitlab/tracking/helpers/weak_password_error_event.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Helpers + module WeakPasswordErrorEvent + # Tracks information if a user record has a weak password. + # No-op unless the error is present. + # + # Captures a minimal set of information, so that GitLab + # remains unaware of which users / demographics are attempting + # to choose weak passwords. + def track_weak_password_error(user, controller, method_name) + return unless user.errors[:password].grep(/must not contain commonly used combinations.*/).any? + + Gitlab::Tracking.event( + 'Gitlab::Tracking::Helpers::WeakPasswordErrorEvent', + 'track_weak_password_error', + controller: controller, + method: method_name + ) + end + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index a6d6cffec17..e203fb486e7 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -52,6 +52,8 @@ module Gitlab wiki_page_url(object.wiki, object, **options) when ::DesignManagement::Design design_url(object, **options) + when ::Packages::Package + package_url(object, **options) else raise NotImplementedError, "No URL builder defined for #{object.inspect}" end @@ -133,6 +135,17 @@ module Gitlab instance.project_design_management_designs_raw_image_url(design.project, design, ref, **options) end end + + def package_url(package, **options) + project = package.project + + if package.infrastructure_package? + return instance.project_infrastructure_registry_url(project, package, +**options) + end + + instance.project_package_url(project, package, **options) + end end end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index d6b1e62c84f..065ede75c60 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -4,9 +4,9 @@ module Gitlab module Usage class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') - SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze - AVAILABLE_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze - VALID_SERVICE_PING_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze + SKIP_VALIDATION_STATUS = 'removed' + AVAILABLE_STATUSES = %w[active broken].to_set.freeze + VALID_SERVICE_PING_STATUSES = %w[active broken].to_set.freeze InvalidError = Class.new(RuntimeError) @@ -144,7 +144,7 @@ module Gitlab end def skip_validation? - !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status]) + !!attributes[:skip_validation] || @skip_validation || attributes[:status] == SKIP_VALIDATION_STATUS end end end diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb index a32c413dba8..02d9fa74289 100644 --- a/lib/gitlab/usage/metrics/aggregates.rb +++ b/lib/gitlab/usage/metrics/aggregates.rb @@ -7,14 +7,13 @@ module Gitlab UNION_OF_AGGREGATED_METRICS = 'OR' INTERSECTION_OF_AGGREGATED_METRICS = 'AND' ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze - AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml') AggregatedMetricError = Class.new(StandardError) UnknownAggregationOperator = Class.new(AggregatedMetricError) UnknownAggregationSource = Class.new(AggregatedMetricError) DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError) DATABASE_SOURCE = 'database' - REDIS_SOURCE = 'redis' + REDIS_SOURCE = 'redis_hll' SOURCES = { DATABASE_SOURCE => Sources::PostgresHll, diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index cd72f16d46d..78f1ddc8a29 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -8,22 +8,9 @@ module Gitlab include Gitlab::Usage::TimeFrame def initialize(recorded_at) - @aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH) @recorded_at = recorded_at end - def all_time_data - aggregated_metrics_data(Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME) - end - - def monthly_data - aggregated_metrics_data(Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME) - end - - def weekly_data - aggregated_metrics_data(Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME) - end - def calculate_count_for_aggregation(aggregation:, time_frame:) with_validate_configuration(aggregation, time_frame) do source = SOURCES[aggregation[:source]] @@ -40,16 +27,7 @@ module Gitlab private - attr_accessor :aggregated_metrics, :recorded_at - - def aggregated_metrics_data(time_frame) - aggregated_metrics.each_with_object({}) do |aggregation, data| - next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], type: :development) - next unless aggregation[:time_frame].include?(time_frame) - - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, time_frame: time_frame) - end - end + attr_accessor :recorded_at def with_validate_configuration(aggregation, time_frame) source = aggregation[:source] @@ -83,16 +61,6 @@ module Gitlab Gitlab::Utils::UsageData::FALLBACK end - def load_metrics(wildcard) - Dir[wildcard].each_with_object([]) do |path, metrics| - metrics.push(*load_yaml_from_path(path)) - end - end - - def load_yaml_from_path(path) - YAML.safe_load(File.read(path), aliases: true)&.map(&:with_indifferent_access) - end - def time_constraints(time_frame) case time_frame when Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME @@ -108,5 +76,3 @@ module Gitlab end end end - -Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_mod_with('Gitlab::Usage::Metrics::Aggregates::Aggregate') diff --git a/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb index 63ead5a8cb0..66be7a7b64e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb @@ -25,7 +25,7 @@ module Gitlab def initialize(metric_definition) super - @source = parse_data_source_to_legacy_value(metric_definition) + @source = metric_definition[:data_source] @aggregate = options.fetch(:aggregate, {}) end @@ -48,15 +48,6 @@ module Gitlab attr_accessor :source, :aggregate - # TODO: This method is a temporary measure that - # handles backwards compatibility until - # point 5 from is resolved https://gitlab.com/gitlab-org/gitlab/-/issues/370963#implementation - def parse_data_source_to_legacy_value(metric_definition) - return 'redis' if metric_definition[:data_source] == 'redis_hll' - - metric_definition[:data_source] - end - def aggregate_config { source: source, diff --git a/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb new file mode 100644 index 00000000000..a7f8bca8e08 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountMergeRequestAuthorsMetric < DatabaseMetric + operation :distinct_count, column: :author_id + + relation { MergeRequest } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 6dec0349a38..f0d5298870c 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -34,10 +34,10 @@ module Gitlab @metric_finish = block end - def relation(&block) - return @metric_relation&.call unless block + def relation(relation_proc = nil, &block) + return unless relation_proc || block - @metric_relation = block + @metric_relation = (relation_proc || block) end def metric_options(&block) @@ -106,7 +106,11 @@ module Gitlab end def relation - self.class.metric_relation.call.where(time_constraints) + if self.class.metric_relation.arity == 1 + self.class.metric_relation.call(options) + else + self.class.metric_relation.call + end.where(time_constraints) end def time_constraints diff --git a/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_metric.rb index 0c421dc3311..c7cf6c57059 100644 --- a/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_metric.rb @@ -4,7 +4,7 @@ module Gitlab module Usage module Metrics module Instrumentations - class DistinctCountProjectsWithExpirationPolicyDisabledMetric < DatabaseMetric + class DistinctCountProjectsWithExpirationPolicyMetric < DatabaseMetric operation :distinct_count, column: :project_id start { Project.minimum(:id) } @@ -12,7 +12,11 @@ module Gitlab cache_start_and_finish_as :project_id - relation { ::ContainerExpirationPolicy.where(enabled: false) } + relation ->(options) do + options.each_with_object(::ContainerExpirationPolicy.all) do |(key, value), ar_relation| + ar_relation.where!(key => value) + end + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/dormant_user_period_setting_metric.rb b/lib/gitlab/usage/metrics/instrumentations/dormant_user_period_setting_metric.rb new file mode 100644 index 00000000000..c05664aa9c8 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/dormant_user_period_setting_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DormantUserPeriodSettingMetric < GenericMetric + value do + ::Gitlab::CurrentSettings.deactivate_dormant_users_period + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/dormant_user_setting_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/dormant_user_setting_enabled_metric.rb new file mode 100644 index 00000000000..82d8570276a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/dormant_user_setting_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DormantUserSettingEnabledMetric < GenericMetric + value do + ::Gitlab::CurrentSettings.deactivate_dormant_users + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb new file mode 100644 index 00000000000..b1a2de29fd7 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class InProductMarketingEmailCtaClickedMetric < DatabaseMetric + operation :count + + def initialize(metric_definition) + super + + unless track.in?(allowed_track) + raise ArgumentError, "track '#{track}' must be one of: #{allowed_track.join(', ')}" + end + + return if series.in?(allowed_series) + + raise ArgumentError, "series '#{series}' must be one of: #{allowed_series.join(', ')}" + end + + relation { Users::InProductMarketingEmail } + + private + + def relation + scope = super.where.not(cta_clicked_at: nil) + scope = scope.where(series: series) + scope.where(track: track) + end + + def track + options[:track] + end + + def series + options[:series] + end + + def allowed_track + Users::InProductMarketingEmail::ACTIVE_TRACKS.keys + end + + def allowed_series + @allowed_series ||= begin + series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) + 0.upto(series_amount - 1).to_a + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb new file mode 100644 index 00000000000..50dec606d9b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class InProductMarketingEmailSentMetric < DatabaseMetric + operation :count + + def initialize(metric_definition) + super + + unless track.in?(allowed_track) + raise ArgumentError, "track '#{track}' must be one of: #{allowed_track.join(', ')}" + end + + return if series.in?(allowed_series) + + raise ArgumentError, "series '#{series}' must be one of: #{allowed_series.join(', ')}" + end + + relation { Users::InProductMarketingEmail } + + private + + def relation + scope = super + scope = scope.where(series: series) + scope.where(track: track) + end + + def track + options[:track] + end + + def series + options[:series] + end + + def allowed_track + Users::InProductMarketingEmail::ACTIVE_TRACKS.keys + end + + def allowed_series + @allowed_series ||= begin + series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) + 0.upto(series_amount - 1).to_a + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/name_suggestion.rb b/lib/gitlab/usage/metrics/name_suggestion.rb index 238a7a51a20..44723b6f3d4 100644 --- a/lib/gitlab/usage/metrics/name_suggestion.rb +++ b/lib/gitlab/usage/metrics/name_suggestion.rb @@ -7,6 +7,7 @@ module Gitlab FREE_TEXT_METRIC_NAME = "<please fill metric name>" REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" + EMPTY_CONSTRAINT = "()" class << self def for(operation, relation: nil, column: nil) @@ -52,7 +53,8 @@ module Gitlab end arel = arel_query(relation: relation, column: arel_column, distinct: distinct) - constraints = parse_constraints(relation: relation, arel: arel) + where_constraints = parse_where_constraints(relation: relation, arel: arel) + having_constraints = parse_having_constraints(relation: relation, arel: arel) # In some cases due to performance reasons metrics are instrumented with joined relations # where relation listed in FROM statement is not the one that includes counted attribute @@ -66,23 +68,35 @@ module Gitlab # count_environment_id_from_clusters_with_deployments actual_source = parse_source(relation, arel_column) - append_constraints_prompt(actual_source, [constraints], parts) + append_constraints_prompt(actual_source, [where_constraints], [having_constraints], parts) parts << actual_source - parts += process_joined_relations(actual_source, arel, relation, constraints) + parts += process_joined_relations(actual_source, arel, relation, where_constraints) parts.compact.join('_').delete('"') end - def append_constraints_prompt(target, constraints, parts) - applicable_constraints = constraints.select { |constraint| constraint.include?(target) } + def append_constraints_prompt(target, where_constraints, having_constraints, parts) + where_constraints.select! do |constraint| + constraint.include?(target) + end + having_constraints.delete(EMPTY_CONSTRAINT) + applicable_constraints = where_constraints + having_constraints return unless applicable_constraints.any? parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } end - def parse_constraints(relation:, arel:) + def parse_where_constraints(relation:, arel:) + connection = relation.connection + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::WhereConstraints + .new(connection) + .accept(arel, collector(connection)) + .value + end + + def parse_having_constraints(relation:, arel:) connection = relation.connection - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::HavingConstraints .new(connection) .accept(arel, collector(connection)) .value @@ -152,7 +166,7 @@ module Gitlab subtree.each do |parent, children| parts << "<#{conjunction}>" join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) - append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) + append_constraints_prompt(parent, [wheres, join_constraints].compact, [], parts) parts << parent collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) end diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb new file mode 100644 index 00000000000..8dd3b1ff5c6 --- /dev/null +++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module NamesSuggestions + module RelationParsers + class HavingConstraints < ::Arel::Visitors::PostgreSQL + # rubocop:disable Naming/MethodName + def visit_Arel_Nodes_SelectCore(object, collector) + collect_nodes_for(object.havings, collector, "") || collector + end + # rubocop:enable Naming/MethodName + + def quote(value) + value.to_s + end + + def quote_table_name(name) + name.to_s + end + + def quote_column_name(name) + name.to_s + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/where_constraints.rb index 199395e4b20..9f829067214 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/where_constraints.rb @@ -5,7 +5,7 @@ module Gitlab module Metrics module NamesSuggestions module RelationParsers - class Constraints < ::Arel::Visitors::PostgreSQL + class WhereConstraints < ::Arel::Visitors::PostgreSQL # rubocop:disable Naming/MethodName def visit_Arel_Nodes_SelectCore(object, collector) collect_nodes_for(object.wheres, collector, "") || collector @@ -13,15 +13,15 @@ module Gitlab # rubocop:enable Naming/MethodName def quote(value) - "#{value}" + value.to_s end def quote_table_name(name) - "#{name}" + name.to_s end def quote_column_name(name) - "#{name}" + name.to_s end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 87ccb9a31da..5021dac453f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -30,7 +30,6 @@ module Gitlab deployment_minimum_id deployment_maximum_id auth_providers - aggregated_metrics recorded_at ).freeze @@ -157,11 +156,9 @@ module Gitlab }.merge( runners_usage, integrations_usage, - usage_counters, user_preferences_usage, container_expiration_policies_usage, - service_desk_counts, - email_campaign_counts + service_desk_counts ).tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end @@ -261,16 +258,6 @@ module Gitlab } end - # @return [Hash<Symbol, Integer>] - def usage_counters - usage_data_counters.map { |counter| redis_usage_data(counter) }.reduce({}, :merge) - end - - # @return [Array<#totals>] An array of objects that respond to `#totals` - def usage_data_counters - Gitlab::UsageDataCounters.unmigrated_counters - end - def components_usage_data { git: { version: alt_usage_data(fallback: { major: -1 }) { Gitlab::Git.version } }, @@ -349,17 +336,13 @@ module Gitlab # rubocop: disable UsageData/LargeTable base = ::ContainerExpirationPolicy.active # rubocop: enable UsageData/LargeTable - results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish) # rubocop: disable UsageData/LargeTable - %i[keep_n cadence older_than].each do |option| - ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend - results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish) - end + ::ContainerExpirationPolicy.older_than_options.keys.each do |value| + results["projects_with_expiration_policy_enabled_with_older_than_set_to_#{value}".to_sym] = distinct_count(base.where(older_than: value), :project_id, start: start, finish: finish) end # rubocop: enable UsageData/LargeTable - results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish) results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish) results @@ -632,21 +615,16 @@ module Gitlab { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end - def aggregated_metrics_data - { - counts_weekly: { aggregated_metrics: aggregated_metrics.weekly_data }, - counts_monthly: { aggregated_metrics: aggregated_metrics.monthly_data }, - counts: aggregated_metrics - .all_time_data - .to_h { |key, value| ["aggregate_#{key}".to_sym, value.round] } - } - end - def action_monthly_active_users(time_period) + counter = Gitlab::UsageDataCounters::EditorUniqueCounter date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last } - event_monthly_active_users(date_range) - .merge!(ide_monthly_active_users(date_range)) + { + action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, + action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, + action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, + action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } + } end def with_duration @@ -688,7 +666,6 @@ module Gitlab .merge(usage_activity_by_stage) .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params)) .merge(redis_hll_counters) - .deep_merge(aggregated_metrics_data) end def metric_time_period(time_period) @@ -705,34 +682,6 @@ module Gitlab end end - def aggregated_metrics - @aggregated_metrics ||= ::Gitlab::Usage::Metrics::Aggregates::Aggregate.new(recorded_at) - end - - def event_monthly_active_users(date_range) - data = { - action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION, - action_monthly_active_users_design_management: Gitlab::UsageDataCounters::TrackUniqueEvents::DESIGN_ACTION, - action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION, - action_monthly_active_users_git_write: Gitlab::UsageDataCounters::TrackUniqueEvents::GIT_WRITE_ACTION - } - - data.each do |key, event| - data[key] = redis_usage_data { Gitlab::UsageDataCounters::TrackUniqueEvents.count_unique_events(event_action: event, **date_range) } - end - end - - def ide_monthly_active_users(date_range) - counter = Gitlab::UsageDataCounters::EditorUniqueCounter - - { - action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, - action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, - action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } - } - end - def distinct_count_service_desk_enabled_projects(time_period) project_creator_id_start = minimum_id(User) project_creator_id_finish = maximum_id(User) @@ -758,37 +707,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord - def email_campaign_counts - # rubocop:disable UsageData/LargeTable - sent_emails = count(Users::InProductMarketingEmail.group(:track, :series)) - clicked_emails = count(Users::InProductMarketingEmail.where.not(cta_clicked_at: nil).group(:track, :series)) - - Users::InProductMarketingEmail::ACTIVE_TRACKS.keys.each_with_object({}) do |track, result| - series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) - # rubocop: enable UsageData/LargeTable: - - 0.upto(series_amount - 1).map do |series| - sent_count = sent_in_product_marketing_email_count(sent_emails, track, series) - clicked_count = clicked_in_product_marketing_email_count(clicked_emails, track, series) - - result["in_product_marketing_email_#{track}_#{series}_sent"] = sent_count - result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count unless track == 'experience' - end - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def sent_in_product_marketing_email_count(sent_emails, track, series) - # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. - sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails - end - - def clicked_in_product_marketing_email_count(clicked_emails, track, series) - # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. - clicked_emails.is_a?(Hash) ? clicked_emails.fetch([track, series], 0) : clicked_emails - end - def total_alert_issues # Remove prometheus table queries once they are deprecated # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index 37c6e1af7c0..c2961de0eb9 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -2,9 +2,7 @@ module Gitlab module UsageDataCounters - COUNTERS = [].freeze - - COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES = [ + COUNTERS = [ PackageEventCounter, MergeRequestCounter, DesignsCounter, @@ -26,12 +24,8 @@ module Gitlab UnknownEvent = Class.new(UsageDataCounterError) class << self - def unmigrated_counters - self::COUNTERS - end - def counters - unmigrated_counters + migrated_counters + COUNTERS end def count(event_name) @@ -43,12 +37,6 @@ module Gitlab raise UnknownEvent, "Cannot find counter for event #{event_name}" end - - private - - def migrated_counters - COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES - end end end end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 1e8918c7c96..eb040e9e819 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -10,13 +10,17 @@ module Gitlab::UsageDataCounters expanded_template_name = expand_template_name(template) return unless expanded_template_name - Gitlab::UsageDataCounters::HLLRedisCounter.track_event( - ci_template_event_name(expanded_template_name, config_source), values: project.id - ) + event_name = ci_template_event_name(expanded_template_name, config_source) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: project.id) namespace = project.namespace if Feature.enabled?(:route_hll_to_snowplow, namespace) - Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, user: user, project: project) + context = Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: event_name).to_context + label = 'redis_hll_counters.ci_templates.ci_templates_total_unique_counts_monthly' + Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, + project: project, context: [context], user: user, + label: label) end end diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index a64d0ff7e24..f7ddc53f50d 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -55,3 +55,5 @@ - i_package_terraform_module_delete_package - i_package_terraform_module_pull_package - i_package_terraform_module_push_package +- i_package_rpm_push_package +- i_package_rpm_pull_package diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index c13c7657576..c1720b26a22 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -127,6 +127,11 @@ category: testing redis_slot: testing aggregation: weekly +- name: i_testing_coverage_report_uploaded + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_ci_i_testing_coverage_report_uploaded # Project Management group - name: g_project_management_issue_title_changed category: issues_edit diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index debdbd8614f..ef8d02fa365 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -79,3 +79,11 @@ category: user_packages aggregation: weekly redis_slot: package +- name: i_package_rpm_user + category: user_packages + aggregation: weekly + redis_slot: package +- name: i_package_rpm_deploy_token + category: deploy_token_packages + aggregation: weekly + redis_slot: package diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml index ee828fc0f72..d088b6d7e5a 100644 --- a/lib/gitlab/usage_data_counters/known_events/work_items.yml +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -19,6 +19,11 @@ redis_slot: users aggregation: weekly feature_flag: track_work_items_activity +- name: users_updating_work_item_milestone + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity - name: users_updating_work_item_iteration # The event tracks an EE feature. # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. @@ -27,3 +32,11 @@ redis_slot: users aggregation: weekly feature_flag: track_work_items_activity +- name: users_updating_weight_estimate + # The event tracks an EE feature. + # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. + # It will report 0 for CE instances and should not be used with 'AND' aggregators. + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index 8b9ca0fc220..d6e05f30a0d 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -8,6 +8,8 @@ module Gitlab class << self def increment_event_counts(events) + return unless events.present? + validate!(events) events.each do |event, incr| diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index a0fd04596fc..b99c9ebb24f 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -7,6 +7,7 @@ module Gitlab WORK_ITEM_TITLE_CHANGED = 'users_updating_work_item_title' WORK_ITEM_DATE_CHANGED = 'users_updating_work_item_dates' WORK_ITEM_LABELS_CHANGED = 'users_updating_work_item_labels' + WORK_ITEM_MILESTONE_CHANGED = 'users_updating_work_item_milestone' class << self def track_work_item_created_action(author:) @@ -25,6 +26,10 @@ module Gitlab track_unique_action(WORK_ITEM_LABELS_CHANGED, author) end + def track_work_item_milestone_changed_action(author:) + track_unique_action(WORK_ITEM_MILESTONE_CHANGED, author) + end + private def track_unique_action(action, author) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index a67a0758257..d3055569ece 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -14,7 +14,10 @@ module Gitlab # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 # It also checks for ALT_SEPARATOR aka '\' (forward slash) def check_path_traversal!(path) - return unless path.is_a?(String) + return unless path + + path = path.to_s if path.is_a?(Gitlab::HashedPath) + raise PathTraversalAttackError, 'Invalid path' unless path.is_a?(String) path = decode_path(path) path_regex = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)} @@ -164,9 +167,10 @@ module Gitlab end def deep_indifferent_access(data) - if data.is_a?(Array) + case data + when Array data.map(&method(:deep_indifferent_access)) - elsif data.is_a?(Hash) + when Hash data.with_indifferent_access else data @@ -174,9 +178,10 @@ module Gitlab end def deep_symbolized_access(data) - if data.is_a?(Array) + case data + when Array data.map(&method(:deep_symbolized_access)) - elsif data.is_a?(Hash) + when Hash data.deep_symbolize_keys else data diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb index dc43d977a62..cfa09804b98 100644 --- a/lib/gitlab/utils/measuring.rb +++ b/lib/gitlab/utils/measuring.rb @@ -31,7 +31,7 @@ module Gitlab gc_stats: gc_stats, time_to_finish: time_to_finish, number_of_sql_calls: sql_calls_count, - memory_usage: "#{Gitlab::Metrics::System.memory_usage_rss.to_f / 1024 / 1024} MiB", + memory_usage: "#{Gitlab::Metrics::System.memory_usage_rss[:total].to_f / 1024 / 1024} MiB", label: ::Prometheus::PidProvider.worker_id ) diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 50b8428113d..6456ad08924 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -30,10 +30,10 @@ module Gitlab # end # strong_memoize_attr :trigger_from_token # - # strong_memoize_attr :enabled?, :enabled # def enabled? # Feature.enabled?(:some_feature) # end + # strong_memoize_attr :enabled?, :enabled # def strong_memoize(name) key = ivar(name) @@ -45,6 +45,16 @@ module Gitlab end end + def strong_memoize_with(name, *args) + container = strong_memoize(name) { {} } + + if container.key?(args) + container[args] + else + container[args] = yield + end + end + def strong_memoized?(name) instance_variable_defined?(ivar(name)) end @@ -58,23 +68,8 @@ module Gitlab def strong_memoize_attr(method_name, member_name = nil) member_name ||= method_name - if method_defined?(method_name) || private_method_defined?(method_name) - StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend - :do_strong_memoize, self, method_name, member_name) - else - StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend - :queue_strong_memoize, self, method_name, member_name) - end - end - - def method_added(method_name) - super - - if member_name = StrongMemoize - .send(:strong_memoize_queue, self).delete(method_name) # rubocop:disable GitlabSecurity/PublicSend - StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend - :do_strong_memoize, self, method_name, member_name) - end + StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend + :do_strong_memoize, self, method_name, member_name) end end @@ -88,9 +83,10 @@ module Gitlab # # Depending on a type ensure that there's a single memory allocation def ivar(name) - if name.is_a?(Symbol) + case name + when Symbol name.to_s.prepend("@").to_sym - elsif name.is_a?(String) + when String :"@#{name}" else raise ArgumentError, "Invalid type of '#{name}'" @@ -100,14 +96,6 @@ module Gitlab class <<self private - def strong_memoize_queue(klass) - klass.instance_variable_get(:@strong_memoize_queue) || klass.instance_variable_set(:@strong_memoize_queue, {}) - end - - def queue_strong_memoize(klass, method_name, member_name) - strong_memoize_queue(klass)[method_name] = member_name - end - def do_strong_memoize(klass, method_name, member_name) method = klass.instance_method(method_name) diff --git a/lib/gitlab/web_hooks/recursion_detection.rb b/lib/gitlab/web_hooks/recursion_detection.rb index 031d9ec6ec4..7e79283757f 100644 --- a/lib/gitlab/web_hooks/recursion_detection.rb +++ b/lib/gitlab/web_hooks/recursion_detection.rb @@ -41,7 +41,7 @@ module Gitlab ::Gitlab::Redis::SharedState.with do |redis| redis.multi do |multi| - multi.sadd(cache_key, hook.id) + multi.sadd?(cache_key, hook.id) multi.expire(cache_key, TOUCH_CACHE_TTL) end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 906439d5e71..0d5daeefe90 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -33,7 +33,12 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags( + user: ::Feature::Gitaly.user_actor(user), + repository: repository, + project: ::Feature::Gitaly.project_actor(repository.container), + group: ::Feature::Gitaly.group_actor(repository.container) + ) } } @@ -252,7 +257,12 @@ module Gitlab { address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags( + user: ::Feature::Gitaly.user_actor, + repository: repository, + project: ::Feature::Gitaly.project_actor(repository.container), + group: ::Feature::Gitaly.group_actor(repository.container) + ) } end diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index d092cd56e46..9449e51b053 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -66,6 +66,8 @@ module ObjectStorage workhorse_aws_hash elsif config.azure? workhorse_azure_hash + elsif Feature.enabled?(:workhorse_google_client) && config.google? + workhorse_google_hash else {} end @@ -111,6 +113,23 @@ module ObjectStorage url end + def workhorse_google_hash + { + UseWorkhorseClient: use_workhorse_google_client?, + RemoteTempObjectID: object_name, + ObjectStorage: { + Provider: 'Google', + GoCloudConfig: { + URL: google_gocloud_url + } + } + } + end + + def google_gocloud_url + "gs://#{bucket_name}" + end + def use_workhorse_s3_client? return false unless config.use_iam_profile? || config.consolidated_settings? # The Golang AWS SDK does not support V2 signatures @@ -119,6 +138,15 @@ module ObjectStorage true end + def use_workhorse_google_client? + return false unless config.consolidated_settings? + return true if credentials[:google_application_default] + return true if credentials[:google_json_key_location] + return true if credentials[:google_json_key_string] + + false + end + def provider credentials[:provider].to_s end diff --git a/lib/product_analytics/collector_app.rb b/lib/product_analytics/collector_app.rb deleted file mode 100644 index 1008d2f264c..00000000000 --- a/lib/product_analytics/collector_app.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module ProductAnalytics - class CollectorApp - def call(env) - request = Rack::Request.new(env) - params = request.params - - return not_found unless EventParams.has_required_params?(params) - - # Product analytics feature is behind a flag and is disabled by default. - # We expect limited amount of projects with this feature enabled in first release. - # Since collector has no authentication we temporary prevent recording of events - # for project without the feature enabled. During increase of feature adoption, this - # check will be removed for better performance. - project = Project.find(params['aid'].to_i) - return not_found unless Feature.enabled?(:product_analytics, project) - - # Snowplow tracker has own format of events. - # We need to convert them to match the schema of our database. - event_params = EventParams.parse_event_params(params) - - if ProductAnalyticsEvent.create(event_params) - ok - else - not_found - end - rescue ActiveRecord::InvalidForeignKey, ActiveRecord::RecordNotFound - not_found - end - - def ok - [200, {}, []] - end - - def not_found - [404, {}, []] - end - end -end diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 9e76225fc54..436739bed12 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -12,6 +12,8 @@ module Rouge def initialize(options = {}) @tag = options[:tag] @line_number = options[:line_number] || 1 + @ellipsis_indexes = options[:ellipsis_indexes] || [] + @ellipsis_svg = options[:ellipsis_svg] end def stream(tokens) @@ -26,6 +28,8 @@ module Rouge yield highlight_unicode_control_characters(span(token, value.chomp! || value)) end + yield ellipsis if @ellipsis_indexes.include?(@line_number - 1) && @ellipsis_svg.present? + yield %(</span>) @line_number += 1 @@ -34,6 +38,10 @@ module Rouge private + def ellipsis + %(<span class="gl-px-2 gl-rounded-base gl-mx-2 gl-bg-gray-100 gl-cursor-help has-tooltip" title="Content has been trimmed">#{@ellipsis_svg}</span>) + end + def highlight_unicode_control_characters(text) text.gsub(Gitlab::Unicode::BIDI_REGEXP) do |char| %(<span class="unicode-bidi has-tooltip" data-toggle="tooltip" title="#{Gitlab::Unicode.bidi_warning}">#{char}</span>) diff --git a/lib/sbom/package_url.rb b/lib/sbom/package_url.rb new file mode 100644 index 00000000000..d8f4e876b82 --- /dev/null +++ b/lib/sbom/package_url.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +# MIT License +# +# Copyright (c) 2021 package-url +# Portions Copyright 2022 Gitlab B.V. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +module Sbom + # A package URL, or _purl_, is a URL string used to + # identify and locate a software package in a mostly universal and uniform way + # across programing languages, package managers, packaging conventions, tools, + # APIs and databases. + # + # A purl is a URL composed of seven components: + # + # ``` + # scheme:type/namespace/name@version?qualifiers#subpath + # ``` + # + # For example, + # the package URL for this Ruby package at version 0.1.0 is + # `pkg:ruby/mattt/packageurl-ruby@0.1.0`. + # + # More details on the package URL format can be found in the purl specification: + # https://github.com/package-url/purl-spec/blob/0b1559f76b79829e789c4f20e6d832c7314762c5/PURL-SPECIFICATION.rst + class PackageUrl + # Raised when attempting to parse an invalid package URL string. + # @see #parse + InvalidPackageUrl = Class.new(ArgumentError) + + # The URL scheme, which has a constant value of `"pkg"`. + def scheme + 'pkg' + end + + # The package type or protocol, such as `"gem"`, `"npm"`, and `"github"`. + attr_reader :type + + # A name prefix, specific to the type of package. + # For example, an npm scope, a Docker image owner, or a GitHub user. + attr_reader :namespace + + # The name of the package. + attr_reader :name + + # The version of the package. + attr_reader :version + + # Extra qualifying data for a package, specific to the type of package. + # For example, the operating system or architecture. + attr_reader :qualifiers + + # An extra subpath within a package, relative to the package root. + attr_reader :subpath + + # Constructs a package URL from its components + # @param type [String] The package type or protocol. + # @param namespace [String] A name prefix, specific to the type of package. + # @param name [String] The name of the package. + # @param version [String] The version of the package. + # @param qualifiers [Hash] Extra qualifying data for a package, specific to the type of package. + # @param subpath [String] An extra subpath within a package, relative to the package root. + def initialize(type:, name:, namespace: nil, version: nil, qualifiers: nil, subpath: nil) + @type = type&.downcase + @namespace = namespace + @name = name + @version = version + @qualifiers = qualifiers + @subpath = subpath + + ArgumentValidator.new(self).validate! + end + + # Creates a new PackageUrl from a string. + # @param [String] string The package URL string. + # @raise [InvalidPackageUrl] If the string is not a valid package URL. + # @return [PackageUrl] + def self.parse(string) + Decoder.new(string).decode! + end + + # Returns a hash containing the + # scheme, type, namespace, name, version, qualifiers, and subpath components + # of the package URL. + def to_h + { + scheme: scheme, + type: @type, + namespace: @namespace, + name: @name, + version: @version, + qualifiers: @qualifiers, + subpath: @subpath + } + end + + # Returns a string representation of the package URL. + # Package URL representations are created according to the instructions from + # https://github.com/package-url/purl-spec/blob/0b1559f76b79829e789c4f20e6d832c7314762c5/PURL-SPECIFICATION.rst#how-to-build-purl-string-from-its-components. + def to_s + Encoder.new(self).encode + end + end +end diff --git a/lib/sbom/package_url/argument_validator.rb b/lib/sbom/package_url/argument_validator.rb new file mode 100644 index 00000000000..639ee9f89b6 --- /dev/null +++ b/lib/sbom/package_url/argument_validator.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Sbom + class PackageUrl + class ArgumentValidator + QUALIFIER_KEY_REGEXP = /^[A-Za-z\d._-]+$/.freeze + START_WITH_NUMBER_REGEXP = /^\d/.freeze + + def initialize(package) + @type = package.type + @namespace = package.namespace + @name = package.name + @version = package.version + @qualifiers = package.qualifiers + @errors = [] + end + + def validate! + validate_type + validate_name + validate_qualifiers + validate_by_type + + raise ArgumentError, formatted_errors if invalid? + end + + private + + def invalid? + errors.present? + end + + attr_reader :type, :namespace, :name, :version, :qualifiers, :errors + + def formatted_errors + errors.join(', ') + end + + def validate_type + errors.push('Type is required') if type.blank? + end + + def validate_name + errors.push('Name is required') if name.blank? + end + + def validate_qualifiers + return if qualifiers.nil? + + keys = qualifiers.keys + errors.push('Qualifier keys must be unique') unless keys.uniq.size == keys.size + + keys.each do |key| + errors.push(key_error(key, 'contains illegal characters')) unless key.match?(QUALIFIER_KEY_REGEXP) + errors.push(key_error(key, 'may not start with a number')) if key.match?(START_WITH_NUMBER_REGEXP) + end + end + + def key_error(key, text) + "Qualifier key `#{key}` #{text}" + end + + def validate_by_type + case type + when 'conan' + validate_conan + when 'cran' + validate_cran + when 'swift' + validate_swift + end + end + + def validate_conan + return unless namespace.blank? ^ (qualifiers.nil? || qualifiers.exclude?('channel')) + + errors.push('Conan packages require the channel be present if published in a namespace and vice-versa') + end + + def validate_cran + errors.push('Cran packages require a version') if version.blank? + end + + def validate_swift + errors.push('Swift packages require a namespace') if namespace.blank? + errors.push('Swift packages require a version') if version.blank? + end + end + end +end diff --git a/lib/sbom/package_url/decoder.rb b/lib/sbom/package_url/decoder.rb new file mode 100644 index 00000000000..ceadc36660c --- /dev/null +++ b/lib/sbom/package_url/decoder.rb @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +# MIT License +# +# Copyright (c) 2021 package-url +# Portions Copyright 2022 Gitlab B.V. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +module Sbom + class PackageUrl + class Decoder + include StringUtils + + def initialize(string) + @string = string + end + + def decode! + raise ArgumentError, "expected String but given #{@string.class}" unless @string.is_a?(::String) + + decode_subpath! + decode_qualifiers! + decode_scheme! + decode_type! + decode_version! + decode_name! + decode_namespace! + + begin + PackageUrl.new( + type: @type, + name: @name, + namespace: @namespace, + version: @version, + qualifiers: @qualifiers, + subpath: @subpath + ) + rescue ArgumentError => e + raise InvalidPackageUrl, e.message + end + end + + private + + def decode_subpath! + # Split the purl string once from right on '#' + # Given the string: `scheme:type/namespace/name@version?qualifiers#subpath` + # - The left side is the remainder: `scheme:type/namespace/name@version?qualifiers` + # - The right side will be parsed as the subpath: `subpath` + @subpath, @string = partition(@string, '#', from: :right) do |subpath| + decode_segments(subpath) do |segment| + # Discard segments which are blank, `.`, or `..` + segment.empty? || segment == '.' || segment == '..' + end + end + end + + def decode_qualifiers! + # Split the remainder once from right on '?' + # Given string: `scheme:type/namespace/name@version?qualifiers` + # - The left side is the remainder: `scheme:type/namespace/name@version` + # - The right side is the qualifiers string: `qualifiers` + @qualifiers, @string = partition(@string, '?', from: :right) do |qualifiers| + parse_qualifiers(qualifiers) + end + end + + def decode_scheme! + # Split the remainder once from left on ':' + # Given the string: `scheme:type/namespace/name@version` + # - The left side lowercased is the scheme: `scheme` + # - The right side is the remainder: `type/namespace/name@version` + @scheme, @string = partition(@string, ':', from: :left) + raise InvalidPackageUrl, 'invalid or missing "pkg:" URL scheme' unless @scheme == 'pkg' + end + + def decode_type! + # Strip the remainder from leading and trailing '/' + @string = strip(@string, '/') + # Split this once from left on '/' + # Given the string: `type/namespace/name@version` + # - The left side lowercased is the type: `type` + # - The right side is the remainder: `namespace/name@version` + @type, @string = partition(@string, '/', from: :left, &:downcase) + end + + def decode_version! + # Split the remainder once from right on '@' + # Given the string: `namespace/name@version` + # - The left side is the remainder: `namespace/name` + # - The right side is the version: `version` + # - The version must be URI decoded + @version, @string = partition(@string, '@', from: :right) do |version| + URI.decode_www_form_component(version) + end + end + + def decode_name! + # Split the remainder once from right on '/' + # Given the string: `namespace/name` + # - The left side is the remainder: `namespace` + # - The right size is the name: `name` + # - The name must be URI decoded + @name, @string = partition(@string, '/', from: :right, require_separator: false) do |name| + decoded_name = URI.decode_www_form_component(name) + Normalizer.new(type: @type, text: decoded_name).normalize_name + end + end + + def decode_namespace! + # If there is anything remaining, this is the namespace. + # The namespace may contain multiple segments delimited by `/`. + return if @string.blank? + + @namespace = decode_segments(@string, &:empty?) + @namespace = Normalizer.new(type: @type, text: @namespace).normalize_namespace + end + + def decode_segment(segment) + decoded = URI.decode_www_form_component(segment) + + raise InvalidPackageUrl, 'slash-separated segments may not contain `/`' if decoded.include?('/') + + decoded + end + + def decode_segments(string) + string.split('/').filter_map do |segment| + next if block_given? && yield(segment) + + decode_segment(segment) + end.join('/') + end + + def parse_qualifiers(raw_qualifiers) + # - Split the qualifiers on '&'. Each part is a key=value pair + # - For each pair, split the key=value once from left on '=': + # - The key is the lowercase left side + # - The value is the percent-decoded right side + # - Discard any key/value pairs where the value is empty + # - If the key is checksums, + # split the value on ',' to create a list of checksums + # - This list of key/value is the qualifiers object + raw_qualifiers.split('&').each_with_object({}) do |pair, memo| + key, separator, value = pair.partition('=') + + next if separator.empty? + + key = key.downcase + value = URI.decode_www_form_component(value) + + next if value.empty? + + memo[key] = case key + when 'checksums' + value.split(',') + else + value + end + end + end + end + end +end diff --git a/lib/sbom/package_url/encoder.rb b/lib/sbom/package_url/encoder.rb new file mode 100644 index 00000000000..9cf05095571 --- /dev/null +++ b/lib/sbom/package_url/encoder.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +# MIT License +# +# Copyright (c) 2021 package-url +# Portions Copyright 2022 Gitlab B.V. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +module Sbom + class PackageUrl + class Encoder + include StringUtils + + def initialize(package) + @type = package.type + @namespace = package.namespace + @name = package.name + @version = package.version + @qualifiers = package.qualifiers + @subpath = package.subpath + @io = StringIO.new + end + + def encode + encode_scheme! + encode_type! + encode_name! + encode_version! + encode_qualifiers! + encode_subpath! + + io.string + end + + private + + attr_reader :io + + def encode_scheme! + io.write('pkg:') + end + + def encode_type! + # Append the type string to the purl as a lowercase ASCII string + # Append '/' to the purl + io.write(@type) + io.write('/') + end + + def encode_name! + # If the namespace is empty: + # - Apply type-specific normalization to the name if needed + # - UTF-8-encode the name if needed in your programming language + # - Append the percent-encoded name to the purl + # + # If the namespace is not empty: + # - Strip the namespace from leading and trailing '/' + # - Split on '/' as segments + # - Apply type-specific normalization to each segment if needed + # - UTF-8-encode each segment if needed in your programming language + # - Percent-encode each segment + # - Join the segments with '/' + # - Append this to the purl + # - Append '/' to the purl + # - Strip the name from leading and trailing '/' + # - Apply type-specific normalization to the name if needed + # - UTF-8-encode the name if needed in your programming language + # - Append the percent-encoded name to the purl + if @namespace.nil? + io.write(URI.encode_www_form_component(@name, Encoding::UTF_8)) + else + io.write(encode_segments(@namespace, &:empty?)) + io.write('/') + io.write(URI.encode_www_form_component(strip(@name, '/'), Encoding::UTF_8)) + end + end + + def encode_version! + return if @version.nil? + + # - Append '@' to the purl + # - UTF-8-encode the version if needed in your programming language + # - Append the percent-encoded version to the purl + io.write('@') + io.write(URI.encode_www_form_component(@version, Encoding::UTF_8)) + end + + def encode_qualifiers! + return if @qualifiers.nil? || encoded_qualifiers.empty? + + io.write('?') + io.write(encoded_qualifiers) + end + + def encoded_qualifiers + @encoded_qualifiers ||= @qualifiers.filter_map do |key, value| + next if value.empty? + + next "#{key.downcase}=#{value.join(',')}" if key == 'checksums' && value.is_a?(::Array) + + "#{key.downcase}=#{URI.encode_www_form_component(value, Encoding::UTF_8)}" + end.sort.join('&') + end + + def encode_subpath! + return if @subpath.nil? || encoded_subpath.empty? + + io.write('#') + io.write(encoded_subpath) + end + + def encoded_subpath + @encoded_subpath ||= encode_segments(@subpath) do |segment| + # Discard segments which are blank, `.`, or `..` + segment.empty? || segment == '.' || segment == '..' + end + end + end + end +end diff --git a/lib/sbom/package_url/normalizer.rb b/lib/sbom/package_url/normalizer.rb new file mode 100644 index 00000000000..663df6f72a5 --- /dev/null +++ b/lib/sbom/package_url/normalizer.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Sbom + class PackageUrl + class Normalizer + def initialize(type:, text:) + @type = type + @text = text + end + + def normalize_namespace + return if text.nil? + + normalize + end + + def normalize_name + raise ArgumentError, 'Name is required' if text.nil? + + normalize + end + + private + + def normalize + case type + when 'bitbucket', 'github' + downcase + when 'pypi' + normalize_pypi + else + text + end + end + + attr_reader :type, :text + + def downcase + text.downcase + end + + def normalize_pypi + downcase.tr('_', '-') + end + end + end +end diff --git a/lib/sbom/package_url/string_utils.rb b/lib/sbom/package_url/string_utils.rb new file mode 100644 index 00000000000..c1ea8de95b2 --- /dev/null +++ b/lib/sbom/package_url/string_utils.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# MIT License +# +# Copyright (c) 2021 package-url +# Portions Copyright 2022 Gitlab B.V. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +module Sbom + class PackageUrl + module StringUtils + private + + def strip(string, char) + string = string.delete_prefix(char) while string.start_with?(char) + string = string.delete_suffix(char) while string.end_with?(char) + string + end + + def split_segments(string) + strip(string, '/').split('/') + end + + def encode_segments(string) + return '' if string.nil? + + split_segments(string).map do |segment| + next if block_given? && yield(segment) + + URI.encode_www_form_component(segment) + end.join('/') + end + + # Partition the given string on the separator. + # The side being partitioned from is returned as the value, + # with the opposing side being returned as the remainder. + # + # If a block is given, then the (value, remainder) are given + # to the block, and the return value of the block is used as the value. + # + # If `require_separator` is true, then a nil value will be returned + # if the separator is not present. + def partition(string, sep, from: :left, require_separator: true) + value, separator, remainder = if from == :left + left, separator, right = string.partition(sep) + [left, separator, right] + else + left, separator, right = string.rpartition(sep) + [right, separator, left] + end + + return [nil, value] if separator.empty? && require_separator + + value = yield(value) if block_given? + + [value, remainder] + end + end + end +end diff --git a/lib/serializers/symbolized_json.rb b/lib/serializers/symbolized_json.rb deleted file mode 100644 index 78192ce3132..00000000000 --- a/lib/serializers/symbolized_json.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Serializers - # Make the resulting hash have deep symbolized keys - class SymbolizedJson - class << self - def dump(obj) - obj - end - - def load(data) - return if data.nil? - - Gitlab::Utils.deep_symbolized_access(data) - end - end - end -end diff --git a/lib/sidebars/groups/menus/observability_menu.rb b/lib/sidebars/groups/menus/observability_menu.rb index b479ff3c492..656142375af 100644 --- a/lib/sidebars/groups/menus/observability_menu.rb +++ b/lib/sidebars/groups/menus/observability_menu.rb @@ -4,9 +4,11 @@ module Sidebars module Groups module Menus class ObservabilityMenu < ::Sidebars::Menu - override :link - def link - group_observability_index_path(context.group) + override :configure_menu_items + def configure_menu_items + add_item(dashboards_menu_item) + add_item(explore_menu_item) + add_item(manage_menu_item) end override :title @@ -24,9 +26,33 @@ module Sidebars can?(context.current_user, :read_observability, context.group) end - override :active_routes - def active_routes - { controller: :observability, path: 'groups#observability' } + private + + def dashboards_menu_item + ::Sidebars::MenuItem.new( + title: _('Dashboards'), + link: group_observability_dashboards_path(context.group), + active_routes: { path: 'groups/observability#dashboards' }, + item_id: :dashboards + ) + end + + def explore_menu_item + ::Sidebars::MenuItem.new( + title: _('Explore'), + link: group_observability_explore_path(context.group), + active_routes: { path: 'groups/observability#explore' }, + item_id: :explore + ) + end + + def manage_menu_item + ::Sidebars::MenuItem.new( + title: _('Manage Dashboards'), + link: group_observability_manage_path(context.group), + active_routes: { path: 'groups/observability#manage' }, + item_id: :manage + ) end end end diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index df170670aab..ede195a8e59 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -20,6 +20,10 @@ module Sidebars # Push Rules are the only group setting that can also be edited by maintainers. # Create an empty sub-menu here and EE adds Repository menu item (with only Push Rules). return true + elsif Gitlab.ee? && can?(context.current_user, :read_billing, context.group) + # Billing is the only group setting that is visible to auditors. + # Create an empty sub-menu here and EE adds Settings menu item (with only Billing). + return true end false diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb index 24e58e71023..9904d533f47 100644 --- a/lib/sidebars/projects/menus/deployments_menu.rb +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -6,8 +6,8 @@ module Sidebars class DeploymentsMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - add_item(feature_flags_menu_item) add_item(environments_menu_item) + add_item(feature_flags_menu_item) add_item(releases_menu_item) true diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index 2181d89262b..a8ac3d10f83 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -6,7 +6,7 @@ module Sidebars class InfrastructureMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - return false unless context.project.feature_available?(:operations, context.current_user) + return false unless feature_enabled? add_item(kubernetes_menu_item) add_item(terraform_menu_item) @@ -34,6 +34,14 @@ module Sidebars private + def feature_enabled? + if ::Feature.enabled?(:split_operations_visibility_permissions, context.project) + context.project.feature_available?(:infrastructure, context.current_user) + else + context.project.feature_available?(:operations, context.current_user) + end + end + def kubernetes_menu_item unless can?(context.current_user, :read_cluster, context.project) return ::Sidebars::NilMenuItem.new(item_id: :kubernetes) diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index ecd062f333e..035634702db 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -12,7 +12,6 @@ module Sidebars add_item(error_tracking_menu_item) add_item(alert_management_menu_item) add_item(incidents_menu_item) - add_item(product_analytics_menu_item) true end @@ -101,20 +100,6 @@ module Sidebars item_id: :incidents ) end - - def product_analytics_menu_item - if Feature.disabled?(:product_analytics, context.project) || - !can?(context.current_user, :read_product_analytics, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :product_analytics) - end - - ::Sidebars::MenuItem.new( - title: _('Product Analytics'), - link: project_product_analytics_path(context.project), - active_routes: { controller: :product_analytics }, - item_id: :product_analytics - ) - end end end end diff --git a/lib/tasks/contracts/pipelines.rake b/lib/tasks/contracts/pipelines.rake index 3163791460f..5a8d7791233 100644 --- a/lib/tasks/contracts/pipelines.rake +++ b/lib/tasks/contracts/pipelines.rake @@ -24,9 +24,13 @@ namespace :contracts do end Pact::VerificationTask.new(:get_pipeline_header_data) do |pact| + # pact.uri( + # "http://localhost:9292/pacts/provider/GET%20pipeline%20header%20data/consumer/Pipelines%23show/latest", + # pact_helper: "#{provider}/pact_helpers/project/pipeline/show/get_pipeline_header_data_helper.rb" + # ) pact.uri( "#{contracts}/show/pipelines#show-get_pipeline_header_data.json", - pact_helper: "#{provider}/pact_helpers/project/pipeline/show/get_pipeline_header_data_helper.rb" + pact_helper: "#{provider}/pact_helpers/project/pipeline/show/get_pipeline_header_data_helper.rb" ) end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index b58d9473794..12a8cb01e9e 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -84,7 +84,7 @@ namespace :gitlab do puts "Assets SHA256 for `HEAD`: #{Tasks::Gitlab::Assets.head_assets_sha256.inspect}" if Tasks::Gitlab::Assets.head_assets_sha256 != Tasks::Gitlab::Assets.master_assets_sha256 - FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_DIR) if Dir.exist?(Tasks::Gitlab::Assets::PUBLIC_ASSETS_DIR) + FileUtils.rm_rf([Tasks::Gitlab::Assets::PUBLIC_ASSETS_DIR] + Dir.glob('app/assets/javascripts/locale/**/app.js')) # gettext:po_to_json needs to run before rake:assets:precompile because # app/assets/javascripts/locale/**/app.js are pre-compiled by Sprockets @@ -127,20 +127,20 @@ namespace :gitlab do # rewrite the corresponding gzip file (if it exists) gzip = "#{file}.gz" - if File.exist?(gzip) - puts "Fixing #{gzip}" + next unless File.exist?(gzip) - FileUtils.rm(gzip) - mtime = File.stat(file).mtime + puts "Fixing #{gzip}" - File.open(gzip, 'wb+') do |f| - gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION) - gz.mtime = mtime - gz.write IO.binread(file) - gz.close + FileUtils.rm(gzip) + mtime = File.stat(file).mtime - File.utime(mtime, mtime, f.path) - end + File.open(gzip, 'wb+') do |f| + gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION) + gz.mtime = mtime + gz.write IO.binread(file) + gz.close + + File.utime(mtime, mtime, f.path) end end end diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake index bf9ebc56486..b3c98e91d17 100644 --- a/lib/tasks/gitlab/db/validate_config.rake +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -64,15 +64,15 @@ namespace :gitlab do next unless identifier connections_with_tasks = connections.select { |connection| connection[:database_tasks?] } - if connections_with_tasks.many? - names = connections_with_tasks.pluck(:name) - - warnings << "- Many configurations (#{names.join(', ')}) " \ - "share the same database (#{identifier}). " \ - "This will result in failures provisioning or migrating this database. " \ - "Ensure that additional databases are configured " \ - "with 'database_tasks: false' or are pointing to a dedicated database host." - end + next unless connections_with_tasks.many? + + names = connections_with_tasks.pluck(:name) + + warnings << "- Many configurations (#{names.join(', ')}) " \ + "share the same database (#{identifier}). " \ + "This will result in failures provisioning or migrating this database. " \ + "Ensure that additional databases are configured " \ + "with 'database_tasks: false' or are pointing to a dedicated database host." end # Each configuration with `database_tasks: false` should share the database with `main:` diff --git a/lib/tasks/gitlab/openapi.rake b/lib/tasks/gitlab/openapi.rake index fd067a1bf0b..dee365de11c 100644 --- a/lib/tasks/gitlab/openapi.rake +++ b/lib/tasks/gitlab/openapi.rake @@ -9,6 +9,13 @@ end namespace :gitlab do namespace :openapi do + task :validate do + raise 'This task can only be run in the development environment' unless Rails.env.development? + + success = system('yarn swagger:validate doc/api/openapi/openapi_v2.yaml') + abort('Validation of swagger document failed') unless success + end + task :generate do raise 'This task can only be run in the development environment' unless Rails.env.development? @@ -19,5 +26,7 @@ namespace :gitlab do File.write("doc/api/openapi/openapi_v2.yaml", yaml_content) end + + task generate_and_check: [:generate, :validate] end end diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake index 7b9c57b1876..8437ae0a31e 100644 --- a/lib/tasks/gitlab/seed.rake +++ b/lib/tasks/gitlab/seed.rake @@ -2,31 +2,37 @@ namespace :gitlab do namespace :seed do - desc "GitLab | Seed | Seeds issues" - task :issues, [:project_full_path, :backfill_weeks, :average_issues_per_week] => :environment do |t, args| - args.with_defaults(backfill_weeks: 5, average_issues_per_week: 2) + def projects_from_args(args) + full_path = args.project_full_path - projects = - if args.project_full_path - project = Project.find_by_full_path(args.project_full_path) + if full_path + project = Project.find_by_full_path(full_path) - unless project - error_message = "Project '#{args.project_full_path}' does not exist!" - potential_projects = Project.search(args.project_full_path) + unless project + error_message = "Project '#{full_path}' does not exist!" + potential_projects = Project.search(full_path) - if potential_projects.present? - error_message += " Did you mean '#{potential_projects.first.full_path}'?" - end - - puts error_message.color(:red) - exit 1 + if potential_projects.present? + error_message += " Did you mean '#{potential_projects.first.full_path}'?" end - [project] - else - Project.not_mass_generated.find_each + puts error_message.color(:red) + exit 1 end + [project] + else + scope = Project.respond_to?(:not_mass_generated) ? Project.not_mass_generated : Project + scope.find_each + end + end + + desc "GitLab | Seed | Seeds issues" + task :issues, [:project_full_path, :backfill_weeks, :average_issues_per_week] => :environment do |t, args| + args.with_defaults(backfill_weeks: 5, average_issues_per_week: 2) + + projects = projects_from_args(args) + projects.each do |project| puts "\nSeeding issues for the '#{project.full_path}' project" seeder = Quality::Seeders::Issues.new(project: project) @@ -70,5 +76,17 @@ namespace :gitlab do puts "\n#{epics} epics created!" end end + + desc "GitLab | Seed | Seed a project with vulnerabilities" + task :vulnerabilities, [:project_full_path] => :environment do |t, args| + projects = projects_from_args(args) + + projects.each do |project| + puts "\nSeeding vulnerabilities for the '#{project.full_path}' project" + seeder = Quality::Seeders::Vulnerabilities.new(project) + seeder.seed! + puts "\nDone." + end + end end end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 10492e183c5..dc472305304 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -10,15 +10,22 @@ namespace :gitlab do desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names' task schedule: :environment do ::Gitlab::SidekiqMigrateJobs - .new('schedule', logger: Logger.new($stdout)) - .execute(::Gitlab::SidekiqConfig.worker_queue_mappings) + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .migrate_set('schedule') end desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names' task retry: :environment do ::Gitlab::SidekiqMigrateJobs - .new('retry', logger: Logger.new($stdout)) - .execute(::Gitlab::SidekiqConfig.worker_queue_mappings) + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .migrate_set('retry') + end + + desc 'GitLab | Sidekiq | Migrate jobs in queues outside of routing rules' + task queued: :environment do + ::Gitlab::SidekiqMigrateJobs + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .migrate_queues end end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index fd9c7114979..7a2dee3e2e4 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -28,6 +28,7 @@ namespace :tw do CodeOwnerRule.new('Compliance', '@eread'), CodeOwnerRule.new('Composition Analysis', '@rdickenson'), CodeOwnerRule.new('Configure', '@phillipwells'), + CodeOwnerRule.new('Container Registry', '@claytoncornell'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Conversion', '@kpaizee'), CodeOwnerRule.new('Database', '@aqualls'), @@ -37,7 +38,6 @@ namespace :tw do CodeOwnerRule.new('Distribution (Omnibus)', '@axil'), CodeOwnerRule.new('Documentation Guidelines', '@sselhorn'), CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'), - CodeOwnerRule.new('Ecosystem', '@kpaizee'), CodeOwnerRule.new('Editor', '@ashrafkhamis'), CodeOwnerRule.new('Foundations', '@rdickenson'), CodeOwnerRule.new('Fuzz Testing', '@rdickenson'), @@ -48,11 +48,11 @@ namespace :tw do CodeOwnerRule.new('Infrastructure', '@sselhorn'), CodeOwnerRule.new('Integrations', '@ashrafkhamis'), CodeOwnerRule.new('Knowledge', '@aqualls'), - CodeOwnerRule.new('Application Performance', '@sselhorn'), + CodeOwnerRule.new('Application Performance', '@jglassman1'), CodeOwnerRule.new('Monitor', '@msedlakjakubowski'), - CodeOwnerRule.new('Observability', 'msedlakjakubowski'), - CodeOwnerRule.new('Optimize', '@fneill'), - CodeOwnerRule.new('Package', '@claytoncornell'), + CodeOwnerRule.new('Observability', '@msedlakjakubowski'), + CodeOwnerRule.new('Optimize', '@lciutacu'), + CodeOwnerRule.new('Package Registry', '@claytoncornell'), CodeOwnerRule.new('Pipeline Authoring', '@marcel.amirault'), CodeOwnerRule.new('Pipeline Execution', '@marcel.amirault'), CodeOwnerRule.new('Pipeline Insights', '@marcel.amirault'), @@ -66,19 +66,24 @@ namespace :tw do CodeOwnerRule.new('Redirect', 'Redirect'), CodeOwnerRule.new('Release', '@rdickenson'), CodeOwnerRule.new('Respond', '@msedlakjakubowski'), - CodeOwnerRule.new('Runner', '@sselhorn'), - CodeOwnerRule.new('Pods', '@sselhorn'), + CodeOwnerRule.new('Runner', '@fneill'), + CodeOwnerRule.new('Runner SaaS', '@fneill'), + CodeOwnerRule.new('Pods', '@jglassman1'), CodeOwnerRule.new('Security Policies', '@claytoncornell'), CodeOwnerRule.new('Source Code', '@aqualls'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Testing', '@eread'), CodeOwnerRule.new('Threat Insights', '@claytoncornell'), + CodeOwnerRule.new('Tutorials', '@kpaizee'), CodeOwnerRule.new('Utilization', '@fneill'), CodeOwnerRule.new('Vulnerability Research', '@claytoncornell'), - CodeOwnerRule.new('Workspace', '@fneill') + CodeOwnerRule.new('Workspace', '@lciutacu') ].freeze + CODEOWNERS_BLOCK_BEGIN = "# Begin rake-managed-docs-block" + CODEOWNERS_BLOCK_END = "# End rake-managed-docs-block" + Document = Struct.new(:group, :redirect) do def has_a_valid_group? group && !redirect @@ -122,7 +127,17 @@ namespace :tw do end end - deduplicated_mappings.sort.each { |mapping| puts mapping } + new_docs_owners = deduplicated_mappings.sort.join("\n") + + codeowners_path = Rails.root.join('.gitlab/CODEOWNERS') + current_codeowners_content = File.read(codeowners_path) + + docs_replace_regex = Regexp.new("#{CODEOWNERS_BLOCK_BEGIN}\n[\\s\\S]*?\n#{CODEOWNERS_BLOCK_END}") + + new_codeowners_content = current_codeowners_content + .gsub(docs_replace_regex, "#{CODEOWNERS_BLOCK_BEGIN}\n#{new_docs_owners}\n#{CODEOWNERS_BLOCK_END}") + + File.write(codeowners_path, new_codeowners_content) if errors.present? puts "-----" diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index 07dd5ebeacb..d67ad340007 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -58,14 +58,14 @@ namespace :gitlab do # extract a concrete commit for signing off what we actually downloaded # this way we do the right thing even if the repository gets updated in the meantime - get_commits_response = Gitlab::HTTP.get("https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/commits", + get_commits_response = Gitlab::HTTP.get("#{template.project_host}/api/v4/projects/#{uri_encoded_project_path}/repository/commits", query: { page: 1, per_page: 1 } ) raise "Failed to retrieve latest commit for template '#{template.name}'" unless get_commits_response.success? commit_sha = get_commits_response.parsed_response.dig(0, 'id') - project_archive_uri = "https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/archive.tar.gz?sha=#{commit_sha}" + project_archive_uri = "#{template.project_host}/api/v4/projects/#{uri_encoded_project_path}/repository/archive.tar.gz?sha=#{commit_sha}" commit_message = <<~MSG Initialized from '#{template.title}' project template diff --git a/lib/unnested_in_filters/rewriter.rb b/lib/unnested_in_filters/rewriter.rb index bf7be177a0d..ed1e4ce2d9f 100644 --- a/lib/unnested_in_filters/rewriter.rb +++ b/lib/unnested_in_filters/rewriter.rb @@ -120,15 +120,47 @@ module UnnestedInFilters # "vulnerability_reads"."vulnerability_id" DESC # LIMIT 20 # + # If one of the columns being used for filtering or ordering is the primary key, + # then the query will be further optimized to use an index-only scan for initial filtering + # before selecting all columns using the primary key. + # + # Using the prior query as an example, where `vulnerability_id` is the primary key, + # This will be rewritten to: + # + # SELECT + # "vulnerability_reads".* + # FROM + # "vulnerability_reads" + # WHERE + # "vulnerability_reads"."vulnerability_id" + # IN ( + # SELECT + # "vulnerability_reads"."vulnerability_id" + # FROM + # unnest('{1, 4}'::smallint[]) AS "states" ("state"), + # LATERAL ( + # SELECT + # "vulnerability_reads"."vulnerability_id" + # FROM + # "vulnerability_reads" + # WHERE + # (vulnerability_reads."state" = "states"."state") + # ORDER BY + # "vulnerability_reads"."severity" DESC, + # "vulnerability_reads"."vulnerability_id" DESC + # LIMIT 20 + # ) AS vulnerability_reads + # ) + # ORDER BY + # "vulnerability_reads"."severity" DESC, + # "vulnerability_reads"."vulnerability_id" DESC + # LIMIT 20 def rewrite log_rewrite - model.from(from) - .limit(limit_value) - .order(order_values) - .includes(relation.includes_values) - .preload(relation.preload_values) - .eager_load(relation.eager_load_values) + return filter_query unless primary_key_present? + + index_only_filter_query end def rewrite? @@ -147,6 +179,23 @@ module UnnestedInFilters ::Gitlab::AppLogger.info(message: 'Query is being rewritten by `UnnestedInFilters`', model: model.name) end + def filter_query + model.from(from).then { add_relation_defaults(_1) } + end + + def index_only_filter_query + model.where(model.primary_key => filter_query.select(model.primary_key)) + .then { add_relation_defaults(_1) } + end + + def add_relation_defaults(new_relation) + new_relation.limit(limit_value) + .order(order_values) + .includes(relation.includes_values) + .preload(relation.preload_values) + .eager_load(relation.eager_load_values) + end + def from [value_tables.map(&:to_sql) + [lateral]].join(', ') end @@ -156,9 +205,13 @@ module UnnestedInFilters end def join_relation - value_tables.reduce(unscoped_relation) do |memo, tmp_table| + join_relation = value_tables.reduce(unscoped_relation) do |memo, tmp_table| memo.where(tmp_table.as_predicate) end + + join_relation = join_relation.select(combined_attributes) if primary_key_present? + + join_relation end def unscoped_relation @@ -169,8 +222,14 @@ module UnnestedInFilters @in_filters ||= arel_in_nodes.each_with_object({}) { |node, memo| memo[node.left.name] = node.right } end + def model_column_names + @model_column_names ||= model.columns.map(&:name) + end + + # Actively filter any nodes that don't belong to the primary queried table to prevent sql type resolution issues + # Context: https://gitlab.com/gitlab-org/gitlab/-/issues/370271#note_1151019824 def arel_in_nodes - where_clause_arel_nodes.select(&method(:in_predicate?)) + where_clause_arel_nodes.select(&method(:in_predicate?)).select { model_column_names.include?(_1.left.name) } end # `ActiveRecord::WhereClause#ast` is returning a single node when there is only one @@ -194,12 +253,20 @@ module UnnestedInFilters indices.any? do |index| (filter_attributes - Array(index.columns)).empty? && # all the filter attributes are indexed index.columns.last(order_attributes.length) == order_attributes && # index can be used in sorting - (index.columns - (filter_attributes + order_attributes)).empty? # there is no other columns in the index + (index.columns - combined_attributes).empty? # there is no other columns in the index end end + def primary_key_present? + combined_attributes.include?(model.primary_key) + end + + def combined_attributes + filter_attributes + order_attributes + end + def filter_attributes - @filter_attributes ||= where_values_hash.keys + @filter_attributes ||= where_clause.to_h.keys end def order_attributes |