diff options
Diffstat (limited to 'lib')
507 files changed, 7508 insertions, 3527 deletions
diff --git a/lib/api/admin/batched_background_migrations.rb b/lib/api/admin/batched_background_migrations.rb index e8cc08a23be..7e612b5b66a 100644 --- a/lib/api/admin/batched_background_migrations.rb +++ b/lib/api/admin/batched_background_migrations.rb @@ -12,7 +12,15 @@ module API namespace 'admin' do resources 'batched_background_migrations/:id' do - desc 'Retrieve a batched background migration' + desc 'Retrieve a batched background migration' do + success ::API::Entities::BatchedBackgroundMigration + failure [ + { code: 401, message: '401 Unauthorized' }, + { code: 403, message: '403 Forbidden' }, + { code: 404, message: '404 Not found' } + ] + tags %w[batched_background_migrations] + end params do optional :database, type: String, @@ -31,7 +39,15 @@ module API end resources 'batched_background_migrations' do - desc 'Get the list of the batched background migrations' + desc 'Get the list of batched background migrations' do + success ::API::Entities::BatchedBackgroundMigration + failure [ + { code: 401, message: '401 Unauthorized' }, + { code: 403, message: '403 Forbidden' } + ] + is_array true + tags %w[batched_background_migrations] + end params do optional :database, type: String, @@ -48,7 +64,16 @@ module API end resources 'batched_background_migrations/:id/resume' do - desc 'Resume a batched background migration' + desc 'Resume a batched background migration' do + success ::API::Entities::BatchedBackgroundMigration + failure [ + { code: 401, message: '401 Unauthorized' }, + { code: 403, message: '403 Forbidden' }, + { code: 404, message: '404 Not found' }, + { code: 422, message: 'You can resume only `paused` batched background migrations.' } + ] + tags %w[batched_background_migrations] + end params do optional :database, type: String, @@ -73,7 +98,16 @@ module API end resources 'batched_background_migrations/:id/pause' do - desc 'Pause a batched background migration' + desc 'Pause a batched background migration' do + success ::API::Entities::BatchedBackgroundMigration + failure [ + { code: 401, message: '401 Unauthorized' }, + { code: 403, message: '403 Forbidden' }, + { code: 404, message: '404 Not found' }, + { code: 422, message: 'You can pause only `active` batched background migrations.' } + ] + tags %w[batched_background_migrations] + end params do optional :database, type: String, diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb index 49b41b44a18..5ef56d3326f 100644 --- a/lib/api/admin/plan_limits.rb +++ b/lib/api/admin/plan_limits.rb @@ -70,6 +70,8 @@ module API optional :terraform_module_max_file_size, type: Integer, desc: 'Maximum Terraform Module package file size in bytes' optional :storage_size_limit, type: Integer, desc: 'Maximum storage size for the root namespace in megabytes' + optional :pipeline_hierarchy_size, type: Integer, + desc: "Maximum number of downstream pipelines in a pipeline's hierarchy tree" end put "application/plan_limits" do params = declared_params(include_missing: false) diff --git a/lib/api/alert_management_alerts.rb b/lib/api/alert_management_alerts.rb index f57b7d00c81..9e28ee049d0 100644 --- a/lib/api/alert_management_alerts.rb +++ b/lib/api/alert_management_alerts.rb @@ -6,12 +6,21 @@ module API urgency :low params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' - requires :alert_iid, type: Integer, desc: 'The IID of the Alert' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project', + documentation: { example: 17 } + requires :alert_iid, type: Integer, desc: 'The IID of the Alert', + documentation: { example: 23 } end resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/alert_management_alerts/:alert_iid/metric_images' do + desc 'Workhorse authorize metric image file upload' do + success code: 200 + failure [ + { code: 403, message: 'Forbidden' } + ] + tags %w[alert_management] + end post 'authorize' do authorize!(:upload_alert_management_metric_image, find_project_alert(request.params[:alert_iid])) @@ -29,13 +38,20 @@ module API end desc 'Upload a metric image for an alert' do - success Entities::MetricImage + consumes ['multipart/form-data'] + success code: 200, model: Entities::MetricImage + failure [ + { code: 403, message: 'Forbidden' } + ] + tags %w[alert_management] end params do requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The image file to be uploaded', documentation: { type: 'file' } - optional :url, type: String, desc: 'The url to view more metric info' - optional :url_text, type: String, desc: 'A description of the image or URL' + optional :url, type: String, desc: 'The url to view more metric info', + documentation: { example: 'https://example.com/metric' } + optional :url_text, type: String, desc: 'A description of the image or URL', + documentation: { example: 'An example metric' } end post do require_gitlab_workhorse! @@ -61,7 +77,14 @@ module API end end - desc 'Metric Images for alert' + desc 'Metric Images for alert' do + success code: 200, model: Entities::MetricImage + is_array true + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[alert_management] + end get do alert = find_project_alert(params[:alert_iid]) @@ -73,12 +96,21 @@ module API end desc 'Update a metric image for an alert' do - success Entities::MetricImage + consumes ['multipart/form-data'] + success code: 200, model: Entities::MetricImage + failure [ + { code: 403, message: 'Forbidden' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[alert_management] end params do - requires :metric_image_id, type: Integer, desc: 'The ID of metric image' - optional :url, type: String, desc: 'The url to view more metric info' - optional :url_text, type: String, desc: 'A description of the image or URL' + requires :metric_image_id, type: Integer, desc: 'The ID of metric image', + documentation: { example: 42 } + optional :url, type: String, desc: 'The url to view more metric info', + documentation: { example: 'https://example.com/metric' } + optional :url_text, type: String, desc: 'A description of the image or URL', + documentation: { example: 'An example metric' } end put ':metric_image_id' do alert = find_project_alert(params[:alert_iid]) @@ -97,10 +129,16 @@ module API end desc 'Remove a metric image for an alert' do - success Entities::MetricImage + success code: 204, model: Entities::MetricImage + failure [ + { code: 403, message: 'Forbidden' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[alert_management] end params do - requires :metric_image_id, type: Integer, desc: 'The ID of metric image' + requires :metric_image_id, type: Integer, desc: 'The ID of metric image', + documentation: { example: 42 } end delete ':metric_image_id' do alert = find_project_alert(params[:alert_iid]) diff --git a/lib/api/api.rb b/lib/api/api.rb index ffb0cdf8991..b23b11d0c29 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -171,9 +171,11 @@ module API namespace do # Keep in alphabetical order mount ::API::AccessRequests + mount ::API::Admin::BatchedBackgroundMigrations mount ::API::Admin::Ci::Variables mount ::API::Admin::InstanceClusters mount ::API::Admin::PlanLimits + mount ::API::AlertManagementAlerts mount ::API::Appearance mount ::API::Applications mount ::API::Avatar @@ -181,10 +183,13 @@ module API mount ::API::Branches mount ::API::BroadcastMessages mount ::API::BulkImports + mount ::API::Ci::JobArtifacts + mount ::API::Groups mount ::API::Ci::Jobs mount ::API::Ci::ResourceGroups mount ::API::Ci::Runner mount ::API::Ci::Runners + mount ::API::Ci::SecureFiles mount ::API::Ci::Pipelines mount ::API::Ci::PipelineSchedules mount ::API::Ci::Triggers @@ -193,6 +198,13 @@ module API mount ::API::Clusters::Agents mount ::API::Commits mount ::API::CommitStatuses + mount ::API::ComposerPackages + mount ::API::ConanInstancePackages + mount ::API::ConanProjectPackages + mount ::API::ContainerRegistryEvent + mount ::API::ContainerRepositories + mount ::API::DebianGroupPackages + mount ::API::DebianProjectPackages mount ::API::DependencyProxy mount ::API::DeployKeys mount ::API::DeployTokens @@ -200,54 +212,75 @@ module API mount ::API::Environments mount ::API::ErrorTracking::ClientKeys mount ::API::ErrorTracking::ProjectSettings + mount ::API::Events mount ::API::FeatureFlags mount ::API::FeatureFlagsUserLists mount ::API::Features mount ::API::Files mount ::API::FreezePeriods + mount ::API::GenericPackages mount ::API::Geo mount ::API::GoProxy mount ::API::GroupAvatar mount ::API::GroupClusters mount ::API::GroupContainerRepositories + mount ::API::GroupDebianDistributions mount ::API::GroupExport mount ::API::GroupImport mount ::API::GroupPackages mount ::API::GroupVariables + mount ::API::HelmPackages mount ::API::ImportBitbucketServer mount ::API::ImportGithub mount ::API::Integrations + mount ::API::Integrations::JiraConnect::Subscriptions mount ::API::Invitations mount ::API::IssueLinks mount ::API::Keys mount ::API::Lint mount ::API::Markdown + mount ::API::MavenPackages + mount ::API::Members mount ::API::MergeRequestApprovals + mount ::API::MergeRequests mount ::API::MergeRequestDiffs mount ::API::Metadata mount ::API::Metrics::Dashboard::Annotations mount ::API::Metrics::UserStarredDashboards + mount ::API::Namespaces + mount ::API::NpmInstancePackages + mount ::API::NpmProjectPackages + mount ::API::NugetGroupPackages + mount ::API::NugetProjectPackages mount ::API::PackageFiles + mount ::API::Pages mount ::API::PersonalAccessTokens::SelfInformation mount ::API::PersonalAccessTokens mount ::API::ProjectClusters + mount ::API::ProjectContainerRepositories + mount ::API::ProjectDebianDistributions mount ::API::ProjectEvents mount ::API::ProjectExport mount ::API::ProjectHooks mount ::API::ProjectImport + mount ::API::ProjectPackages mount ::API::ProjectRepositoryStorageMoves mount ::API::ProjectSnippets mount ::API::ProjectSnapshots mount ::API::ProjectStatistics mount ::API::ProjectTemplates + mount ::API::Projects mount ::API::ProtectedBranches mount ::API::ProtectedTags + mount ::API::PypiPackages mount ::API::Releases mount ::API::Release::Links mount ::API::RemoteMirrors mount ::API::Repositories mount ::API::ResourceAccessTokens mount ::API::ResourceMilestoneEvents + mount ::API::RpmProjectPackages + mount ::API::RubygemPackages mount ::API::Snippets mount ::API::SnippetRepositoryStorageMoves mount ::API::Statistics @@ -260,6 +293,9 @@ module API mount ::API::Terraform::StateVersion mount ::API::Topics mount ::API::Unleash + mount ::API::UsageData + mount ::API::UsageDataNonSqlMetrics + mount ::API::UsageDataQueries mount ::API::UserCounts mount ::API::Wikis @@ -267,57 +303,27 @@ module API end # Keep in alphabetical order - mount ::API::Admin::BatchedBackgroundMigrations mount ::API::Admin::Sidekiq - mount ::API::AlertManagementAlerts mount ::API::AwardEmoji mount ::API::Boards - mount ::API::Ci::JobArtifacts + mount ::API::Ci::Pipelines + mount ::API::Ci::PipelineSchedules mount ::API::Ci::SecureFiles - mount ::API::ComposerPackages - mount ::API::ConanInstancePackages - mount ::API::ConanProjectPackages - mount ::API::ContainerRegistryEvent - mount ::API::ContainerRepositories - mount ::API::DebianGroupPackages - mount ::API::DebianProjectPackages mount ::API::Discussions mount ::API::ErrorTracking::Collector - mount ::API::Events - mount ::API::GenericPackages mount ::API::GroupBoards - mount ::API::GroupDebianDistributions mount ::API::GroupLabels mount ::API::GroupMilestones - mount ::API::Groups - mount ::API::HelmPackages - mount ::API::Integrations::JiraConnect::Subscriptions mount ::API::Issues mount ::API::Labels - mount ::API::MavenPackages - mount ::API::Members - mount ::API::MergeRequests - mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings - mount ::API::NpmInstancePackages - mount ::API::NpmProjectPackages - mount ::API::NugetGroupPackages - mount ::API::NugetProjectPackages - mount ::API::Pages mount ::API::PagesDomains - mount ::API::ProjectContainerRepositories - mount ::API::ProjectDebianDistributions mount ::API::ProjectEvents mount ::API::ProjectMilestones - mount ::API::ProjectPackages - mount ::API::Projects mount ::API::ProtectedTags - mount ::API::PypiPackages mount ::API::ResourceLabelEvents mount ::API::ResourceStateEvents - mount ::API::RpmProjectPackages - mount ::API::RubygemPackages mount ::API::Search mount ::API::Settings mount ::API::SidekiqMetrics @@ -327,7 +333,6 @@ module API mount ::API::Todos mount ::API::UsageData mount ::API::UsageDataNonSqlMetrics - mount ::API::UsageDataQueries mount ::API::Users mount ::API::Ml::Mlflow end diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb index 69f1521ef2a..2cef1b27504 100644 --- a/lib/api/appearance.rb +++ b/lib/api/appearance.rb @@ -26,6 +26,7 @@ module API end params do optional :title, type: String, desc: 'Instance title on the sign in / sign up page' + optional :short_title, type: String, desc: 'Short title for Progressive Web App' optional :description, type: String, desc: 'Markdown text shown on the sign in / sign up page' # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 optional :logo, type: File, desc: 'Instance image used on the sign in / sign up page' # rubocop:disable Scalability/FileUploads diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index e419a025508..f7a39db7249 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -80,7 +80,7 @@ module API delete "#{endpoint}/:award_id", feature_category: awardable_params[:feature_category] do award = awardable.award_emoji.find(params[:award_id]) - unauthorized! unless award.user == current_user || current_user&.admin? + unauthorized! unless award.user == current_user || current_user&.can_admin_all_resources? destroy_conditionally!(award) end diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb index 352ad04c982..3788f5bec41 100644 --- a/lib/api/ci/job_artifacts.rb +++ b/lib/api/ci/job_artifacts.rb @@ -16,18 +16,25 @@ module API end end - prepend_mod_with('API::Ci::JobArtifacts') # rubocop: disable Cop/InjectEnterpriseEditionModule - params do requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.10' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the job' + requires :ref_name, type: String, + desc: 'Branch or tag name in repository. `HEAD` or `SHA` references are not supported.' + requires :job, type: String, desc: 'The name of the job.' + optional :job_token, type: String, + desc: 'To be used with triggers for multi-project pipelines, ' \ + 'available only on Premium and Ultimate tiers.' end route_setting :authentication, job_token_allowed: true get ':id/jobs/artifacts/:ref_name/download', @@ -43,11 +50,21 @@ module API desc 'Download a specific file from artifacts archive from a ref' do detail 'This feature was introduced in GitLab 11.5' + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the job' - requires :artifact_path, type: String, desc: 'Artifact path' + requires :ref_name, type: String, + desc: 'Branch or tag name in repository. `HEAD` or `SHA` references are not supported.' + requires :job, type: String, desc: 'The name of the job.' + requires :artifact_path, type: String, desc: 'Path to a file inside the artifacts archive.' + optional :job_token, type: String, + desc: 'To be used with triggers for multi-project pipelines, ' \ + 'available only on Premium and Ultimate tiers.' end route_setting :authentication, job_token_allowed: true get ':id/jobs/artifacts/:ref_name/raw/*artifact_path', @@ -69,9 +86,17 @@ module API desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.5' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do requires :job_id, type: Integer, desc: 'The ID of a job' + optional :job_token, type: String, + desc: 'To be used with triggers for multi-project pipelines, ' \ + 'available only on Premium and Ultimate tiers.' end route_setting :authentication, job_token_allowed: true get ':id/jobs/:job_id/artifacts', urgency: :low do @@ -85,10 +110,19 @@ module API desc 'Download a specific file from artifacts archive' do detail 'This feature was introduced in GitLab 10.0' + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do requires :job_id, type: Integer, desc: 'The ID of a job' - requires :artifact_path, type: String, desc: 'Artifact path' + requires :artifact_path, type: String, desc: 'Path to a file inside the artifacts archive.' + optional :job_token, type: String, + desc: 'To be used with triggers for multi-project pipelines, ' \ + 'available only on Premium and Ultimate tiers.' end route_setting :authentication, job_token_allowed: true get ':id/jobs/:job_id/artifacts/*artifact_path', urgency: :low, format: false do @@ -113,6 +147,11 @@ module API desc 'Keep the artifacts to prevent them from being deleted' do success ::API::Entities::Ci::Job + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] end params do requires :job_id, type: Integer, desc: 'The ID of a job' @@ -132,6 +171,12 @@ module API desc 'Delete the artifacts files from a job' do detail 'This feature was introduced in GitLab 11.9' + success code: 204 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 409, message: 'Conflict' } + ] end params do requires :job_id, type: Integer, desc: 'The ID of a job' @@ -148,7 +193,14 @@ module API status :no_content end - desc 'Expire the artifacts files from a project' + desc 'Expire the artifacts files from a project' do + success code: 202 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 409, message: 'Conflict' } + ] + end delete ':id/artifacts' do authorize_destroy_artifacts! @@ -162,3 +214,5 @@ module API end end end + +API::Ci::JobArtifacts.prepend_mod diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index 9e41e1c0d8f..bb57a717f7c 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -49,13 +49,15 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ':id/jobs', urgency: :low, feature_category: :continuous_integration do + check_rate_limit!(:jobs_index, scope: current_user) if enforce_jobs_api_rate_limits(@project) + authorize_read_builds! builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project) - present paginate(builds), with: Entities::Ci::Job + + present paginate(builds, without_count: true), with: Entities::Ci::Job end # rubocop: enable CodeReuse/ActiveRecord @@ -255,16 +257,19 @@ module API pipeline = current_authenticated_job.pipeline project = current_authenticated_job.project - agent_authorizations = ::Clusters::AgentAuthorizationsFinder.new(project).execute project_groups = project.group&.self_and_ancestor_ids&.map { |id| { id: id } } || [] user_access_level = project.team.max_member_access(current_user.id) roles_in_project = Gitlab::Access.sym_options_with_owner .select { |_role, role_access_level| role_access_level <= user_access_level } .map(&:first) - environment = if persisted_environment = current_authenticated_job.actual_persisted_environment - { tier: persisted_environment.tier, slug: persisted_environment.slug } - end + persisted_environment = current_authenticated_job.actual_persisted_environment + environment = { tier: persisted_environment.tier, slug: persisted_environment.slug } if persisted_environment + + agent_authorizations = ::Clusters::Agents::FilterAuthorizationsService.new( + ::Clusters::AgentAuthorizationsFinder.new(project).execute, + environment: persisted_environment&.name + ).execute # See https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kubernetes_ci_access.md#apiv4joballowed_agents-api { diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index c7d1887638a..b073eb49bf1 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -229,15 +229,17 @@ module API params do requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) + optional :debug_trace, type: Boolean, desc: %q(Enable or Disable the debug trace) end patch '/:id/trace', urgency: :low, feature_category: :continuous_integration do job = authenticate_job!(heartbeat_runner: true) error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] + debug_trace = Gitlab::Utils.to_boolean(params[:debug_trace]) result = ::Ci::AppendBuildTraceService - .new(job, content_range: content_range) + .new(job, content_range: content_range, debug_trace: debug_trace) .execute(request.body.read) if result.status == 403 @@ -256,7 +258,7 @@ module API header 'X-GitLab-Trace-Update-Interval', job.trace.update_interval.to_s end - desc 'Authorize artifacts uploading for job' do + desc 'Authorize uploading job artifact' do http_codes [[200, 'Upload allowed'], [403, 'Forbidden'], [405, 'Artifacts support not enabled'], @@ -270,7 +272,7 @@ module API # In current runner, filesize parameter would be empty here. This is because archive is streamed by runner, # so the archive size is not known ahead of time. Streaming is done to not use additional I/O on # Runner to first save, and then send via Network. - optional :filesize, type: Integer, desc: %q(Artifacts filesize) + optional :filesize, type: Integer, desc: %q(Size of artifact file) optional :artifact_type, type: String, desc: %q(The type of artifact), default: 'archive', values: ::Ci::JobArtifact.file_types.keys @@ -292,7 +294,7 @@ module API end end - desc 'Upload artifacts for job' do + desc 'Upload a job artifact' do success Entities::Ci::JobRequest::Response http_codes [[201, 'Artifact uploaded'], [400, 'Bad request'], @@ -304,7 +306,7 @@ module API requires :id, type: Integer, desc: %q(Job's ID) requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact file to store (generated by Multipart middleware)), documentation: { type: 'file' } optional :token, type: String, desc: %q(Job's authentication token) - optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) + optional :expire_in, type: String, desc: %q(Specify when artifact should expire) optional :artifact_type, type: String, desc: %q(The type of artifact), default: 'archive', values: ::Ci::JobArtifact.file_types.keys optional :artifact_format, type: String, desc: %q(The format of artifact), @@ -333,7 +335,7 @@ module API end desc 'Download the artifacts file for job' do - http_codes [[200, 'Upload allowed'], + http_codes [[200, 'Download allowed'], [401, 'Unauthorized'], [403, 'Forbidden'], [404, 'Artifact not found']] diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 988c3f4f566..4a6c58b4987 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -58,19 +58,19 @@ module API end def authenticate_show_runner!(runner) - return if runner.instance_type? || current_user.admin? + return if runner.instance_type? || current_user.can_read_all_resources? forbidden!("No access granted") unless can?(current_user, :read_runner, runner) end def authenticate_update_runner!(runner) - return if current_user.admin? + return if current_user.can_admin_all_resources? forbidden!("No access granted") unless can?(current_user, :update_runner, runner) end def authenticate_delete_runner!(runner) - return if current_user.admin? + return if current_user.can_admin_all_resources? forbidden!("Runner associated with more than one project") if runner.runner_projects.count > 1 forbidden!("No access granted") unless can?(current_user, :delete_runner, runner) @@ -79,14 +79,14 @@ module API def authenticate_enable_runner!(runner) forbidden!("Runner is a group runner") if runner.group_type? - return if current_user.admin? + return if current_user.can_admin_all_resources? forbidden!("Runner is locked") if runner.locked? forbidden!("No access granted") unless can?(current_user, :assign_runner, runner) end def authenticate_list_runners_jobs!(runner) - return if current_user.admin? + return if current_user.can_read_all_resources? forbidden!("No access granted") unless can?(current_user, :read_builds, runner) end diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index dd628a3413f..6483abcc74e 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -7,7 +7,6 @@ module API before do authenticate! - feature_flag_enabled? authorize! :read_secure_files, user_project end @@ -16,11 +15,15 @@ module API default_format :json params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project owned by the + authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'List all Secure Files for a Project' + desc 'Get list of secure files in a project' do + success Entities::Ci::SecureFile + tags %w[secure_files] + end params do use :pagination end @@ -30,9 +33,13 @@ module API present paginate(secure_files), with: Entities::Ci::SecureFile end - desc 'Get an individual Secure File' + desc 'Get the details of a specific secure file in a project' do + success Entities::Ci::SecureFile + tags %w[secure_files] + failure [{ code: 404, message: '404 Not found' }] + end params do - requires :id, type: Integer, desc: 'The Secure File ID' + requires :id, type: Integer, desc: 'The ID of a secure file' end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true @@ -41,7 +48,10 @@ module API present secure_file, with: Entities::Ci::SecureFile end - desc 'Download a Secure File' + desc 'Download secure file' do + failure [{ code: 404, message: '404 Not found' }] + tags %w[secure_files] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true get ':id/secure_files/:secure_file_id/download' do secure_file = user_project.secure_files.find(params[:secure_file_id]) @@ -58,10 +68,15 @@ module API authorize! :admin_secure_files, user_project end - desc 'Upload a Secure File' + desc 'Create a secure file' do + success Entities::Ci::SecureFile + tags %w[secure_files] + failure [{ code: 400, message: '400 Bad Request' }] + end params do - requires :name, type: String, desc: 'The name of the file' - requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The secure file to be uploaded', documentation: { type: 'file' } + requires :name, type: String, desc: 'The name of the file being uploaded. The filename must be unique within + the project' + requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The secure file being uploaded', documentation: { type: 'file' } end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true post ':id/secure_files' do @@ -74,17 +89,17 @@ module API file_too_large! unless secure_file.file.size < ::Ci::SecureFile::FILE_SIZE_LIMIT.to_i if secure_file.save - if Feature.enabled?(:secure_files_metadata_parsers, user_project) - ::Ci::ParseSecureFileMetadataWorker.perform_async(secure_file.id) # rubocop:disable CodeReuse/Worker - end - + ::Ci::ParseSecureFileMetadataWorker.perform_async(secure_file.id) # rubocop:disable CodeReuse/Worker present secure_file, with: Entities::Ci::SecureFile else render_validation_error!(secure_file) end end - desc 'Delete an individual Secure File' + desc 'Remove a secure file' do + tags %w[secure_files] + failure [{ code: 404, message: '404 Not found' }] + end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true delete ':id/secure_files/:secure_file_id' do secure_file = user_project.secure_files.find(params[:secure_file_id]) @@ -97,10 +112,6 @@ module API end helpers do - def feature_flag_enabled? - service_unavailable! unless Feature.enabled?(:ci_secure_files, user_project) - end - def read_only_feature_flag_enabled? service_unavailable! if Feature.enabled?(:ci_secure_files_read_only, user_project, type: :ops) end diff --git a/lib/api/clusters/agent_tokens.rb b/lib/api/clusters/agent_tokens.rb index f65ae465b3d..68eef21903d 100644 --- a/lib/api/clusters/agent_tokens.rb +++ b/lib/api/clusters/agent_tokens.rb @@ -27,7 +27,8 @@ module API use :pagination end get do - agent_tokens = ::Clusters::AgentTokensFinder.new(user_project, current_user, params[:agent_id]).execute + agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) + agent_tokens = ::Clusters::AgentTokensFinder.new(agent, current_user).execute present paginate(agent_tokens), with: Entities::Clusters::AgentTokenBasic end @@ -42,8 +43,7 @@ module API end get ':token_id' do agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) - - token = agent.agent_tokens.find(params[:token_id]) + token = ::Clusters::AgentTokensFinder.new(agent, current_user).find(params[:token_id]) present token, with: Entities::Clusters::AgentToken end @@ -84,8 +84,7 @@ module API authorize! :admin_cluster, user_project agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) - - token = agent.agent_tokens.find(params[:token_id]) + token = ::Clusters::AgentTokensFinder.new(agent, current_user).find(params[:token_id]) # Skipping explicit error handling and relying on exceptions token.revoked! diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 954b572c9b1..531235dc9b2 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -110,13 +110,23 @@ module API authorize! :update_pipeline, pipeline + # rubocop: disable Performance/ActiveRecordSubtransactionMethods + stage = pipeline.stages.safe_find_or_create_by!(name: 'external') do |stage| + stage.position = GenericCommitStatus::EXTERNAL_STAGE_IDX + stage.project = pipeline.project + end + # rubocop: enable Performance/ActiveRecordSubtransactionMethods + status = GenericCommitStatus.running_or_pending.find_or_initialize_by( project: user_project, pipeline: pipeline, name: name, ref: ref, user: current_user, - protected: user_project.protected_for?(ref) + protected: user_project.protected_for?(ref), + ci_stage: stage, + stage_idx: stage.position, + stage: 'external' ) updatable_optional_attributes = %w[target_url description coverage] @@ -152,7 +162,7 @@ module API def all_matching_pipelines pipelines = user_project.ci_pipelines.newest_first(sha: commit.sha) pipelines = pipelines.for_ref(params[:ref]) if params[:ref] - pipelines = pipelines.for_id(params[:pipeline_id]) if params[:pipeline_id] + pipelines = pipelines.id_in(params[:pipeline_id]) if params[:pipeline_id] pipelines end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 63a13b83a9b..ad2bbf90917 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -24,6 +24,26 @@ module API forbidden!("You are not allowed to push into this branch") end end + + def track_commit_events + return unless find_user_from_warden + + Gitlab::UsageDataCounters::WebIdeCounter.increment_commits_count + Gitlab::UsageDataCounters::EditorUniqueCounter.track_web_ide_edit_action(author: current_user, project: user_project) + namespace = user_project.namespace + + return unless Feature.enabled?(:route_hll_to_snowplow_phase3, namespace) + + Gitlab::Tracking.event( + 'API::Commits', + :commit, + project: user_project, + namespace: namespace, + user: current_user, + label: 'counts.web_ide_commits', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis, key_path: 'counts.web_ide_commits').to_context] + ) + end end params do @@ -204,10 +224,7 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - if find_user_from_warden - Gitlab::UsageDataCounters::WebIdeCounter.increment_commits_count - Gitlab::UsageDataCounters::EditorUniqueCounter.track_web_ide_edit_action(author: current_user, project: user_project) - end + track_commit_events present commit_detail, with: Entities::CommitDetail, include_stats: params[:stats], current_user: current_user else diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index d9806fa37d1..3819e6d236d 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -61,32 +61,56 @@ module API end params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of a group' end resource :group, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do + after_validation do user_group end - desc 'Composer packages endpoint at group level' + desc 'Composer packages endpoint at group level' do + detail 'This feature was introduced in GitLab 13.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[composer_packages] + end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/packages', urgency: :low do presenter.root end - desc 'Composer packages endpoint at group level for packages list' + desc 'Composer packages endpoint at group level for packages list' do + detail 'This feature was introduced in GitLab 13.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[composer_packages] + end params do - requires :sha, type: String, desc: 'Shasum of current json' + requires :sha, type: String, desc: 'Shasum of current json', documentation: { example: '673594f85a55fe3c0eb45df7bd2fa9d95a1601ab' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/p/:sha', urgency: :low do presenter.provider end - desc 'Composer v2 packages p2 endpoint at group level for package versions metadata' + desc 'Composer v2 packages p2 endpoint at group level for package versions metadata' do + detail 'This feature was introduced in GitLab 13.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[composer_packages] + end params do - requires :package_name, type: String, file_path: true, desc: 'The Composer package name' + requires :package_name, type: String, file_path: true, desc: 'The Composer package name', documentation: { example: 'my-composer-package' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/p2/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true, urgency: :low do @@ -95,9 +119,17 @@ module API presenter.package_versions end - desc 'Composer packages endpoint at group level for package versions metadata' + desc 'Composer packages endpoint at group level for package versions metadata' do + detail 'This feature was introduced in GitLab 12.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[composer_packages] + end params do - requires :package_name, type: String, file_path: true, desc: 'The Composer package name' + requires :package_name, type: String, file_path: true, desc: 'The Composer package name', documentation: { example: 'my-composer-package' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true, urgency: :low do @@ -109,17 +141,27 @@ module API end params do - requires :id, type: Integer, desc: 'The ID of a project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Composer packages endpoint for registering packages' namespace ':id/packages/composer' do route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true + desc 'Composer packages endpoint for registering packages' do + detail 'This feature was introduced in GitLab 13.1' + success code: 201 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[composer_packages] + end params do - optional :branch, type: String, desc: 'The name of the branch' - optional :tag, type: String, desc: 'The name of the tag' + optional :branch, type: String, desc: 'The name of the branch', documentation: { example: 'release' } + optional :tag, type: String, desc: 'The name of the tag', documentation: { example: 'v1.0.0' } exactly_one_of :tag, :branch end post urgency: :low do @@ -142,15 +184,25 @@ module API created! end + desc 'Composer package endpoint to download a package archive' do + detail 'This feature was introduced in GitLab 13.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[composer_packages] + end params do - requires :sha, type: String, desc: 'Shasum of current json' - requires :package_name, type: String, file_path: true, desc: 'The Composer package name' + requires :sha, type: String, desc: 'Shasum of current json', documentation: { example: '673594f85a55fe3c0eb45df7bd2fa9d95a1601ab' } + requires :package_name, type: String, file_path: true, desc: 'The Composer package name', documentation: { example: 'my-composer-package' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get 'archives/*package_name', urgency: :default do - authorize_read_package!(authorized_user_project) + project = authorized_user_project(action: :read_package) - package = authorized_user_project + package = project .packages .composer .with_name(params[:package_name]) @@ -160,10 +212,10 @@ module API not_found! unless metadata - track_package_event('pull_package', :composer, project: authorized_user_project, namespace: authorized_user_project.namespace) + track_package_event('pull_package', :composer, project: project, namespace: project.namespace) package.touch_last_downloaded_at - send_git_archive authorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true + send_git_archive project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true end end end diff --git a/lib/api/conan_project_packages.rb b/lib/api/conan_project_packages.rb index 636b5dca5ed..e282443e85c 100644 --- a/lib/api/conan_project_packages.rb +++ b/lib/api/conan_project_packages.rb @@ -4,7 +4,7 @@ module API class ConanProjectPackages < ::API::Base params do - requires :id, type: Integer, desc: 'The ID of a project', regexp: %r{\A[1-9]\d*\z} + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index fdbffb1689b..e65e8f8710c 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -53,6 +53,11 @@ module API desc 'Ping the Conan API' do detail 'This feature was introduced in GitLab 12.2' + success code: 200 + failure [ + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -63,10 +68,15 @@ module API desc 'Search for packages' do detail 'This feature was introduced in GitLab 12.4' + success code: 200 + failure [ + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end params do - requires :q, type: String, desc: 'Search query' + requires :q, type: String, desc: 'Search query', documentation: { example: 'Hello*' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -86,6 +96,12 @@ module API desc 'Authenticate user against conan CLI' do detail 'This feature was introduced in GitLab 12.2' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -98,6 +114,12 @@ module API desc 'Check for valid user credentials per conan CLI' do detail 'This feature was introduced in GitLab 12.4' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -109,10 +131,10 @@ module API end params do - requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' - requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' - requires :package_username, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package username' - requires :package_channel, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package channel' + requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name', documentation: { example: 'my-package' } + requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version', documentation: { example: '1.0' } + requires :package_username, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package username', documentation: { example: 'my-group+my-project' } + requires :package_channel, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package channel', documentation: { example: 'stable' } end namespace 'conans/:package_name/:package_version/:package_username/:package_channel', requirements: PACKAGE_REQUIREMENTS do after_validation do @@ -122,14 +144,21 @@ module API # Get the snapshot # # the snapshot is a hash of { filename: md5 hash } - # md5 hash is the has of that file. This hash is used to diff the files existing on the client + # md5 hash is the hash of that file. This hash is used to diff the files existing on the client # to determine which client files need to be uploaded if no recipe exists the snapshot is empty desc 'Package Snapshot' do detail 'This feature was introduced in GitLab 12.5' + success code: 200, model: ::API::Entities::ConanPackage::ConanPackageSnapshot + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' + requires :conan_package_reference, type: String, desc: 'Conan package ID', documentation: { example: '103f6067a947f366ef91fc1b7da351c588d1827f' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -149,6 +178,13 @@ module API desc 'Recipe Snapshot' do detail 'This feature was introduced in GitLab 12.5' + success code: 200, model: ::API::Entities::ConanPackage::ConanRecipeSnapshot + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -168,9 +204,16 @@ module API # where the url is the download url for the file desc 'Package Digest' do detail 'This feature was introduced in GitLab 12.5' + success code: 200, model: ::API::Entities::ConanPackage::ConanPackageManifest + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' + requires :conan_package_reference, type: String, desc: 'Conan package ID', documentation: { example: '103f6067a947f366ef91fc1b7da351c588d1827f' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -181,6 +224,13 @@ module API desc 'Recipe Digest' do detail 'This feature was introduced in GitLab 12.5' + success code: 200, model: ::API::Entities::ConanPackage::ConanRecipeManifest + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -197,10 +247,17 @@ module API # where the url is the download url for the file desc 'Package Download Urls' do detail 'This feature was introduced in GitLab 12.5' + success code: 200, model: ::API::Entities::ConanPackage::ConanPackageManifest + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' + requires :conan_package_reference, type: String, desc: 'Conan package ID', documentation: { example: '103f6067a947f366ef91fc1b7da351c588d1827f' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -211,6 +268,13 @@ module API desc 'Recipe Download Urls' do detail 'This feature was introduced in GitLab 12.5' + success code: 200, model: ::API::Entities::ConanPackage::ConanRecipeManifest + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -228,10 +292,17 @@ module API # where the url is the upload url for the file that the conan client will use desc 'Package Upload Urls' do detail 'This feature was introduced in GitLab 12.4' + success code: 200, model: ::API::Entities::ConanPackage::ConanUploadUrls + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' + requires :conan_package_reference, type: String, desc: 'Conan package ID', documentation: { example: '103f6067a947f366ef91fc1b7da351c588d1827f' } end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -245,6 +316,13 @@ module API desc 'Recipe Upload Urls' do detail 'This feature was introduced in GitLab 12.4' + success code: 200, model: ::API::Entities::ConanPackage::ConanUploadUrls + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -258,6 +336,13 @@ module API desc 'Delete Package' do detail 'This feature was introduced in GitLab 12.5' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -272,11 +357,11 @@ module API end params do - requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' - requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' - requires :package_username, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package username' - requires :package_channel, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package channel' - requires :recipe_revision, type: String, regexp: CONAN_REVISION_REGEX, desc: 'Conan Recipe Revision' + requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name', documentation: { example: 'my-package' } + requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version', documentation: { example: '1.0' } + requires :package_username, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package username', documentation: { example: 'my-group+my-project' } + requires :package_channel, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package channel', documentation: { example: 'stable' } + requires :recipe_revision, type: String, regexp: CONAN_REVISION_REGEX, desc: 'Conan Recipe Revision', documentation: { example: '0' } end namespace 'files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision', requirements: PACKAGE_REQUIREMENTS do before do @@ -288,12 +373,19 @@ module API end params do - requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES + requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES, documentation: { example: 'conanfile.py' } end namespace 'export/:file_name', requirements: FILE_NAME_REQUIREMENTS do desc 'Download recipe files' do detail 'This feature was introduced in GitLab 12.6' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -304,6 +396,14 @@ module API desc 'Upload recipe package files' do detail 'This feature was introduced in GitLab 12.6' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end params do @@ -318,6 +418,14 @@ module API desc 'Workhorse authorize the conan recipe file' do detail 'This feature was introduced in GitLab 12.6' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -328,13 +436,19 @@ module API end params do - requires :conan_package_reference, type: String, desc: 'Conan Package ID' - requires :package_revision, type: String, desc: 'Conan Package Revision' - requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES + requires :conan_package_reference, type: String, desc: 'Conan Package ID', documentation: { example: '103f6067a947f366ef91fc1b7da351c588d1827f' } + requires :package_revision, type: String, desc: 'Conan Package Revision', documentation: { example: '0' } + requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES, documentation: { example: 'conaninfo.txt' } end namespace 'package/:conan_package_reference/:package_revision/:file_name', requirements: FILE_NAME_REQUIREMENTS do desc 'Download package files' do detail 'This feature was introduced in GitLab 12.5' + success code: 200 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -345,6 +459,14 @@ module API desc 'Workhorse authorize the conan package file' do detail 'This feature was introduced in GitLab 12.6' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true @@ -355,6 +477,14 @@ module API desc 'Upload package files' do detail 'This feature was introduced in GitLab 12.6' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[conan_packages] end params do diff --git a/lib/api/concerns/packages/debian_distribution_endpoints.rb b/lib/api/concerns/packages/debian_distribution_endpoints.rb index 380966136df..76b996f2301 100644 --- a/lib/api/concerns/packages/debian_distribution_endpoints.rb +++ b/lib/api/concerns/packages/debian_distribution_endpoints.rb @@ -25,21 +25,23 @@ module API namespace 'debian_distributions' do helpers do params :optional_distribution_params do - optional :suite, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Suite' - optional :origin, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Origin' - optional :label, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Label' - optional :version, type: String, regexp: Gitlab::Regex.debian_version_regex, desc: 'The Debian Version' - optional :description, type: String, desc: 'The Debian Description' - optional :valid_time_duration_seconds, type: Integer, desc: 'The duration before the Release file should be considered expired by the client' + optional :suite, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Suite', documentation: { example: 'unstable' } + optional :origin, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Origin', documentation: { example: 'Grep' } + optional :label, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Label', documentation: { example: 'grep.be' } + optional :version, type: String, regexp: Gitlab::Regex.debian_version_regex, desc: 'The Debian Version', documentation: { example: '12' } + optional :description, type: String, desc: 'The Debian Description', documentation: { example: 'My description' } + optional :valid_time_duration_seconds, type: Integer, desc: 'The duration before the Release file should be considered expired by the client', documentation: { example: 604800 } optional :components, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, regexp: Gitlab::Regex.debian_component_regex, - desc: 'The list of Components' + desc: 'The list of Components', + documentation: { example: 'main' } optional :architectures, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, regexp: Gitlab::Regex.debian_architecture_regex, - desc: 'The list of Architectures' + desc: 'The list of Architectures', + documentation: { example: 'amd64' } end end @@ -63,11 +65,18 @@ module API # POST {projects|groups}/:id/debian_distributions desc 'Create a Debian Distribution' do detail 'This feature was introduced in 14.0' - success ::API::Entities::Packages::Debian::Distribution + success code: 201, model: ::API::Entities::Packages::Debian::Distribution + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_distribution] end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } use :optional_distribution_params end post '/' do @@ -87,12 +96,18 @@ module API # GET {projects|groups}/:id/debian_distributions desc 'Get a list of Debian Distributions' do detail 'This feature was introduced in 14.0' - success ::API::Entities::Packages::Debian::Distribution + success code: 200, model: ::API::Entities::Packages::Debian::Distribution + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_distribution] end params do use :pagination - optional :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + optional :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } use :optional_distribution_params end get '/' do @@ -107,11 +122,17 @@ module API # GET {projects|groups}/:id/debian_distributions/:codename desc 'Get a Debian Distribution' do detail 'This feature was introduced in 14.0' - success ::API::Entities::Packages::Debian::Distribution + success code: 200, model: ::API::Entities::Packages::Debian::Distribution + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_distribution] end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } end get '/:codename' do authorize_read_package!(project_or_group) @@ -122,11 +143,17 @@ module API # GET {projects|groups}/:id/debian_distributions/:codename/key desc 'Get a Debian Distribution Key' do detail 'This feature was introduced in 14.4' - success ::API::Entities::Packages::Debian::Distribution + success code: 200, model: ::API::Entities::Packages::Debian::Distribution + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_distribution] end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } end get '/:codename/key.asc' do authorize_read_package!(project_or_group) @@ -141,11 +168,18 @@ module API # PUT {projects|groups}/:id/debian_distributions/:codename desc 'Update a Debian Distribution' do detail 'This feature was introduced in 14.0' - success ::API::Entities::Packages::Debian::Distribution + success code: 200, model: ::API::Entities::Packages::Debian::Distribution + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_distribution] end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } use :optional_distribution_params end put '/:codename' do @@ -165,10 +199,18 @@ module API # DELETE {projects|groups}/:id/debian_distributions/:codename desc 'Delete a Debian Distribution' do detail 'This feature was introduced in 14.0' + success code: 202 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_distribution] end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } use :optional_distribution_params end delete '/:codename' do diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index 2883944a745..842250d351b 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -24,11 +24,11 @@ module API helpers do params :shared_package_file_params do - requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex - requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' - requires :package_name, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex - requires :package_version, type: String, desc: 'The Debian Source Package Version', regexp: Gitlab::Regex.debian_version_regex - requires :file_name, type: String, desc: 'The Debian File Name' + requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex, documentation: { example: 'my-distro' } + requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)', documentation: { example: 'a' } + requires :package_name, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex, documentation: { example: 'my-pkg' } + requires :package_version, type: String, desc: 'The Debian Source Package Version', regexp: Gitlab::Regex.debian_version_regex, documentation: { example: '1.0.0' } + requires :file_name, type: String, desc: 'The Debian File Name', documentation: { example: 'example_1.0.0~alpha2_amd64.deb' } end def distribution_from!(container) @@ -79,7 +79,7 @@ module API content_type :txt, 'text/plain' params do - requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex + requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex, documentation: { example: 'my-distro' } end namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do @@ -87,6 +87,14 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files desc 'The Release file signature' do detail 'This feature was introduced in GitLab 13.5' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -98,6 +106,14 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files desc 'The unsigned Release file' do detail 'This feature was introduced in GitLab 13.5' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -109,6 +125,14 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files desc 'The signed Release file' do detail 'This feature was introduced in GitLab 13.5' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -117,12 +141,12 @@ module API end params do - requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex, documentation: { example: 'main' } end namespace ':component', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do params do - requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex, documentation: { example: 'binary-amd64' } end namespace 'debian-installer/binary-:architecture' do @@ -130,6 +154,14 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices desc 'The installer (udeb) binary files index' do detail 'This feature was introduced in GitLab 15.4' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -141,6 +173,14 @@ module API # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 desc 'The installer (udeb) binary files index by hash' do detail 'This feature was introduced in GitLab 15.4' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -154,6 +194,14 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Sources.22_Indices desc 'The source files index' do detail 'This feature was introduced in GitLab 15.4' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -165,6 +213,14 @@ module API # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 desc 'The source files index by hash' do detail 'This feature was introduced in GitLab 15.4' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -174,7 +230,7 @@ module API end params do - requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex, documentation: { example: 'binary-amd64' } end namespace 'binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do @@ -182,6 +238,14 @@ module API # https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices desc 'The binary files index' do detail 'This feature was introduced in GitLab 13.5' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -193,6 +257,14 @@ module API # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 desc 'The binary files index by hash' do detail 'This feature was introduced in GitLab 15.4' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index 4cc680068b6..f26b3a1d8c2 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -26,13 +26,39 @@ module API authenticate_non_get! end + helpers do + def redirect_or_present_audit_report + redirect_registry_request( + forward_to_registry: true, + package_type: :npm, + path: options[:path][0], + body: Gitlab::Json.dump(request.POST), + target: project_or_nil, + method: route.request_method + ) do + authorize_read_package!(project) + + status :ok + present [] + end + end + end + params do requires :package_name, type: String, desc: 'Package name' end namespace '-/package/*package_name' do desc 'Get all tags for a given an NPM package' do detail 'This feature was introduced in GitLab 12.7' - success ::API::Entities::NpmPackageTag + success [ + { code: 200, model: ::API::Entities::NpmPackageTag } + ] + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[npm_packages] end get 'dist-tags', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do package_name = params[:package_name] @@ -56,6 +82,14 @@ module API namespace 'dist-tags/:tag', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do desc 'Create or Update the given tag for the given NPM package and version' do detail 'This feature was introduced in GitLab 12.7' + success code: 204 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[npm_packages] end put format: false do package_name = params[:package_name] @@ -79,6 +113,14 @@ module API desc 'Deletes the given tag' do detail 'This feature was introduced in GitLab 12.7' + success code: 204 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[npm_packages] end delete format: false do package_name = params[:package_name] @@ -104,6 +146,16 @@ module API desc 'NPM registry metadata endpoint' do detail 'This feature was introduced in GitLab 11.8' + success [ + { code: 200, model: ::API::Entities::NpmPackage, message: 'Ok' }, + { code: 302, message: 'Found (redirect)' } + ] + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[npm_packages] end params do requires :package_name, type: String, desc: 'Package name' @@ -130,6 +182,44 @@ module API with: ::API::Entities::NpmPackage end end + + desc 'NPM registry bulk advisory endpoint' do + detail 'This feature was introduced in GitLab 15.6' + success [ + { code: 200, message: 'Ok' }, + { code: 307, message: 'Temporary Redirect' } + ] + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + is_array true + tags %w[npm_packages] + end + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + post '-/npm/v1/security/advisories/bulk' do + redirect_or_present_audit_report + end + + desc 'NPM registry quick audit endpoint' do + detail 'This feature was introduced in GitLab 15.6' + success [ + { code: 200, message: 'Ok' }, + { code: 307, message: 'Temporary Redirect' } + ] + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + is_array true + tags %w[npm_packages] + end + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + post '-/npm/v1/security/audits/quick' do + redirect_or_present_audit_report + end end end end diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb index e0328e488c6..31ecb529c3c 100644 --- a/lib/api/concerns/packages/nuget_endpoints.rb +++ b/lib/api/concerns/packages/nuget_endpoints.rb @@ -55,6 +55,13 @@ module API # https://docs.microsoft.com/en-us/nuget/api/service-index desc 'The NuGet Service Index' do detail 'This feature was introduced in GitLab 12.6' + success code: 200, model: ::API::Entities::Nuget::ServiceIndex + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end get 'index', format: :json, urgency: :default do authorize_read_package!(project_or_group) @@ -67,7 +74,7 @@ module API # https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource params do - requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX + requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'MyNuGetPkg' } end namespace '/metadata/*package_name' do after_validation do @@ -76,6 +83,13 @@ module API desc 'The NuGet Metadata Service - Package name level' do detail 'This feature was introduced in GitLab 12.8' + success code: 200, model: ::API::Entities::Nuget::PackagesMetadata + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end get 'index', format: :json, urgency: :low do present ::Packages::Nuget::PackagesMetadataPresenter.new(find_packages(params[:package_name])), @@ -84,9 +98,16 @@ module API desc 'The NuGet Metadata Service - Package name and version level' do detail 'This feature was introduced in GitLab 12.8' + success code: 200, model: ::API::Entities::Nuget::PackageMetadata + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end params do - requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX + requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: '1.0.0' } end get '*package_version', format: :json, urgency: :low do present ::Packages::Nuget::PackageMetadataPresenter.new(find_package(params[:package_name], params[:package_version])), @@ -96,9 +117,9 @@ module API # https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource params do - optional :q, type: String, desc: 'The search term' - optional :skip, type: Integer, desc: 'The number of results to skip', default: 0, regexp: NON_NEGATIVE_INTEGER_REGEX - optional :take, type: Integer, desc: 'The number of results to return', default: Kaminari.config.default_per_page, regexp: POSITIVE_INTEGER_REGEX + optional :q, type: String, desc: 'The search term', documentation: { example: 'MyNuGet' } + optional :skip, type: Integer, desc: 'The number of results to skip', default: 0, regexp: NON_NEGATIVE_INTEGER_REGEX, documentation: { example: 1 } + optional :take, type: Integer, desc: 'The number of results to return', default: Kaminari.config.default_per_page, regexp: POSITIVE_INTEGER_REGEX, documentation: { example: 1 } optional :prerelease, type: ::Grape::API::Boolean, desc: 'Include prerelease versions', default: true end namespace '/query' do @@ -108,6 +129,13 @@ module API desc 'The NuGet Search Service' do detail 'This feature was introduced in GitLab 12.8' + success code: 200, model: ::API::Entities::Nuget::SearchResults + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end get format: :json, urgency: :low do search_options = { diff --git a/lib/api/container_registry_event.rb b/lib/api/container_registry_event.rb index 9acf2fca1b3..9e59401ddf6 100644 --- a/lib/api/container_registry_event.rb +++ b/lib/api/container_registry_event.rb @@ -26,15 +26,21 @@ module API desc 'Receives notifications from the container registry when an operation occurs' do detail 'This feature was introduced in GitLab 12.10' consumes [:json, DOCKER_DISTRIBUTION_EVENTS_V1_JSON] + success code: 200, message: 'Success' + failure [ + { code: 401, message: 'Invalid Token' } + ] + tags %w[container_registry_event] end params do requires :events, type: Array, desc: 'Event notifications' do requires :action, type: String, desc: 'The action to perform, `push`, `delete`', values: %w[push delete].freeze optional :target, type: Hash, desc: 'The target of the action' do - optional :tag, type: String, desc: 'The target tag' - optional :repository, type: String, desc: 'The target repository' - optional :digest, type: String, desc: 'Unique identifier for target image manifest' + optional :tag, type: String, desc: 'The target tag', documentation: { example: 'latest' } + optional :repository, type: String, desc: 'The target repository', documentation: { example: 'group/p1' } + optional :digest, type: String, desc: 'Unique identifier for target image manifest', + documentation: { example: 'imagedigest' } end end end diff --git a/lib/api/container_repositories.rb b/lib/api/container_repositories.rb index f2dd1fa21fd..b6b5fe10332 100644 --- a/lib/api/container_repositories.rb +++ b/lib/api/container_repositories.rb @@ -14,12 +14,17 @@ module API namespace 'registry' do params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' + requires :id, types: [String, Integer], desc: 'The ID of the repository' end resource :repositories, requirements: { id: /[0-9]*/ } do desc 'Get a container repository' do detail 'This feature was introduced in GitLab 13.6.' success Entities::ContainerRegistry::Repository + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Repository Not Found' } + ] + tags %w[container_registry] end params do optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 0962d749558..105a0955912 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -30,7 +30,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, types: [String, Integer], desc: 'The group ID or full group path.' end namespace ':id/-/packages/debian' do @@ -42,8 +42,15 @@ module API use :shared_package_file_params end - desc 'The package' do + desc 'Download Debian package' do detail 'This feature was introduced in GitLab 14.2' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index df3b6e774ae..23a542e4183 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -45,8 +45,15 @@ module API use :shared_package_file_params end - desc 'The package' do + desc 'Download Debian package' do detail 'This feature was introduced in GitLab 14.2' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] end route_setting :authentication, authenticate_non_public: true @@ -55,13 +62,25 @@ module API end params do - requires :file_name, type: String, desc: 'The file name' + requires :file_name, type: String, desc: 'The file name', documentation: { example: 'example_1.0.0~alpha2_amd64.deb' } end namespace ':file_name', requirements: FILE_NAME_REQUIREMENTS do format :txt content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + desc 'Upload Debian package' do + detail 'This feature was introduced in GitLab 14.0' + success code: 201 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] + end + # PUT {projects|groups}/:id/packages/debian/:file_name params do requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } @@ -91,6 +110,16 @@ module API end # PUT {projects|groups}/:id/packages/debian/:file_name/authorize + desc 'Authorize Debian package upload' do + detail 'This feature was introduced in GitLab 13.5' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[debian_packages] + end route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true put 'authorize' do authorize_workhorse!( diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 141f089b5e1..3a0eea677b8 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -50,6 +50,14 @@ module API type: DateTime, desc: 'Return deployments updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`)' + optional :finished_after, + type: DateTime, + desc: 'Return deployments finished after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`)' + + optional :finished_before, + type: DateTime, + desc: 'Return deployments finished before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`)' + optional :environment, type: String, desc: 'The name of the environment to filter deployments by' @@ -64,7 +72,7 @@ module API authorize! :read_deployment, user_project deployments = - DeploymentsFinder.new(params.merge(project: user_project)) + DeploymentsFinder.new(declared_params(include_missing: false).merge(project: user_project)) .execute.with_api_entity_associations present paginate(deployments), with: Entities::Deployment diff --git a/lib/api/entities/appearance.rb b/lib/api/entities/appearance.rb index a09faf55f48..94a39568393 100644 --- a/lib/api/entities/appearance.rb +++ b/lib/api/entities/appearance.rb @@ -4,6 +4,7 @@ module API module Entities class Appearance < Grape::Entity expose :title + expose :short_title expose :description expose :logo do |appearance, options| diff --git a/lib/api/entities/basic_success.rb b/lib/api/entities/basic_success.rb new file mode 100644 index 00000000000..37388f56221 --- /dev/null +++ b/lib/api/entities/basic_success.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + # Simple representation for endpoints that returns a trivial success response. + class BasicSuccess < Grape::Entity + expose :success, documentation: { type: 'boolean' } do + true + end + end + end +end diff --git a/lib/api/entities/batched_background_migration.rb b/lib/api/entities/batched_background_migration.rb index eba17ff98f4..08e4681e0aa 100644 --- a/lib/api/entities/batched_background_migration.rb +++ b/lib/api/entities/batched_background_migration.rb @@ -3,12 +3,12 @@ module API module Entities class BatchedBackgroundMigration < Grape::Entity - expose :id - expose :job_class_name - expose :table_name - expose :status, &:status_name - expose :progress - expose :created_at + expose :id, documentation: { type: :string, example: "1234" } + expose :job_class_name, documentation: { type: :string, example: "CopyColumnUsingBackgroundMigrationJob" } + expose :table_name, documentation: { type: :string, example: "events" } + expose :status_name, as: :status, override: true, documentation: { type: :string, example: "active" } + expose :progress, documentation: { type: :float, example: 50 } + expose :created_at, documentation: { type: :dateTime, example: "2022-11-28T16:26:39+02:00" } end end end diff --git a/lib/api/entities/ci/job_request/hook.rb b/lib/api/entities/ci/job_request/hook.rb new file mode 100644 index 00000000000..2d155bb1c45 --- /dev/null +++ b/lib/api/entities/ci/job_request/hook.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Hook < Grape::Entity + expose :name, :script + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/response.rb b/lib/api/entities/ci/job_request/response.rb index 9de415ebacb..cfdbeed79b6 100644 --- a/lib/api/entities/ci/job_request/response.rb +++ b/lib/api/entities/ci/job_request/response.rb @@ -23,6 +23,9 @@ module API expose :runner_variables, as: :variables expose :steps, using: Entities::Ci::JobRequest::Step + expose :runtime_hooks, as: :hooks, + using: Entities::Ci::JobRequest::Hook, + if: ->(job) { ::Feature.enabled?(:ci_hooks_pre_get_sources_script, job.project) } expose :image, using: Entities::Ci::JobRequest::Image expose :services, using: Entities::Ci::JobRequest::Service expose :artifacts, using: Entities::Ci::JobRequest::Artifacts diff --git a/lib/api/entities/ci/runner_details.rb b/lib/api/entities/ci/runner_details.rb index 9b1decca274..8aa134dc669 100644 --- a/lib/api/entities/ci/runner_details.rb +++ b/lib/api/entities/ci/runner_details.rb @@ -14,7 +14,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord expose :projects, with: Entities::BasicProjectDetails do |runner, options| - if options[:current_user].admin? # rubocop: disable Cop/UserAdmin + if options[:current_user].can_read_all_resources? runner.projects else options[:current_user].authorized_projects.where(id: runner.runner_projects.pluck(:project_id)) @@ -23,7 +23,7 @@ module API # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord expose :groups, with: Entities::BasicGroupDetails do |runner, options| - if options[:current_user].admin? # rubocop: disable Cop/UserAdmin + if options[:current_user].can_read_all_resources? runner.groups else options[:current_user].authorized_groups.where(id: runner.runner_namespaces.pluck(:namespace_id)) diff --git a/lib/api/entities/ci/secure_file.rb b/lib/api/entities/ci/secure_file.rb index d957e4488fd..a234ada6f82 100644 --- a/lib/api/entities/ci/secure_file.rb +++ b/lib/api/entities/ci/secure_file.rb @@ -4,13 +4,14 @@ module API module Entities module Ci class SecureFile < Grape::Entity - expose :id - expose :name - expose :checksum - expose :checksum_algorithm - expose :created_at - expose :expires_at - expose :metadata + expose :id, documentation: { type: 'integer', example: 123 } + expose :name, documentation: { type: 'string', example: 'upload-keystore.jks' } + expose :checksum, +documentation: { type: 'string', example: '16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac' } + expose :checksum_algorithm, documentation: { type: 'string', example: 'sha256' } + expose :created_at, documentation: { type: 'dateTime', example: '2022-02-22T22:22:22.222Z' } + expose :expires_at, documentation: { type: 'dateTime', example: '2022-09-21T14:56:00.000Z' } + expose :metadata, documentation: { type: 'Hash', example: { "id" => "75949910542696343243264405377658443914" } } end end end diff --git a/lib/api/entities/commit_signature.rb b/lib/api/entities/commit_signature.rb index 9430dd5e2a2..9c30c3c59ea 100644 --- a/lib/api/entities/commit_signature.rb +++ b/lib/api/entities/commit_signature.rb @@ -10,6 +10,8 @@ module API ::API::Entities::GpgCommitSignature.represent commit_signature(commit), options elsif commit.signature.is_a?(::CommitSignatures::X509CommitSignature) ::API::Entities::X509Signature.represent commit.signature, options + elsif commit.signature.is_a?(::CommitSignatures::SshSignature) + ::API::Entities::SshSignature.represent(commit.signature, options) end end diff --git a/lib/api/entities/conan_package/conan_package_manifest.rb b/lib/api/entities/conan_package/conan_package_manifest.rb index e6acfe1912f..70ab498c56a 100644 --- a/lib/api/entities/conan_package/conan_package_manifest.rb +++ b/lib/api/entities/conan_package/conan_package_manifest.rb @@ -4,7 +4,7 @@ module API module Entities module ConanPackage class ConanPackageManifest < Grape::Entity - expose :package_urls, merge: true + expose :package_urls, merge: true, documentation: { type: 'object', example: '{ "conan_package.tgz": "https://gitlab.example.com/api/v4/packages/conan/v1/files/my-package/1.0/my-group+my-project/stable/packages/103f6067a947f366ef91fc1b7da351c588d1827f/0/conan_package.tgz"' } end end end diff --git a/lib/api/entities/conan_package/conan_package_snapshot.rb b/lib/api/entities/conan_package/conan_package_snapshot.rb index d7fdda09b5a..5cf623c53df 100644 --- a/lib/api/entities/conan_package/conan_package_snapshot.rb +++ b/lib/api/entities/conan_package/conan_package_snapshot.rb @@ -4,7 +4,11 @@ module API module Entities module ConanPackage class ConanPackageSnapshot < Grape::Entity - expose :package_snapshot, merge: true + expose :package_snapshot, merge: true, + documentation: { + type: 'object', + example: '{ "conan_package.tgz": "749b29bdf72587081ca03ec033ee59dc" }' + } end end end diff --git a/lib/api/entities/conan_package/conan_recipe_manifest.rb b/lib/api/entities/conan_package/conan_recipe_manifest.rb index ecaa142cef9..0b29f0c5058 100644 --- a/lib/api/entities/conan_package/conan_recipe_manifest.rb +++ b/lib/api/entities/conan_package/conan_recipe_manifest.rb @@ -4,7 +4,7 @@ module API module Entities module ConanPackage class ConanRecipeManifest < Grape::Entity - expose :recipe_urls, merge: true + expose :recipe_urls, merge: true, documentation: { type: 'object', example: '{ "conan_sources.tgz": "https://gitlab.example.com/api/v4/packages/conan/v1/files/my-package/1.0/my-group+my-project/stable/0/export/conan_sources.tgz" }' } end end end diff --git a/lib/api/entities/conan_package/conan_recipe_snapshot.rb b/lib/api/entities/conan_package/conan_recipe_snapshot.rb index 09a60d23727..f9806e90816 100644 --- a/lib/api/entities/conan_package/conan_recipe_snapshot.rb +++ b/lib/api/entities/conan_package/conan_recipe_snapshot.rb @@ -4,7 +4,11 @@ module API module Entities module ConanPackage class ConanRecipeSnapshot < Grape::Entity - expose :recipe_snapshot, merge: true + expose :recipe_snapshot, merge: true, + documentation: { + type: 'object', + example: '{ "conan_sources.tgz": "eadf19b33f4c3c7e113faabf26e76277" }' + } end end end diff --git a/lib/api/entities/conan_package/conan_upload_urls.rb b/lib/api/entities/conan_package/conan_upload_urls.rb index c14963c87f5..fd5ea80068c 100644 --- a/lib/api/entities/conan_package/conan_upload_urls.rb +++ b/lib/api/entities/conan_package/conan_upload_urls.rb @@ -4,7 +4,7 @@ module API module Entities module ConanPackage class ConanUploadUrls < Grape::Entity - expose :upload_urls, merge: true + expose :upload_urls, merge: true, documentation: { type: 'object', example: '{ "conan_package.tgz": "https://gitlab.example.com/api/v4/packages/conan/v1/files/my-package/1.0/my-group+my-project/stable/0/package/103f6067a947f366ef91fc1b7da351c588d1827f/0/conan_package.tgz" }' } end end end diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index d12c8142e69..cadd45cb0eb 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -4,9 +4,9 @@ module API module Entities module ContainerRegistry class Tag < Grape::Entity - expose :name - expose :path - expose :location + expose :name, documentation: { type: 'string', example: 'latest' } + expose :path, documentation: { type: 'string', example: 'namespace1/project1/test_image_1:latest' } + expose :location, documentation: { type: 'string', example: 'registry.dev/namespace1/project1/test_image_1:latest' } end class Repository < Grape::Entity @@ -19,10 +19,11 @@ module API expose :location, documentation: { type: 'string', example: 'gitlab.example.com/group/project/releases' } expose :created_at, documentation: { type: 'dateTime', example: '2019-01-10T13:39:08.229Z' } expose :expiration_policy_started_at, as: :cleanup_policy_started_at, documentation: { type: 'dateTime', example: '2020-08-17T03:12:35.489Z' } - expose :tags_count, if: -> (_, options) { options[:tags_count] } + expose :tags_count, if: -> (_, options) { options[:tags_count] }, documentation: { type: 'integer', example: 3 } expose :tags, using: Tag, if: -> (_, options) { options[:tags] } - expose :delete_api_path, if: ->(object, options) { Ability.allowed?(options[:user], :admin_container_image, object) } - expose :size, if: -> (_, options) { options[:size] } + expose :delete_api_path, if: ->(object, options) { Ability.allowed?(options[:user], :admin_container_image, object) }, + documentation: { type: 'string', example: 'delete/api/path' } + expose :size, if: -> (_, options) { options[:size] }, documentation: { type: 'integer', example: 12345 } private @@ -32,11 +33,11 @@ module API end class TagDetails < Tag - expose :revision - expose :short_revision - expose :digest - expose :created_at - expose :total_size + expose :revision, documentation: { type: 'string', example: 'tagrevision' } + expose :short_revision, documentation: { type: 'string', example: 'shortrevison' } + expose :digest, documentation: { type: 'string', example: 'shadigest' } + expose :created_at, documentation: { type: 'dateTime', example: '2022-01-10T13:39:08.229Z' } + expose :total_size, documentation: { type: 'integer', example: 3 } end end end diff --git a/lib/api/entities/event.rb b/lib/api/entities/event.rb index f750d728e03..e81e89a8393 100644 --- a/lib/api/entities/event.rb +++ b/lib/api/entities/event.rb @@ -3,11 +3,15 @@ module API module Entities class Event < Grape::Entity - expose :id - expose :project_id, :action_name - expose :target_id, :target_iid, :target_type, :author_id - expose :target_title - expose :created_at + expose :id, documentation: { type: 'integer', example: 1 } + expose :project_id, documentation: { type: 'integer', example: 2 } + expose :action_name, documentation: { type: 'string', example: 'closed' } + expose :target_id, documentation: { type: 'integer', example: 160 } + expose :target_iid, documentation: { type: 'integer', example: 157 } + expose :target_type, documentation: { type: 'string', example: 'Issue' } + expose :author_id, documentation: { type: 'integer', example: 25 } + expose :target_title, documentation: { type: 'string', example: 'Public project search field' } + expose :created_at, documentation: { type: 'string', example: '2017-02-09T10:43:19.667Z' } expose :note, using: Entities::Note, if: ->(event, options) { event.note? } expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author } expose :wiki_page, using: Entities::WikiPageBasic, if: ->(event, _options) { event.wiki_page? } @@ -17,7 +21,7 @@ module API using: Entities::PushEventPayload, if: -> (event, _) { event.push_action? } - expose :author_username do |event, options| + expose :author_username, documentation: { type: 'string', example: 'root' } do |event, options| event.author&.username end end diff --git a/lib/api/entities/issuable_references.rb b/lib/api/entities/issuable_references.rb index 1bf078847cf..7b966b85800 100644 --- a/lib/api/entities/issuable_references.rb +++ b/lib/api/entities/issuable_references.rb @@ -3,15 +3,15 @@ module API module Entities class IssuableReferences < Grape::Entity - expose :short do |issuable| + expose :short, documentation: { type: "string", example: "&6" } do |issuable| issuable.to_reference end - expose :relative do |issuable, options| + expose :relative, documentation: { type: "string", example: "&6" } do |issuable, options| issuable.to_reference(options[:group] || options[:project]) end - expose :full do |issuable| + expose :full, documentation: { type: "string", example: "test&6" } do |issuable| issuable.to_reference(full: true) end end diff --git a/lib/api/entities/issuable_time_stats.rb b/lib/api/entities/issuable_time_stats.rb index f93b4651b1f..717d2282441 100644 --- a/lib/api/entities/issuable_time_stats.rb +++ b/lib/api/entities/issuable_time_stats.rb @@ -7,12 +7,12 @@ module API Gitlab::TimeTrackingFormatter.output(time_spent) end - expose :time_estimate - expose :total_time_spent - expose :human_time_estimate + expose :time_estimate, documentation: { type: 'integer', example: 12600 } + expose :total_time_spent, documentation: { type: 'integer', example: 3600 } + expose :human_time_estimate, documentation: { type: 'string', example: '3h 30m' } with_options(format_with: :time_tracking_formatter) do - expose :total_time_spent, as: :human_total_time_spent + expose :total_time_spent, as: :human_total_time_spent, documentation: { type: 'string', example: '1h' } end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/api/entities/metric_image.rb b/lib/api/entities/metric_image.rb index fd5e3a62e40..3e4566832c9 100644 --- a/lib/api/entities/metric_image.rb +++ b/lib/api/entities/metric_image.rb @@ -3,7 +3,13 @@ module API module Entities class MetricImage < Grape::Entity - expose :id, :created_at, :filename, :file_path, :url, :url_text + expose :id, documentation: { type: 'integer', example: 23 } + expose :created_at, documentation: { type: 'dateTime', example: '2020-11-13T00:06:18.084Z' } + expose :filename, documentation: { type: 'string', example: 'file.png' } + expose :file_path, documentation: { type: 'string', + example: '/uploads/-/system/alert_metric_image/file/23/file.png' } + expose :url, documentation: { type: 'string', example: 'https://example.com/metric' } + expose :url_text, documentation: { type: 'string', example: 'An example metric' } end end end diff --git a/lib/api/entities/milestone.rb b/lib/api/entities/milestone.rb index b191210a234..ea73ade46cd 100644 --- a/lib/api/entities/milestone.rb +++ b/lib/api/entities/milestone.rb @@ -10,7 +10,7 @@ module API expose :state, :created_at, :updated_at expose :due_date expose :start_date - expose :expired?, as: :expired + expose :expired expose :web_url do |milestone, _options| Gitlab::UrlBuilder.build(milestone) diff --git a/lib/api/entities/ml/mlflow/experiment.rb b/lib/api/entities/ml/mlflow/experiment.rb index 54e0fe63985..51650c36d98 100644 --- a/lib/api/entities/ml/mlflow/experiment.rb +++ b/lib/api/entities/ml/mlflow/experiment.rb @@ -9,6 +9,7 @@ module API expose :name expose(:lifecycle_stage) { |experiment| experiment.deleted_on? ? 'deleted' : 'active' } expose(:artifact_location) { |experiment| 'not_implemented' } + expose :metadata, as: :tags, using: KeyValue end end end diff --git a/lib/api/entities/ml/mlflow/run_param.rb b/lib/api/entities/ml/mlflow/key_value.rb index 75fee738f8b..cf2c32f6f44 100644 --- a/lib/api/entities/ml/mlflow/run_param.rb +++ b/lib/api/entities/ml/mlflow/key_value.rb @@ -4,7 +4,7 @@ module API module Entities module Ml module Mlflow - class RunParam < Grape::Entity + class KeyValue < Grape::Entity expose :name, as: :key expose :value end diff --git a/lib/api/entities/ml/mlflow/run.rb b/lib/api/entities/ml/mlflow/run.rb index 8b16c67611f..01d85e8862b 100644 --- a/lib/api/entities/ml/mlflow/run.rb +++ b/lib/api/entities/ml/mlflow/run.rb @@ -9,7 +9,8 @@ module API expose :itself, using: RunInfo, as: :info expose :data do expose :metrics, using: Metric - expose :params, using: RunParam + expose :params, using: KeyValue + expose :metadata, as: :tags, using: KeyValue end end end diff --git a/lib/api/entities/namespace.rb b/lib/api/entities/namespace.rb index f11303d41a6..15bc7d158c4 100644 --- a/lib/api/entities/namespace.rb +++ b/lib/api/entities/namespace.rb @@ -3,7 +3,7 @@ module API module Entities class Namespace < Entities::NamespaceBasic - expose :members_count_with_descendants, if: -> (namespace, opts) { expose_members_count_with_descendants?(namespace, opts) } do |namespace, _| + expose :members_count_with_descendants, documentation: { type: 'integer', example: 5 }, if: -> (namespace, opts) { expose_members_count_with_descendants?(namespace, opts) } do |namespace, _| namespace.users_with_descendants.count end diff --git a/lib/api/entities/namespace_basic.rb b/lib/api/entities/namespace_basic.rb index 2b9dd0b5f4d..4264326cdc2 100644 --- a/lib/api/entities/namespace_basic.rb +++ b/lib/api/entities/namespace_basic.rb @@ -3,9 +3,15 @@ module API module Entities class NamespaceBasic < Grape::Entity - expose :id, :name, :path, :kind, :full_path, :parent_id, :avatar_url + expose :id, documentation: { type: 'integer', example: 2 } + expose :name, documentation: { type: 'string', example: 'project' } + expose :path, documentation: { type: 'string', example: 'my_project' } + expose :kind, documentation: { type: 'string', example: 'project' } + expose :full_path, documentation: { type: 'string', example: 'group/my_project' } + expose :parent_id, documentation: { type: 'integer', example: 1 } + expose :avatar_url, documentation: { type: 'string', example: 'https://example.com/avatar/12345' } - expose :web_url do |namespace| + expose :web_url, documentation: { type: 'string', example: 'https://example.com/group/my_project' } do |namespace| if namespace.user_namespace? Gitlab::Routing.url_helpers.user_url(namespace.owner) else diff --git a/lib/api/entities/namespace_existence.rb b/lib/api/entities/namespace_existence.rb index d93078ecdac..ac9511930ab 100644 --- a/lib/api/entities/namespace_existence.rb +++ b/lib/api/entities/namespace_existence.rb @@ -3,7 +3,8 @@ module API module Entities class NamespaceExistence < Grape::Entity - expose :exists, :suggests + expose :exists, documentation: { type: 'boolean' } + expose :suggests, documentation: { type: 'string', is_array: true, example: 'my-group1' } end end end diff --git a/lib/api/entities/npm_package.rb b/lib/api/entities/npm_package.rb index b094f3acdb6..ad864f86fd5 100644 --- a/lib/api/entities/npm_package.rb +++ b/lib/api/entities/npm_package.rb @@ -3,9 +3,19 @@ module API module Entities class NpmPackage < Grape::Entity - expose :name - expose :versions - expose :dist_tags, as: 'dist-tags' + expose :name, documentation: { type: 'string', example: 'my_package' } + expose :versions, + documentation: { + type: 'object', + example: '{ + "1.0.0": { + "name": "my_package", + "version": "1.0.0", + "dist": { "shasum": "12345", "tarball": "https://..." } + } + }' + } + expose :dist_tags, as: 'dist-tags', documentation: { type: 'object', example: '{ "latest":"1.0.1" }' } end end end diff --git a/lib/api/entities/npm_package_tag.rb b/lib/api/entities/npm_package_tag.rb index 7f458fa037f..0a20d18e917 100644 --- a/lib/api/entities/npm_package_tag.rb +++ b/lib/api/entities/npm_package_tag.rb @@ -3,7 +3,7 @@ module API module Entities class NpmPackageTag < Grape::Entity - expose :dist_tags, merge: true + expose :dist_tags, merge: true, documentation: { type: 'object', example: '{ "latest":"1.0.1" }' } end end end diff --git a/lib/api/entities/nuget/dependency.rb b/lib/api/entities/nuget/dependency.rb index b61c37f5882..adb11376cfa 100644 --- a/lib/api/entities/nuget/dependency.rb +++ b/lib/api/entities/nuget/dependency.rb @@ -4,10 +4,10 @@ module API module Entities module Nuget class Dependency < Grape::Entity - expose :id, as: :@id - expose :type, as: :@type - expose :name, as: :id - expose :range + expose :id, as: :@id, documentation: { type: 'string', example: 'http://gitlab.com/Sandbox.App/1.0.0.json#dependency' } + expose :type, as: :@type, documentation: { type: 'string', example: 'PackageDependency' } + expose :name, as: :id, documentation: { type: 'string', example: 'Dependency' } + expose :range, documentation: { type: 'string', example: '2.0.0' } end end end diff --git a/lib/api/entities/nuget/dependency_group.rb b/lib/api/entities/nuget/dependency_group.rb index dcab9359fcf..8d943050cd8 100644 --- a/lib/api/entities/nuget/dependency_group.rb +++ b/lib/api/entities/nuget/dependency_group.rb @@ -4,10 +4,12 @@ module API module Entities module Nuget class DependencyGroup < Grape::Entity - expose :id, as: :@id - expose :type, as: :@type - expose :target_framework, as: :targetFramework, expose_nil: false - expose :dependencies, using: ::API::Entities::Nuget::Dependency + expose :id, as: :@id, documentation: { type: 'string', example: 'http://gitlab.com/Sandbox.App/1.0.0.json#dependencygroup' } + expose :type, as: :@type, documentation: { type: 'string', example: 'PackageDependencyGroup' } + expose :target_framework, as: :targetFramework, expose_nil: false, + documentation: { type: 'string', example: 'fwk test' } + expose :dependencies, using: ::API::Entities::Nuget::Dependency, + documentation: { is_array: true, type: 'API::Entities::Nuget::Dependency' } end end end diff --git a/lib/api/entities/nuget/metadatum.rb b/lib/api/entities/nuget/metadatum.rb index 87caef41a85..256b916cb64 100644 --- a/lib/api/entities/nuget/metadatum.rb +++ b/lib/api/entities/nuget/metadatum.rb @@ -4,9 +4,9 @@ module API module Entities module Nuget class Metadatum < Grape::Entity - expose :project_url, as: :projectUrl, expose_nil: false - expose :license_url, as: :licenseUrl, expose_nil: false - expose :icon_url, as: :iconUrl, expose_nil: false + expose :project_url, as: :projectUrl, expose_nil: false, documentation: { type: 'string', example: 'http://sandbox.com/project' } + expose :license_url, as: :licenseUrl, expose_nil: false, documentation: { type: 'string', example: 'http://sandbox.com/license' } + expose :icon_url, as: :iconUrl, expose_nil: false, documentation: { type: 'string', example: 'http://sandbox.com/icon' } end end end diff --git a/lib/api/entities/nuget/package_metadata.rb b/lib/api/entities/nuget/package_metadata.rb index e1c2a1ae161..1c94426bdd6 100644 --- a/lib/api/entities/nuget/package_metadata.rb +++ b/lib/api/entities/nuget/package_metadata.rb @@ -4,9 +4,10 @@ module API module Entities module Nuget class PackageMetadata < Grape::Entity - expose :json_url, as: :@id - expose :archive_url, as: :packageContent - expose :catalog_entry, as: :catalogEntry, using: ::API::Entities::Nuget::PackageMetadataCatalogEntry + expose :json_url, as: :@id, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/1/packages/nuget/metadata/MyNuGetPkg/1.3.0.17.json' } + expose :archive_url, as: :packageContent, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/1/packages/nuget/download/MyNuGetPkg/1.3.0.17/helloworld.1.3.0.17.nupkg' } + expose :catalog_entry, as: :catalogEntry, using: ::API::Entities::Nuget::PackageMetadataCatalogEntry, + documentation: { type: 'API::Entities::Nuget::PackageMetadataCatalogEntry' } end end end diff --git a/lib/api/entities/nuget/package_metadata_catalog_entry.rb b/lib/api/entities/nuget/package_metadata_catalog_entry.rb index 5533f857596..ce328c5a5ca 100644 --- a/lib/api/entities/nuget/package_metadata_catalog_entry.rb +++ b/lib/api/entities/nuget/package_metadata_catalog_entry.rb @@ -4,15 +4,17 @@ module API module Entities module Nuget class PackageMetadataCatalogEntry < Grape::Entity - expose :json_url, as: :@id - expose :authors - expose :dependency_groups, as: :dependencyGroups, using: ::API::Entities::Nuget::DependencyGroup - expose :package_name, as: :id - expose :package_version, as: :version - expose :tags - expose :archive_url, as: :packageContent - expose :summary - expose :metadatum, using: ::API::Entities::Nuget::Metadatum, merge: true + expose :json_url, as: :@id, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/1/packages/nuget/metadata/MyNuGetPkg/1.3.0.17.json' } + expose :authors, documentation: { type: 'string', example: 'Author' } + expose :dependency_groups, as: :dependencyGroups, using: ::API::Entities::Nuget::DependencyGroup, + documentation: { is_array: true, type: 'API::Entities::Nuget::DependencyGroup' } + expose :package_name, as: :id, documentation: { type: 'string', example: 'MyNuGetPkg' } + expose :package_version, as: :version, documentation: { type: 'string', example: '1.3.0.17' } + expose :tags, documentation: { type: 'string', example: 'tag#1 tag#2' } + expose :archive_url, as: :packageContent, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/1/packages/nuget/download/MyNuGetPkg/1.3.0.17/helloworld.1.3.0.17.nupkg' } + expose :summary, documentation: { type: 'string', example: 'Summary' } + expose :metadatum, using: ::API::Entities::Nuget::Metadatum, merge: true, + documentation: { type: 'API::Entities::Nuget::Metadatum' } end end end diff --git a/lib/api/entities/nuget/packages_metadata.rb b/lib/api/entities/nuget/packages_metadata.rb index 1cdf2491725..e556df0ce1f 100644 --- a/lib/api/entities/nuget/packages_metadata.rb +++ b/lib/api/entities/nuget/packages_metadata.rb @@ -4,8 +4,9 @@ module API module Entities module Nuget class PackagesMetadata < Grape::Entity - expose :count - expose :items, using: ::API::Entities::Nuget::PackagesMetadataItem + expose :count, documentation: { type: 'integer', example: 1 } + expose :items, using: ::API::Entities::Nuget::PackagesMetadataItem, + documentation: { is_array: true, type: 'API::Entities::Nuget::PackagesMetadataItem' } end end end diff --git a/lib/api/entities/nuget/packages_metadata_item.rb b/lib/api/entities/nuget/packages_metadata_item.rb index 84cc79166f3..420a4c3941c 100644 --- a/lib/api/entities/nuget/packages_metadata_item.rb +++ b/lib/api/entities/nuget/packages_metadata_item.rb @@ -4,11 +4,12 @@ module API module Entities module Nuget class PackagesMetadataItem < Grape::Entity - expose :json_url, as: :@id - expose :lower_version, as: :lower - expose :upper_version, as: :upper - expose :packages_count, as: :count - expose :packages, as: :items, using: ::API::Entities::Nuget::PackageMetadata + expose :json_url, as: :@id, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/1/packages/nuget/metadata/MyNuGetPkg/1.3.0.17.json' } + expose :lower_version, as: :lower, documentation: { type: 'string', example: '1.3.0.17' } + expose :upper_version, as: :upper, documentation: { type: 'string', example: '1.3.0.17' } + expose :packages_count, as: :count, documentation: { type: 'integer', example: 1 } + expose :packages, as: :items, using: ::API::Entities::Nuget::PackageMetadata, + documentation: { is_array: true, type: 'API::Entities::Nuget::PackageMetadata' } end end end diff --git a/lib/api/entities/nuget/packages_versions.rb b/lib/api/entities/nuget/packages_versions.rb index 498c6970d5c..e0330300ca7 100644 --- a/lib/api/entities/nuget/packages_versions.rb +++ b/lib/api/entities/nuget/packages_versions.rb @@ -4,7 +4,7 @@ module API module Entities module Nuget class PackagesVersions < Grape::Entity - expose :versions + expose :versions, documentation: { type: 'string', is_array: true, example: '1.3.0.17' } end end end diff --git a/lib/api/entities/nuget/search_result.rb b/lib/api/entities/nuget/search_result.rb index 8e028cbad95..bb3698de30b 100644 --- a/lib/api/entities/nuget/search_result.rb +++ b/lib/api/entities/nuget/search_result.rb @@ -4,17 +4,18 @@ module API module Entities module Nuget class SearchResult < Grape::Entity - expose :type, as: :@type - expose :authors - expose :name, as: :id - expose :name, as: :title - expose :summary - expose :total_downloads, as: :totalDownloads - expose :verified - expose :version + expose :type, as: :@type, documentation: { type: 'string', example: 'Package' } + expose :authors, documentation: { type: 'string', example: 'Author' } + expose :name, as: :id, documentation: { type: 'string', example: 'MyNuGetPkg' } + expose :name, as: :title, documentation: { type: 'string', example: 'MyNuGetPkg' } + expose :summary, documentation: { type: 'string', example: 'Summary' } + expose :total_downloads, as: :totalDownloads, documentation: { type: 'integer', example: 1 } + expose :verified, documentation: { type: 'boolean' } + expose :version, documentation: { type: 'string', example: '1.3.0.17' } expose :versions, using: ::API::Entities::Nuget::SearchResultVersion - expose :tags - expose :metadatum, using: ::API::Entities::Nuget::Metadatum, merge: true + expose :tags, documentation: { type: 'string', example: 'tag#1 tag#2' } + expose :metadatum, using: ::API::Entities::Nuget::Metadatum, merge: true, + documentation: { is_array: true, type: 'API::Entities::Nuget::Metadatum' } end end end diff --git a/lib/api/entities/nuget/search_result_version.rb b/lib/api/entities/nuget/search_result_version.rb index 9032c964c44..fb8d8b75f83 100644 --- a/lib/api/entities/nuget/search_result_version.rb +++ b/lib/api/entities/nuget/search_result_version.rb @@ -4,9 +4,9 @@ module API module Entities module Nuget class SearchResultVersion < Grape::Entity - expose :json_url, as: :@id - expose :version - expose :downloads + expose :json_url, as: :@id, documentation: { type: 'string', example: 'https://gitlab.example.com/api/v4/projects/1/packages/nuget/metadata/MyNuGetPkg/1.3.0.17.json' } + expose :version, documentation: { type: 'string', example: '1.3.0.17' } + expose :downloads, documentation: { type: 'integer', example: 1 } end end end diff --git a/lib/api/entities/nuget/search_results.rb b/lib/api/entities/nuget/search_results.rb index 22a77dc7b6c..117904a1aff 100644 --- a/lib/api/entities/nuget/search_results.rb +++ b/lib/api/entities/nuget/search_results.rb @@ -4,8 +4,9 @@ module API module Entities module Nuget class SearchResults < Grape::Entity - expose :total_count, as: :totalHits - expose :data, using: ::API::Entities::Nuget::SearchResult + expose :total_count, as: :totalHits, documentation: { type: 'integer', example: 1 } + expose :data, using: ::API::Entities::Nuget::SearchResult, + documentation: { is_array: true, type: 'API::Entities::Nuget::SearchResult' } end end end diff --git a/lib/api/entities/nuget/service_index.rb b/lib/api/entities/nuget/service_index.rb index e57bd04adb9..4ab6c5ddc8b 100644 --- a/lib/api/entities/nuget/service_index.rb +++ b/lib/api/entities/nuget/service_index.rb @@ -4,8 +4,8 @@ module API module Entities module Nuget class ServiceIndex < Grape::Entity - expose :version - expose :resources + expose :version, documentation: { type: 'string', example: '1.3.0.17' } + expose :resources, documentation: { type: 'object', is_array: true, example: '{ "@id": "https://gitlab.com/api/v4/projects/1/packages/nuget/query", "@type": "SearchQueryService", "comment": "Filter and search for packages by keyword."}' } end end end diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index c92a4677220..ab6cc0fcb0a 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -26,7 +26,7 @@ module API expose :status, documentation: { type: 'string', example: 'default' } expose :_links do - expose :web_path do |package| + expose :web_path, if: ->(package) { package.default? } do |package| package_path(package) end diff --git a/lib/api/entities/packages/debian/distribution.rb b/lib/api/entities/packages/debian/distribution.rb index 97a3c479f40..a11f4337f38 100644 --- a/lib/api/entities/packages/debian/distribution.rb +++ b/lib/api/entities/packages/debian/distribution.rb @@ -5,17 +5,18 @@ module API module Packages module Debian class Distribution < Grape::Entity - expose :id - expose :codename - expose :suite - expose :origin - expose :label - expose :version - expose :description - expose :valid_time_duration_seconds + expose :id, documentation: { type: 'integer', example: 1 } + expose :codename, documentation: { type: 'string', example: 'unstable' } + expose :suite, documentation: { type: 'string', example: 'unstable' } + expose :origin, documentation: { type: 'string', example: 'Grep' } + expose :label, documentation: { type: 'string', example: 'grep.be' } + expose :version, documentation: { type: 'string', example: '12' } + expose :description, documentation: { type: 'string', example: 'My description' } + expose :valid_time_duration_seconds, documentation: { type: 'integer', example: 604800 } - expose :component_names, as: :components - expose :architecture_names, as: :architectures + expose :component_names, as: :components, documentation: { is_array: true, type: 'string', example: 'main' } + expose :architecture_names, as: :architectures, + documentation: { is_array: true, type: 'string', example: 'amd64' } end end end diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb index 34018f03eb1..d69be0077f2 100644 --- a/lib/api/entities/plan_limit.rb +++ b/lib/api/entities/plan_limit.rb @@ -17,6 +17,7 @@ module API expose :maven_max_file_size, documentation: { type: 'integer', example: 3221225472 } expose :npm_max_file_size, documentation: { type: 'integer', example: 524288000 } expose :nuget_max_file_size, documentation: { type: 'integer', example: 524288000 } + expose :pipeline_hierarchy_size, documentation: { type: 'integer', example: 1000 } expose :pypi_max_file_size, documentation: { type: 'integer', example: 3221225472 } expose :terraform_module_max_file_size, documentation: { type: 'integer', example: 1073741824 } expose :storage_size_limit, documentation: { type: 'integer', example: 15000 } diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 1c1bafbf161..37be6903d8b 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -81,6 +81,10 @@ module API expose(:container_registry_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :container_registry) } expose(:security_and_compliance_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :security_and_compliance) } expose(:releases_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :releases) } + expose(:environments_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :environments) } + expose(:feature_flags_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :feature_flags) } + expose(:infrastructure_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :infrastructure) } + expose(:monitor_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :monitor) } expose :emails_disabled, documentation: { type: 'boolean' } expose :shared_runners_enabled, documentation: { type: 'boolean' } diff --git a/lib/api/entities/project_integration.rb b/lib/api/entities/project_integration.rb index 29bb60a19e5..f4709ce6dab 100644 --- a/lib/api/entities/project_integration.rb +++ b/lib/api/entities/project_integration.rb @@ -5,8 +5,8 @@ module API class ProjectIntegration < Entities::ProjectIntegrationBasic # Expose serialized properties expose :properties, documentation: { type: 'Hash', example: { "token" => "secr3t" } } do |integration, options| - integration.api_field_names.to_h do |name| - [name, integration.public_send(name)] # rubocop:disable GitlabSecurity/PublicSend + integration.api_field_names.index_with do |name| + integration.public_send(name) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/api/entities/push_event_payload.rb b/lib/api/entities/push_event_payload.rb index 6aad5f10177..2d8f0d9344c 100644 --- a/lib/api/entities/push_event_payload.rb +++ b/lib/api/entities/push_event_payload.rb @@ -3,8 +3,14 @@ module API module Entities class PushEventPayload < Grape::Entity - expose :commit_count, :action, :ref_type, :commit_from, :commit_to, :ref, - :commit_title, :ref_count + expose :commit_count, documentation: { type: 'integer', example: 1 } + expose :action, documentation: { type: 'string', example: 'pushed' } + expose :ref_type, documentation: { type: 'string', example: 'branch' } + expose :commit_from, documentation: { type: 'string', example: '50d4420237a9de7be1304607147aec22e4a14af7' } + expose :commit_to, documentation: { type: 'string', example: 'c5feabde2d8cd023215af4d2ceeb7a64839fc428' } + expose :ref, documentation: { type: 'string', example: 'master' } + expose :commit_title, documentation: { type: 'string', example: 'Add simple search to projects in public area' } + expose :ref_count, documentation: { type: 'integer', example: 1 } end end end diff --git a/lib/api/entities/ssh_key.rb b/lib/api/entities/ssh_key.rb index 3db10bb8ec2..37e8ad7b1f5 100644 --- a/lib/api/entities/ssh_key.rb +++ b/lib/api/entities/ssh_key.rb @@ -12,6 +12,7 @@ module API example: 'ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt1256k6Yjz\ GGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCdd\ NaP0L+hM7zhFNzjFvpaMgJw0=' } + expose :usage_type, documentation: { type: 'string', example: 'auth' } end end end diff --git a/lib/api/entities/ssh_signature.rb b/lib/api/entities/ssh_signature.rb new file mode 100644 index 00000000000..dc3800c87c5 --- /dev/null +++ b/lib/api/entities/ssh_signature.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module API + module Entities + class SshSignature < Grape::Entity + expose :verification_status, documentation: { type: 'string', example: 'unverified' } + expose :key, using: 'API::Entities::SSHKey' + end + end +end diff --git a/lib/api/entities/tag_signature.rb b/lib/api/entities/tag_signature.rb new file mode 100644 index 00000000000..e75fd04109a --- /dev/null +++ b/lib/api/entities/tag_signature.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + class TagSignature < Grape::Entity + expose :signature_type, documentation: { type: 'string', example: 'PGP' } + + expose :signature, merge: true do |tag| + ::API::Entities::X509Signature.represent tag.signature if tag.signature_type == :X509 + end + end + end +end diff --git a/lib/api/entities/todo.rb b/lib/api/entities/todo.rb index 5bbbb59f565..02dfdb68af9 100644 --- a/lib/api/entities/todo.rb +++ b/lib/api/entities/todo.rb @@ -32,6 +32,7 @@ module API def todo_target_url(todo) return design_todo_target_url(todo) if todo.for_design? + return todo.access_request_url if todo.member_access_requested? target_type = todo.target_type.gsub('::', '_').underscore target_url = "#{todo.resource_parent.class.to_s.underscore}_#{target_type}_url" diff --git a/lib/api/events.rb b/lib/api/events.rb index 0a0141484ef..d3e8892f3bc 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -15,8 +15,15 @@ module API desc "List currently authenticated user's events" do detail 'This feature was introduced in GitLab 9.3.' success Entities::Event + is_array true + failure [ + { code: 401, message: 'Unauthorized' } + ] end params do + optional :scope, type: String, + desc: 'Include all events across a user’s projects', + documentation: { example: 'all' } use :pagination use :event_filter_params use :sort_params @@ -32,12 +39,17 @@ module API end params do - requires :id, type: String, desc: 'The ID or Username of the user' + requires :id, type: String, desc: 'The ID or username of the user' end resource :users do desc 'Get the contribution events of a specified user' do detail 'This feature was introduced in GitLab 8.13.' success Entities::Event + tags %w[events] + is_array true + failure [ + { code: 404, message: 'Not found' } + ] end params do use :pagination diff --git a/lib/api/features.rb b/lib/api/features.rb index 6b6f5cbfb3f..9142591aebd 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -9,6 +9,8 @@ module API feature_category :feature_flags urgency :low + BadValueError = Class.new(StandardError) + # TODO: remove these helpers with feature flag set_feature_flag_service helpers do def gate_value(params) @@ -18,6 +20,8 @@ module API when '0', 'false' false else + raise BadValueError unless params[:value].match? /^\d+(\.\d+)?$/ + # https://github.com/jnunemaker/flipper/blob/master/lib/flipper/typecast.rb#L47 if params[:value].to_s.include?('.') params[:value].to_f @@ -153,7 +157,9 @@ module API present Feature.get(params[:name]), # rubocop:disable Gitlab/AvoidFeatureGet with: Entities::Feature, current_user: current_user end - rescue Feature::Target::UnknowTargetError => e + rescue BadValueError + bad_request!("Value must be boolean or numeric, got #{params[:value]}") + rescue Feature::Target::UnknownTargetError => e bad_request!(e.message) end diff --git a/lib/api/files.rb b/lib/api/files.rb index fa749299b9a..b02f1a8728b 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -172,14 +172,24 @@ module API desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false, documentation: { example: 'main' } + optional :lfs, type: Boolean, + desc: 'Retrieve binary data for a file that is an lfs pointer', + default: false end get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do assign_file_vars! - no_cache_headers - set_http_headers(blob_data) + if params[:lfs] && @blob.stored_externally? + lfs_object = LfsObject.find_by_oid(@blob.lfs_oid) + not_found! unless lfs_object&.project_allowed_access?(@project) + + present_carrierwave_file!(lfs_object.file) + else + no_cache_headers + set_http_headers(blob_data) - send_git_blob @repo, @blob + send_git_blob @repo, @blob + end end desc 'Get file metadata from repository' diff --git a/lib/api/freeze_periods.rb b/lib/api/freeze_periods.rb index 40f1be83028..abd8f4c0b94 100644 --- a/lib/api/freeze_periods.rb +++ b/lib/api/freeze_periods.rb @@ -34,7 +34,7 @@ module API get ":id/freeze_periods" do authorize! :read_freeze_period, user_project - freeze_periods = ::FreezePeriodsFinder.new(user_project, current_user).execute + freeze_periods = ::Ci::FreezePeriodsFinder.new(user_project, current_user).execute present paginate(freeze_periods), with: Entities::FreezePeriod, current_user: current_user end diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 3584f8d025a..da5b0930543 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -28,6 +28,13 @@ module API namespace ':package_name/*package_version/:file_name', requirements: GENERIC_PACKAGES_REQUIREMENTS do desc 'Workhorse authorize generic package file' do detail 'This feature was introduced in GitLab 13.5' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[generic_packages] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true @@ -47,6 +54,17 @@ module API desc 'Upload package file' do detail 'This feature was introduced in GitLab 13.5' + success [ + { code: 200 }, + { code: 201 } + ] + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[generic_packages] end params do @@ -88,6 +106,13 @@ module API desc 'Download package file' do detail 'This feature was introduced in GitLab 13.5' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[generic_packages] end params do diff --git a/lib/api/group_debian_distributions.rb b/lib/api/group_debian_distributions.rb index 1f43bb0e2b3..0364e2e7b56 100644 --- a/lib/api/group_debian_distributions.rb +++ b/lib/api/group_debian_distributions.rb @@ -3,7 +3,7 @@ module API class GroupDebianDistributions < ::API::Base params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the group' end before do diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ca99e30fbf7..23db10dbdbf 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -100,7 +100,7 @@ module API options = { with: serializer, current_user: current_user, - statistics: params[:statistics] && current_user&.admin? + statistics: params[:statistics] && current_user&.can_read_all_resources? } groups = groups.with_statistics if options[:statistics] @@ -186,7 +186,7 @@ module API end def check_subscription!(group) - render_api_error!("This group can't be removed because it is linked to a subscription.", :bad_request) if group.paid? + render_api_error!("This group can't be removed because it is linked to a subscription.", :bad_request) if group.prevent_delete? end end @@ -195,6 +195,8 @@ module API desc 'Get a groups list' do success Entities::Group + is_array true + tags %w[groups] end params do use :group_list_params @@ -207,6 +209,7 @@ module API desc 'Create a group. Available only for users who can create groups.' do success Entities::Group + tags %w[groups] end params do requires :name, type: String, desc: 'The name of the group' @@ -240,6 +243,7 @@ module API resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Update a group. Available only for users who can administrate groups.' do success Entities::Group + tags %w[groups] end params do optional :name, type: String, desc: 'The name of the group' @@ -265,6 +269,7 @@ module API desc 'Get a single group, with containing projects.' do success Entities::GroupDetail + tags %w[groups] end params do use :with_custom_attributes @@ -278,7 +283,9 @@ module API present_group_details(params, group, with_projects: params[:with_projects]) end - desc 'Remove a group.' + desc 'Remove a group.' do + tags %w[groups] + end delete ":id", feature_category: :subgroups, urgency: :low do group = find_group!(params[:id]) authorize! :admin_group, group @@ -289,6 +296,8 @@ module API desc 'Get a list of projects in this group.' do success Entities::Project + is_array true + tags %w[groups] end params do optional :archived, type: Boolean, desc: 'Limit by archived status' @@ -329,6 +338,8 @@ module API desc 'Get a list of shared projects in this group' do success Entities::Project + is_array true + tags %w[groups] end params do optional :archived, type: Boolean, desc: 'Limit by archived status' @@ -357,6 +368,8 @@ module API desc 'Get a list of subgroups in this group.' do success Entities::Group + is_array true + tags %w[groups] end params do use :group_list_params @@ -369,6 +382,8 @@ module API desc 'Get a list of descendant groups of this group.' do success Entities::Group + is_array true + tags %w[groups] end params do use :group_list_params @@ -382,6 +397,7 @@ module API desc 'Transfer a project to the group namespace. Available only for admin.' do success Entities::GroupDetail + tags %w[groups] end params do requires :project_id, type: String, desc: 'The ID or path of the project' @@ -400,7 +416,11 @@ module API end end - desc 'Get the groups to where the current group can be transferred to' + desc 'Get the groups to where the current group can be transferred to' do + success Entities::Group + is_array true + tags %w[groups] + end params do optional :search, type: String, desc: 'Return list of namespaces matching the search criteria' use :pagination @@ -415,7 +435,9 @@ module API present_groups params, groups, serializer: Entities::PublicGroupDetails end - desc 'Transfer a group to a new parent group or promote a subgroup to a root group' + desc 'Transfer a group to a new parent group or promote a subgroup to a root group' do + tags %w[groups] + end params do optional :group_id, type: Integer, @@ -440,6 +462,7 @@ module API desc 'Share a group with a group' do success Entities::GroupDetail + tags %w[groups] end params do requires :group_id, type: Integer, desc: 'The ID of the group to share' diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index fa2537bcfc4..8260d8a88f8 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -32,15 +32,21 @@ module API end params do - requires :id, type: String, desc: 'The ID or full path of a project' + requires :id, types: [Integer, String], desc: 'The ID or full path of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/helm', requirements: HELM_REQUIREMENTS do desc 'Download a chart index' do detail 'This feature was introduced in GitLab 14.0' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' } + ] + tags %w[helm_packages] end params do - requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex + requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex, documentation: { example: 'stable' } end get ":channel/index.yaml" do @@ -56,10 +62,17 @@ module API desc 'Download a chart' do detail 'This feature was introduced in GitLab 14.0' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[helm_packages] end params do - requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex - requires :file_name, type: String, desc: 'Helm package file name' + requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex, documentation: { example: 'stable' } + requires :file_name, type: String, desc: 'Helm package file name', documentation: { example: 'mychart' } end get ":channel/charts/:file_name.tgz" do project = authorized_user_project(action: :read_package) @@ -67,16 +80,23 @@ module API package_file = Packages::Helm::PackageFilesFinder.new(project, params[:channel], file_name: "#{params[:file_name]}.tgz").most_recent! - track_package_event('pull_package', :helm, project: project, namespace: project.namespace) + track_package_event('pull_package', :helm, project: project, namespace: project.namespace, property: 'i_package_helm_user') present_package_file!(package_file) end desc 'Authorize a chart upload from workhorse' do detail 'This feature was introduced in GitLab 14.0' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[helm_packages] end params do - requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex + requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex, documentation: { example: 'stable' } end post "api/:channel/charts/authorize" do authorize_workhorse!( @@ -88,9 +108,16 @@ module API desc 'Upload a chart' do detail 'This feature was introduced in GitLab 14.0' + success code: 201 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[helm_packages] end params do - requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex + requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex, documentation: { example: 'stable' } requires :chart, type: ::API::Validations::Types::WorkhorseFile, desc: 'The chart file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end post "api/:channel/charts" do @@ -110,7 +137,8 @@ module API package, chart_params.merge(build: current_authenticated_job) ).execute - track_package_event('push_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace) + track_package_event('push_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace, + property: 'i_package_helm_user') ::Packages::Helm::ExtractionWorker.perform_async(params[:channel], chart_package_file.id) # rubocop:disable CodeReuse/Worker diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 75e7612bd5b..0b5a471ea12 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -167,6 +167,10 @@ module API current_authenticated_job.project == project end + def enforce_jobs_api_rate_limits(project) + ::Feature.enabled?(:ci_enforce_rate_limits_jobs_api, project) + end + # rubocop: disable CodeReuse/ActiveRecord def find_group(id) if id.to_s =~ INTEGER_ID_REGEX @@ -301,7 +305,7 @@ module API def authenticated_as_admin! authenticate! - forbidden! unless current_user.admin? + forbidden! unless current_user.can_admin_all_resources? end def authorize!(action, subject = :global, reason = nil) @@ -710,7 +714,7 @@ module API unauthorized! unless initial_current_user - unless initial_current_user.admin? + unless initial_current_user.can_admin_all_resources? forbidden!('Must be admin to use sudo') end diff --git a/lib/api/helpers/award_emoji.rb b/lib/api/helpers/award_emoji.rb index 3ea35381c97..f8417366ea4 100644 --- a/lib/api/helpers/award_emoji.rb +++ b/lib/api/helpers/award_emoji.rb @@ -7,7 +7,7 @@ module API [ { type: 'issue', resource: :projects, find_by: :iid, feature_category: :team_planning }, { type: 'merge_request', resource: :projects, find_by: :iid, feature_category: :code_review }, - { type: 'snippet', resource: :projects, find_by: :id, feature_category: :snippets } + { type: 'snippet', resource: :projects, find_by: :id, feature_category: :source_code_management } ] end @@ -18,18 +18,16 @@ module API # rubocop: disable CodeReuse/ActiveRecord def awardable @awardable ||= - begin - if params.include?(:note_id) - note_id = params.delete(:note_id) + if params.include?(:note_id) + note_id = params.delete(:note_id) - awardable.notes.find(note_id) - elsif params.include?(:issue_iid) - user_project.issues.find_by!(iid: params[:issue_iid]) - elsif params.include?(:merge_request_iid) - user_project.merge_requests.find_by!(iid: params[:merge_request_iid]) - elsif params.include?(:snippet_id) - user_project.snippets.find(params[:snippet_id]) - end + awardable.notes.find(note_id) + elsif params.include?(:issue_iid) + user_project.issues.find_by!(iid: params[:issue_iid]) + elsif params.include?(:merge_request_iid) + user_project.merge_requests.find_by!(iid: params[:merge_request_iid]) + elsif params.include?(:snippet_id) + user_project.snippets.find(params[:snippet_id]) end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/helpers/discussions_helpers.rb b/lib/api/helpers/discussions_helpers.rb index c94199b17bc..182ada54a12 100644 --- a/lib/api/helpers/discussions_helpers.rb +++ b/lib/api/helpers/discussions_helpers.rb @@ -8,7 +8,7 @@ module API # extend it. { Issue => :team_planning, - Snippet => :snippets, + Snippet => :source_code_management, MergeRequest => :code_review, Commit => :code_review } diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 99273e81730..543449c0349 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -415,14 +415,6 @@ module API desc: 'The URL of the external wiki' } ], - 'flowdock' => [ - { - required: true, - name: :token, - type: String, - desc: 'Flowdock token' - } - ], 'hangouts-chat' => [ { required: true, @@ -893,7 +885,6 @@ module API ::Integrations::EmailsOnPush, ::Integrations::Ewm, ::Integrations::ExternalWiki, - ::Integrations::Flowdock, ::Integrations::HangoutsChat, ::Integrations::Harbor, ::Integrations::Irker, diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb index eed9fa30d3c..ee3bb49c97f 100644 --- a/lib/api/helpers/merge_requests_helpers.rb +++ b/lib/api/helpers/merge_requests_helpers.rb @@ -11,100 +11,107 @@ module API params :ee_approval_params do end - params :merge_requests_negatable_params do - optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' - optional :author_username, type: String, desc: 'Return merge requests which are authored by the user with the given username' + params :merge_requests_negatable_params do |options| + optional :author_id, type: Integer, + desc: "#{options[:prefix]}Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`." + optional :author_username, type: String, + desc: "#{options[:prefix]}Returns merge requests created by the given `username`. Mutually exclusive with `author_id`." mutually_exclusive :author_id, :author_username - - optional :assignee_id, - types: [Integer, String], - integer_none_any: true, - desc: 'Return merge requests which are assigned to the user with the given ID' - optional :assignee_username, - type: Array[String], - check_assignees_count: true, - coerce_with: Validations::Validators::CheckAssigneesCount.coerce, - desc: 'Return merge requests which are assigned to the user with the given username' + optional :assignee_id, types: [Integer, String], + integer_none_any: true, + desc: "#{options[:prefix]}Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee." + optional :assignee_username, type: Array[String], + check_assignees_count: true, + coerce_with: Validations::Validators::CheckAssigneesCount.coerce, + desc: "#{options[:prefix]}Returns merge requests created by the given `username`. Mutually exclusive with `author_id`.", + documentation: { is_array: true } mutually_exclusive :assignee_id, :assignee_username - optional :reviewer_username, - type: String, - desc: 'Return merge requests which have the user as a reviewer with the given username' - - optional :labels, - type: Array[String], - coerce_with: Validations::Types::CommaSeparatedToArray.coerce, - desc: 'Comma-separated list of label names' - optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' - optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' + optional :reviewer_username, type: String, + desc: "#{options[:prefix]}Returns merge requests which have the user as a reviewer with the given `username`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_id`. Introduced in GitLab 13.8." + optional :labels, type: Array[String], + coerce_with: Validations::Types::CommaSeparatedToArray.coerce, + desc: "#{options[:prefix]}Returns merge requests matching a comma-separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. Predefined names are case-insensitive.", + documentation: { is_array: true } + optional :milestone, type: String, + desc: "#{options[:prefix]}Returns merge requests for a specific milestone. `None` returns merge requests with no milestone. `Any` returns merge requests that have an assigned milestone." + optional :my_reaction_emoji, type: String, + desc: "#{options[:prefix]}Returns merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction." end params :merge_requests_base_params do - use :merge_requests_negatable_params - optional :reviewer_id, - types: [Integer, String], - integer_none_any: true, - desc: 'Return merge requests which have the user as a reviewer with the given ID' + use :merge_requests_negatable_params, prefix: '' + + optional :reviewer_id, types: [Integer, String], + integer_none_any: true, + desc: 'Returns merge requests which have the user as a reviewer with the given user `id`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_username`.' mutually_exclusive :reviewer_id, :reviewer_username - optional :state, - type: String, - values: %w[opened closed locked merged all], - default: 'all', - desc: 'Return opened, closed, locked, merged, or all merge requests' - optional :order_by, - type: String, - values: Helpers::MergeRequestsHelpers.sort_options, - default: 'created_at', - desc: "Return merge requests ordered by #{Helpers::MergeRequestsHelpers.sort_options_help} fields." - optional :sort, - type: String, - values: %w[asc desc], - default: 'desc', - desc: 'Return merge requests sorted in `asc` or `desc` order.' - optional :with_labels_details, type: Boolean, desc: 'Return titles of labels and other details', default: false - optional :with_merge_status_recheck, type: Boolean, desc: 'Request that stale merge statuses be rechecked asynchronously', default: false - optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' - optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' - optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time' - optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' - optional :view, - type: String, - values: %w[simple], - desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' - - optional :scope, - type: String, - values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], - desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' - optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' - optional :source_project_id, type: Integer, desc: 'Return merge requests with the given source project id' - optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' - optional :search, - type: String, - desc: 'Search merge requests for text present in the title, description, or any combination of these' - optional :in, type: String, desc: '`title`, `description`, or a string joining them with comma' - optional :wip, type: String, values: %w[yes no], desc: 'Search merge requests for WIP in the title' - optional :not, type: Hash, desc: 'Parameters to negate' do - use :merge_requests_negatable_params - optional :reviewer_id, - types: Integer, - desc: 'Return merge requests which have the user as a reviewer with the given ID' + optional :state, type: String, + values: %w[opened closed locked merged all], + default: 'all', + desc: 'Returns `all` merge requests or just those that are `opened`, `closed`, `locked`, or `merged`.' + optional :order_by, type: String, + values: Helpers::MergeRequestsHelpers.sort_options, + default: 'created_at', + desc: "Returns merge requests ordered by #{Helpers::MergeRequestsHelpers.sort_options_help} fields. Introduced in GitLab 14.8." + optional :sort, type: String, + values: %w[asc desc], + default: 'desc', + desc: 'Returns merge requests sorted in `asc` or `desc` order.' + optional :with_labels_details, type: Boolean, + default: false, + desc: 'If `true`, response returns more details for each label in labels field: `:name`,`:color`, `:description`, `:description_html`, `:text_color`' + optional :with_merge_status_recheck, type: Boolean, + default: false, + desc: 'If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Introduced in GitLab 13.0.' + optional :created_after, type: DateTime, + desc: 'Returns merge requests created on or after the given time. Expected in ISO 8601 format.', + documentation: { example: '2019-03-15T08:00:00Z' } + optional :created_before, type: DateTime, + desc: 'Returns merge requests created on or before the given time. Expected in ISO 8601 format.', + documentation: { example: '2019-03-15T08:00:00Z' } + optional :updated_after, type: DateTime, + desc: 'Returns merge requests updated on or after the given time. Expected in ISO 8601 format.', + documentation: { example: '2019-03-15T08:00:00Z' } + optional :updated_before, type: DateTime, + desc: 'Returns merge requests updated on or before the given time. Expected in ISO 8601 format.', + documentation: { example: '2019-03-15T08:00:00Z' } + optional :view, type: String, + values: %w[simple], + desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' + optional :scope, type: String, + values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], + desc: 'Returns merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' + optional :source_branch, type: String, desc: 'Returns merge requests with the given source branch' + optional :source_project_id, type: Integer, desc: 'Returns merge requests with the given source project id' + optional :target_branch, type: String, desc: 'Returns merge requests with the given target branch' + optional :search, type: String, + desc: 'Search merge requests against their `title` and `description`.' + optional :in, type: String, + desc: 'Modify the scope of the search attribute. `title`, `description`, or a string joining them with comma.', + documentation: { example: 'title,description' } + optional :wip, type: String, + values: %w[yes no], + desc: 'Filter merge requests against their `wip` status. `yes` to return only draft merge requests, `no` to return non-draft merge requests.' + optional :not, type: Hash, desc: 'Returns merge requests that do not match the parameters supplied' do + use :merge_requests_negatable_params, prefix: '`<Negated>` ' + + optional :reviewer_id, types: Integer, + desc: '`<Negated>` Returns merge requests which have the user as a reviewer with the given user `id`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_username`.' mutually_exclusive :reviewer_id, :reviewer_username end - - optional :deployed_before, - 'Return merge requests deployed before the given date/time' - optional :deployed_after, - 'Return merge requests deployed after the given date/time' - optional :environment, - 'Returns merge requests deployed to the given environment' + optional :deployed_before, desc: 'Returns merge requests deployed before the given date/time. Expected in ISO 8601 format.', + documentation: { example: '2019-03-15T08:00:00Z' } + optional :deployed_after, desc: 'Returns merge requests deployed after the given date/time. Expected in ISO 8601 format', + documentation: { example: '2019-03-15T08:00:00Z' } + optional :environment, desc: 'Returns merge requests deployed to the given environment', + documentation: { example: '2019-03-15T08:00:00Z' } end params :optional_scope_param do - optional :scope, - type: String, - values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], - default: 'created_by_me', - desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' + optional :scope, type: String, + values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], + default: 'created_by_me', + desc: 'Returns merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' end def handle_merge_request_errors!(merge_request) diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 45671b09be9..302dac4abf7 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -9,7 +9,7 @@ module API { Issue => :team_planning, MergeRequest => :code_review, - Snippet => :snippets + Snippet => :source_code_management } end @@ -90,7 +90,12 @@ module API params = finder_params_by_noteable_type_and_id(noteable_type, noteable_id) noteable = NotesFinder.new(current_user, params).target - noteable = nil unless can?(current_user, noteable_read_ability_name(noteable), noteable) + + # Checking `read_note` permission here, because API code does not seem to use NoteFinder to find notes, + # but rather pulls notes directly through notes association, so there is no chance to check read_note + # permission at service level. With WorkItem model we need to make sure that it has WorkItem::Widgets::Note + # available in order to access notes. + noteable = nil unless can_read_notes?(noteable) noteable || not_found!(noteable_type) end @@ -147,6 +152,13 @@ module API def disable_query_limiting Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/211538') end + + private + + def can_read_notes?(noteable) + Ability.allowed?(current_user, noteable_read_ability_name(noteable), noteable) && + Ability.allowed?(current_user, :read_note, noteable) + end end end end diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index a9d91895cfe..3ea558f3569 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -47,14 +47,14 @@ module API end def recipe_upload_urls - { upload_urls: file_names.select(&method(:recipe_file?)).to_h do |file_name| - [file_name, build_recipe_file_upload_url(file_name)] + { upload_urls: file_names.select(&method(:recipe_file?)).index_with do |file_name| + build_recipe_file_upload_url(file_name) end } end def package_upload_urls - { upload_urls: file_names.select(&method(:package_file?)).to_h do |file_name| - [file_name, build_package_file_upload_url(file_name)] + { upload_urls: file_names.select(&method(:package_file?)).index_with do |file_name| + build_package_file_upload_url(file_name) end } end @@ -128,7 +128,7 @@ module API strong_memoize(:project) do case package_scope when :project - find_project!(params[:id]) + user_project(action: :read_package) when :instance full_path = ::Packages::Conan::Metadatum.full_path_from(package_username: params[:package_username]) find_project!(full_path) diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index 1ae863a5a25..4b0e63c8f3b 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -19,7 +19,9 @@ module API def redirect_registry_request(forward_to_registry: false, package_type: nil, target: nil, **options) if forward_to_registry && redirect_registry_request_available?(package_type, target) && maven_forwarding_ff_enabled?(package_type, target) ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward") - redirect(registry_url(package_type, options)) + redirect(registry_url(package_type, options), body: options[:body]) + # For the requests with POST methods we need to set status 307 in order to keep request's method + status :temporary_redirect if options[:method] == 'POST' else yield end @@ -32,7 +34,7 @@ module API case package_type when :npm - "#{base_url}#{options[:package_name]}" + "#{base_url}#{[options[:path], options[:package_name]].compact.join('/')}" when :pypi "#{base_url}#{options[:package_name]}/" when :maven diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index 96a10d43401..8d913268405 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -78,10 +78,18 @@ module API end end - def track_package_event(event_name, scope, **args) - ::Packages::CreateEventService.new(nil, current_user, event_name: event_name, scope: scope).execute + def track_package_event(action, scope, **args) + ::Packages::CreateEventService.new(nil, current_user, event_name: action, scope: scope).execute category = args.delete(:category) || self.options[:for].name - ::Gitlab::Tracking.event(category, event_name.to_s, **args) + event_name = "i_package_#{scope}_user" + ::Gitlab::Tracking.event( + category, + action.to_s, + property: event_name, + label: 'redis_hll_counters.user_packages.user_packages_total_unique_counts_monthly', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context], + **args + ) end def present_package_file!(package_file, supports_direct_download: true) diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index c95bf0f0c21..9d370176e62 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -37,6 +37,10 @@ module API optional :container_registry_access_level, type: String, values: %w(disabled private enabled), desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry' optional :security_and_compliance_access_level, type: String, values: %w(disabled private enabled), desc: 'Security and compliance access level. One of `disabled`, `private` or `enabled`' optional :releases_access_level, type: String, values: %w(disabled private enabled), desc: 'Releases access level. One of `disabled`, `private` or `enabled`' + optional :environments_access_level, type: String, values: %w(disabled private enabled), desc: 'Environments access level. One of `disabled`, `private` or `enabled`' + optional :feature_flags_access_level, type: String, values: %w(disabled private enabled), desc: 'Feature flags access level. One of `disabled`, `private` or `enabled`' + optional :infrastructure_access_level, type: String, values: %w(disabled private enabled), desc: 'Infrastructure access level. One of `disabled`, `private` or `enabled`' + optional :monitor_access_level, type: String, values: %w(disabled private enabled), desc: 'Monitor access level. One of `disabled`, `private` or `enabled`' optional :emails_disabled, type: Boolean, desc: 'Disable email notifications' optional :show_default_award_emojis, type: Boolean, desc: 'Show default award emojis' @@ -183,6 +187,10 @@ module API :mr_default_target_self, :enforce_auth_checks_on_uploads, :releases_access_level, + :environments_access_level, + :feature_flags_access_level, + :infrastructure_access_level, + :monitor_access_level, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, diff --git a/lib/api/integrations/jira_connect/subscriptions.rb b/lib/api/integrations/jira_connect/subscriptions.rb index a6e931ba7bb..cc2199e0ef6 100644 --- a/lib/api/integrations/jira_connect/subscriptions.rb +++ b/lib/api/integrations/jira_connect/subscriptions.rb @@ -11,14 +11,22 @@ module API namespace :integrations do namespace :jira_connect do resource :subscriptions do - desc 'Subscribe a namespace to a JiraConnectInstallation' + desc 'Subscribe a namespace to a JiraConnectInstallation' do + detail 'Subscribes the namespace to the JiraConnectInstallation' + success ::API::Entities::BasicSuccess + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[jira_connect_subscriptions] + end params do requires :jwt, type: String, desc: 'JWT token for authorization with the Jira Connect installation' requires :namespace_path, type: String, desc: 'Path for the namespace that should be subscribed' end post do - not_found! unless Feature.enabled?(:jira_connect_oauth, current_user) - jwt = Atlassian::JiraConnect::Jwt::Symmetric.new(params[:jwt]) installation = JiraConnectInstallation.find_by_client_key(jwt.iss_claim) diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index c4464666020..dbd5c5f9db1 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -191,7 +191,7 @@ module API get '/authorized_keys', feature_category: :source_code_management, urgency: :high do fingerprint = Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint_sha256 - key = Key.find_by_fingerprint_sha256(fingerprint) + key = Key.auth.find_by_fingerprint_sha256(fingerprint) not_found!('Key') if key.nil? present key, with: Entities::SSHKey end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index d06d1e9862a..777d5019a29 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -6,7 +6,6 @@ module API class Kubernetes < ::API::Base include Gitlab::Utils::StrongMemoize - feature_category :kubernetes_management before do check_feature_enabled authenticate_gitlab_kas_request! @@ -86,7 +85,7 @@ module API detail 'Retrieves agent info for the given token' end route_setting :authentication, cluster_agent_token_allowed: true - get '/agent_info', urgency: :low do + get '/agent_info', feature_category: :kubernetes_management, urgency: :low do project = agent.project status 200 @@ -104,7 +103,7 @@ module API detail 'Retrieves project info (if authorized)' end route_setting :authentication, cluster_agent_token_allowed: true - get '/project_info', urgency: :low do + get '/project_info', feature_category: :kubernetes_management, urgency: :low do project = find_project(params[:id]) not_found! unless agent_has_access_to_project?(project) @@ -127,7 +126,7 @@ module API requires :agent_id, type: Integer, desc: 'ID of the configured Agent' requires :agent_config, type: JSON, desc: 'Configuration for the Agent' end - post '/' do + post '/', feature_category: :kubernetes_management do agent = ::Clusters::Agent.find(params[:agent_id]) ::Clusters::Agents::RefreshAuthorizationService.new(agent, config: params[:agent_config]).execute @@ -147,10 +146,10 @@ module API end optional :unique_counters, type: Hash do - optional :agent_users_using_ci_tunnel, type: Set[Integer], desc: 'A set of user ids that have interacted a CI Tunnel to' + optional :agent_users_using_ci_tunnel, type: Array[Integer], desc: 'An array of user ids that have interacted with CI Tunnel' end end - post '/' do + post '/', feature_category: :kubernetes_management do increment_count_events increment_unique_events diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index 276560f3433..f348e20cc0b 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -35,6 +35,11 @@ module API context[:skip_project_check] = true end + # Disable comments in markdown for IE browsers because comments in IE + # could allow script execution. + browser = Browser.new(headers['User-Agent']) + context[:allow_comments] = !browser.ie? + present({ html: Banzai.render_and_post_process(params[:text], context) }, with: Entities::Markdown) end end diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 30cdaba76ba..411a53a481b 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -107,7 +107,7 @@ module API def fetch_package(file_name:, project: nil, group: nil) order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) && - !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM) + !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM) ::Packages::Maven::PackageFinder.new( current_user, @@ -150,10 +150,17 @@ module API desc 'Download the maven package file at instance level' do detail 'This feature was introduced in GitLab 11.6' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[maven_packages] end params do - requires :path, type: String, desc: 'Package path' - requires :file_name, type: String, desc: 'Package file name' + requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } + requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get 'packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do @@ -190,14 +197,24 @@ module API desc 'Download the maven package file at a group level' do detail 'This feature was introduced in GitLab 11.7' + success [ + { code: 200 }, + { code: 302 } + ] + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[maven_packages] end params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the group' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do - requires :path, type: String, desc: 'Package path' - requires :file_name, type: String, desc: 'Package file name' + requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } + requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do @@ -225,10 +242,20 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Download the maven package file' do detail 'This feature was introduced in GitLab 11.3' + success [ + { code: 200 }, + { code: 302 } + ] + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[maven_packages] end params do - requires :path, type: String, desc: 'Package path' - requires :file_name, type: String, desc: 'Package file name' + requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } + requires :file_name, type: String, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do @@ -250,10 +277,18 @@ module API desc 'Workhorse authorize the maven package file upload' do detail 'This feature was introduced in GitLab 11.3' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[maven_packages] end params do - requires :path, type: String, desc: 'Package path' - requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.maven_file_name_regex + requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } + requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.maven_file_name_regex, documentation: { example: 'mypkg-1.0-SNAPSHOT.pom' } end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true put ':id/packages/maven/*path/:file_name/authorize', requirements: MAVEN_ENDPOINT_REQUIREMENTS do @@ -266,10 +301,19 @@ module API desc 'Upload the maven package file' do detail 'This feature was introduced in GitLab 11.3' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' }, + { code: 422, message: 'Unprocessable Entity' } + ] + tags %w[maven_packages] end params do - requires :path, type: String, desc: 'Package path' - requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.maven_file_name_regex + requires :path, type: String, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } + requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.maven_file_name_regex, documentation: { example: 'mypkg-1.0-SNAPSHOT.pom' } requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true diff --git a/lib/api/members.rb b/lib/api/members.rb index f4e38207aca..76f4364106b 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -20,6 +20,8 @@ module API resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Gets a list of group or project members viewable by the authenticated user.' do success Entities::Member + is_array true + tags %w[members] end params do optional :query, type: String, desc: 'A query string to search for members' @@ -42,6 +44,8 @@ module API desc 'Gets a list of group or project members viewable by the authenticated user, including those who gained membership through ancestor group.' do success Entities::Member + is_array true + tags %w[members] end params do optional :query, type: String, desc: 'A query string to search for members' @@ -63,6 +67,7 @@ module API desc 'Gets a member of a group or project.' do success Entities::Member + tags %w[members] end params do requires :user_id, type: Integer, desc: 'The user ID of the member' @@ -82,6 +87,7 @@ module API desc 'Gets a member of a group or project, including those who gained membership through ancestor group' do success Entities::Member + tags %w[members] end params do requires :user_id, type: Integer, desc: 'The user ID of the member' @@ -101,6 +107,7 @@ module API desc 'Adds a member to a group or project.' do success Entities::Member + tags %w[members] end params do requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' @@ -126,6 +133,7 @@ module API desc 'Updates a member of a group or project.' do success Entities::Member + tags %w[members] end params do requires :user_id, type: Integer, desc: 'The user ID of the new member' @@ -153,7 +161,9 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'Removes a user from a group or project.' + desc 'Removes a user from a group or project.' do + tags %w[members] + end params do requires :user_id, type: Integer, desc: 'The user ID of the member' optional :skip_subresources, type: Boolean, default: false, diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb index 7622ec717cc..35fdcfe3ab0 100644 --- a/lib/api/merge_request_approvals.rb +++ b/lib/api/merge_request_approvals.rb @@ -88,6 +88,24 @@ module API present_approval(merge_request) end + + desc 'Remove all merge request approvals' do + detail 'Clear all approvals of merge request. This feature was added in GitLab 15.4' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] + end + put 'reset_approvals', urgency: :low do + merge_request = find_project_merge_request(params[:merge_request_iid]) + + unauthorized! unless current_user.can?(:reset_merge_request_approvals, merge_request) + + merge_request.approvals.delete_all + + status :accepted + end end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index bb2861aa221..a9572cf7ce6 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -134,7 +134,13 @@ module API resource :merge_requests do desc 'List merge requests' do + detail 'Get all merge requests the authenticated user has access to. By default it returns only merge requests created by the current user. To get all merge requests, use parameter `scope=all`.' success Entities::MergeRequestBasic + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[merge_requests] end params do use :merge_requests_params @@ -150,16 +156,24 @@ module API end params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, type: String, desc: 'The ID or URL-encoded path of the group owned by the authenticated user.' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get a list of group merge requests' do + desc 'List group merge requests' do + detail 'Get all merge requests for this group and its subgroups.' success Entities::MergeRequestBasic + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[merge_requests] end params do use :merge_requests_params - optional :non_archived, type: Boolean, desc: 'Return merge requests from non archived projects', - default: true + optional :non_archived, type: Boolean, + default: true, + desc: 'Returns merge requests from non archived projects only.' end get ":id/merge_requests", feature_category: :code_review, urgency: :low do validate_anonymous_search_access! if declared_params[:search].present? @@ -170,36 +184,62 @@ module API end params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project.' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include TimeTrackingEndpoints helpers do params :optional_params do - optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' - optional :assignee_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Comma-separated list of assignee ids' - optional :reviewer_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Comma-separated list of reviewer ids' - optional :description, type: String, desc: 'The description of the merge request' - optional :labels, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' - optional :add_labels, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' - optional :remove_labels, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' - optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' - optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' - optional :allow_collaboration, type: Boolean, desc: 'Allow commits from members who can merge to the target branch' + optional :assignee_id, type: Integer, desc: 'Assignee user ID.' + optional :assignee_ids, type: Array[Integer], + coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, + desc: 'The IDs of the users to assign the merge request to, as a comma-separated list. Set to 0 or provide an empty value to unassign all assignees.', + documentation: { is_array: true } + optional :reviewer_ids, type: Array[Integer], + coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, + desc: 'The IDs of the users to review the merge request, as a comma-separated list. Set to 0 or provide an empty value to unassign all reviewers.', + documentation: { is_array: true } + optional :description, type: String, desc: 'Description of the merge request. Limited to 1,048,576 characters.' + optional :labels, type: Array[String], + coerce_with: Validations::Types::CommaSeparatedToArray.coerce, + desc: 'Comma-separated label names for a merge request. Set to an empty string to unassign all labels.', + documentation: { is_array: true } + optional :add_labels, type: Array[String], + coerce_with: Validations::Types::CommaSeparatedToArray.coerce, + desc: 'Comma-separated label names to add to a merge request.', + documentation: { is_array: true } + optional :remove_labels, type: Array[String], + coerce_with: Validations::Types::CommaSeparatedToArray.coerce, + desc: 'Comma-separated label names to remove from a merge request.', + documentation: { is_array: true } + optional :milestone_id, type: Integer, desc: 'The global ID of a milestone to assign the merge reques to.' + optional :remove_source_branch, type: Boolean, desc: 'Flag indicating if a merge request should remove the source branch when merging.' + optional :allow_collaboration, type: Boolean, desc: 'Allow commits from members who can merge to the target branch.' optional :allow_maintainer_to_push, type: Boolean, as: :allow_collaboration, desc: '[deprecated] See allow_collaboration' - optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge' + optional :squash, type: Grape::API::Boolean, desc: 'Squash commits into a single commit when merging.' use :optional_params_ee end end - desc 'List merge requests' do + desc 'List project merge requests' do + detail 'Get all merge requests for this project.' success Entities::MergeRequestBasic + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[merge_requests] end params do use :merge_requests_params - optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IID array of merge requests' + + optional :iids, type: Array[Integer], + coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, + desc: 'Returns the request having the given `iid`.', + documentation: { is_array: true } end get ":id/merge_requests", feature_category: :code_review, urgency: :low do authorize! :read_merge_request, user_project @@ -226,15 +266,24 @@ module API **options end - desc 'Create a merge request' do + desc 'Create merge request' do + detail 'Create a new merge request.' + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' }, + { code: 422, message: 'Unprocessable entity' } + ] success Entities::MergeRequest + tags %w[merge_requests] end params do - requires :title, type: String, desc: 'The title of the merge request' - requires :source_branch, type: String, desc: 'The source branch' - requires :target_branch, type: String, desc: 'The target branch' + requires :title, type: String, desc: 'The title of the merge request.' + requires :source_branch, type: String, desc: 'The source branch.' + requires :target_branch, type: String, desc: 'The target branch.' optional :target_project_id, type: Integer, - desc: 'The target project of the merge request defaults to the :id of the project' + desc: 'The target project of the merge request defaults to the :id of the project.' use :optional_params end post ":id/merge_requests", feature_category: :code_review, urgency: :low do @@ -253,9 +302,17 @@ module API present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end - desc 'Delete a merge request' + desc 'Delete a merge request' do + detail 'Only for administrators and project owners. Deletes the merge request in question. ' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 412, message: 'Precondition failed' } + ] + tags %w[merge_requests] + end params do - requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The internal ID of the merge request.' end delete ":id/merge_requests/:merge_request_iid", feature_category: :code_review, urgency: :low do merge_request = find_project_merge_request(params[:merge_request_iid]) @@ -268,13 +325,19 @@ module API end params do - requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' - optional :render_html, type: Boolean, desc: 'Returns the description and title rendered HTML' - optional :include_diverged_commits_count, type: Boolean, desc: 'Returns the commits count behind the target branch' - optional :include_rebase_in_progress, type: Boolean, desc: 'Returns whether a rebase operation is ongoing ' + requires :merge_request_iid, type: Integer, desc: 'The internal ID of the merge request.' + optional :render_html, type: Boolean, desc: 'If `true`, response includes rendered HTML for title and description.' + optional :include_diverged_commits_count, type: Boolean, desc: 'If `true`, response includes the commits behind the target branch.' + optional :include_rebase_in_progress, type: Boolean, desc: 'If `true`, response includes whether a rebase operation is in progress.' end - desc 'Get a single merge request' do + desc 'Get single merge request' do + detail 'Shows information about a single merge request. Note: the `changes_count` value in the response is a string, not an integer. This is because when an merge request has too many changes to display and store, it is capped at 1,000. In that case, the API returns the string `"1000+"` for the changes count.' + success Entities::MergeRequest + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end get ':id/merge_requests/:merge_request_iid', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -289,8 +352,13 @@ module API include_rebase_in_progress: params[:include_rebase_in_progress] end - desc 'Get the participants of a merge request' do + desc 'Get single merge request participants' do + detail 'Get a list of merge request participants.' success Entities::UserBasic + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end get ':id/merge_requests/:merge_request_iid/participants', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -300,8 +368,13 @@ module API present paginate(participants), with: Entities::UserBasic end - desc 'Get the reviewers of a merge request' do + desc 'Get single merge request reviewers' do + detail 'Get a list of merge request reviewers.' success Entities::MergeRequestReviewer + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end get ':id/merge_requests/:merge_request_iid/reviewers', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -311,8 +384,13 @@ module API present paginate(reviewers), with: Entities::MergeRequestReviewer end - desc 'Get the commits of a merge request' do + desc 'Get single merge request commits' do + detail 'Get a list of merge request commits.' success Entities::Commit + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end get ':id/merge_requests/:merge_request_iid/commits', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -324,8 +402,13 @@ module API present commits, with: Entities::Commit end - desc 'Get the context commits of a merge request' do + desc 'List merge request context commits' do + detail 'Get a list of merge request context commits.' success Entities::Commit + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end get ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review, urgency: :high do merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -336,10 +419,20 @@ module API end params do - requires :commits, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, allow_blank: false, desc: 'List of context commits sha' - end - desc 'create context commits of merge request' do + requires :commits, type: Array[String], + coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + allow_blank: false, + desc: 'The context commits’ SHA.', + documentation: { is_array: true } + end + desc 'Create merge request context commits' do + detail 'Create a list of merge request context commits.' success Entities::Commit + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end post ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do commit_ids = params[:commits] @@ -363,9 +456,21 @@ module API end params do - requires :commits, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, allow_blank: false, desc: 'List of context commits sha' + requires :commits, type: Array[String], + coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, + allow_blank: false, + desc: 'The context commits’ SHA.', + documentation: { is_array: true } + end + desc 'Delete merge request context commits' do + detail 'Delete a list of merge request context commits.' + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end - desc 'remove context commits of merge request' delete ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do commit_ids = params[:commits] merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -382,8 +487,13 @@ module API status 204 end - desc 'Show the merge request changes' do + desc 'Get single merge request changes' do + detail 'Shows information about the merge request including its files and changes.' success Entities::MergeRequestChanges + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end get ':id/merge_requests/:merge_request_iid/changes', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -395,17 +505,46 @@ module API access_raw_diffs: to_boolean(params.fetch(:access_raw_diffs, false)) end - desc 'Get the merge request pipelines' do + desc 'Get the merge request diffs' do + detail 'Get a list of merge request diffs.' + success Entities::Diff + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] + end + params do + use :pagination + end + get ':id/merge_requests/:merge_request_iid/diffs', feature_category: :code_review, urgency: :low do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) + + present paginate(merge_request.merge_request_diff.paginated_diffs(params[:page], params[:per_page])).diffs, with: Entities::Diff + end + + desc 'Get single merge request pipelines' do + detail 'Get a list of merge request pipelines.' success Entities::Ci::PipelineBasic + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end get ':id/merge_requests/:merge_request_iid/pipelines', urgency: :low, feature_category: :continuous_integration do pipelines = merge_request_pipelines_with_access - present paginate(pipelines), with: Entities::Ci::PipelineBasic end - desc 'Create a pipeline for merge request' do + desc 'Create merge request pipeline' do + detail 'Create a new pipeline for a merge request. A pipeline created via this endpoint doesn’t run a regular branch/tag pipeline. It requires `.gitlab-ci.yml` to be configured with `only: [merge_requests]` to create jobs.' success ::API::Entities::Ci::Pipeline + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' }, + { code: 405, message: 'Method not allowed' } + ] + tags %w[merge_requests] end post ':id/merge_requests/:merge_request_iid/pipelines', urgency: :low, feature_category: :continuous_integration do pipeline = ::MergeRequests::CreatePipelineService @@ -423,15 +562,25 @@ module API end end - desc 'Update a merge request' do + desc 'Update merge request' do + detail 'Updates an existing merge request. You can change the target branch, title, or even close the merge request.' success Entities::MergeRequest + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[merge_requests] end params do - optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' - optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' - optional :state_event, type: String, values: %w[close reopen], - desc: 'Status of the merge request' - optional :discussion_locked, type: Boolean, desc: 'Whether the MR discussion is locked' + optional :title, type: String, allow_blank: false, desc: 'The title of the merge request.' + optional :target_branch, type: String, allow_blank: false, desc: 'The target branch.' + optional :state_event, type: String, + values: %w[close reopen], + desc: 'New state (close/reopen).' + optional :discussion_locked, type: Boolean, + desc: 'Flag indicating if the merge request’s discussion is locked. If the discussion is locked only project members can add, edit or resolve comments.' use :optional_params at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of) @@ -456,17 +605,27 @@ module API end desc 'Merge a merge request' do + detail 'Accept and merge changes submitted with the merge request using this API.' success Entities::MergeRequest + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 405, message: 'Method not allowed' }, + { code: 409, message: 'Conflict' }, + { code: 422, message: 'Unprocessable entity' } + ] + tags %w[merge_requests] end params do - optional :merge_commit_message, type: String, desc: 'Custom merge commit message' - optional :squash_commit_message, type: String, desc: 'Custom squash commit message' + optional :merge_commit_message, type: String, desc: 'Custom merge commit message.' + optional :squash_commit_message, type: String, desc: 'Custom squash commit message.' optional :should_remove_source_branch, type: Boolean, - desc: 'When true, the source branch will be deleted if possible' + desc: 'If `true`, removes the source branch.' optional :merge_when_pipeline_succeeds, type: Boolean, - desc: 'When true, this merge request will be merged when the pipeline succeeds' - optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' - optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge' + desc: 'If `true`, the merge request is merged when the pipeline succeeds.' + optional :sha, type: String, desc: 'If present, then this SHA must match the HEAD of the source branch, otherwise the merge fails.' + optional :squash, type: Grape::API::Boolean, desc: 'If `true`, the commits are squashed into a single commit on merge.' end put ':id/merge_requests/:merge_request_iid/merge', feature_category: :code_review, urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796') @@ -512,7 +671,13 @@ module API end end - desc 'Returns the up to date merge-ref HEAD commit' + desc 'Returns the up to date merge-ref HEAD commit' do + detail 'Returns the up to date merge-ref HEAD commit' + failure [ + { code: 400, message: 'Bad request' } + ] + tags %w[merge_requests] + end get ':id/merge_requests/:merge_request_iid/merge_ref', feature_category: :code_review do merge_request = find_project_merge_request(params[:merge_request_iid]) @@ -525,8 +690,16 @@ module API end end - desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do + desc 'Cancel Merge When Pipeline Succeeds' do + detail 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' success Entities::MergeRequest + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' }, + { code: 405, message: 'Method not allowed' }, + { code: 406, message: 'Not acceptable' } + ] + tags %w[merge_requests] end post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds', feature_category: :code_review do merge_request = find_project_merge_request(params[:merge_request_iid]) @@ -536,11 +709,17 @@ module API AutoMergeService.new(merge_request.target_project, current_user).cancel(merge_request) end - desc 'Rebase the merge request against its target branch' do - detail 'This feature was added in GitLab 11.6' + desc 'Rebase a merge request' do + detail 'Automatically rebase the `source_branch` of the merge request against its `target_branch`. This feature was added in GitLab 11.6' + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' } + ] + tags %w[merge_requests] end params do - optional :skip_ci, type: Boolean, desc: 'Do not create CI pipeline' + optional :skip_ci, type: Boolean, desc: 'Set to true to skip creating a CI pipeline.' end put ':id/merge_requests/:merge_request_iid/rebase', feature_category: :code_review, urgency: :low do merge_request = find_project_merge_request(params[:merge_request_iid]) @@ -554,22 +733,13 @@ module API rescue ::MergeRequest::RebaseLockTimeout => e render_api_error!(e.message, 409) end - - desc 'Remove merge request approvals' do - detail 'This feature was added in GitLab 15.4' - end - put ':id/merge_requests/:merge_request_iid/reset_approvals', feature_category: :code_review, urgency: :low do - merge_request = find_project_merge_request(params[:merge_request_iid]) - - unauthorized! unless current_user.bot? && merge_request.eligible_for_approval_by?(current_user) - - merge_request.approvals.delete_all - - status :accepted - end - - desc 'List issues that will be closed on merge' do + desc 'List issues that close on merge' do + detail 'Get all the issues that would be closed by merging the provided merge request.' success Entities::MRNote + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[merge_requests] end params do use :pagination diff --git a/lib/api/ml/mlflow.rb b/lib/api/ml/mlflow.rb index 56bfac1530e..54bbe0ee465 100644 --- a/lib/api/ml/mlflow.rb +++ b/lib/api/ml/mlflow.rb @@ -126,14 +126,31 @@ module API end params do requires :name, type: String, desc: 'Experiment name' + optional :tags, type: Array, desc: 'Tags with information about the experiment' optional :artifact_location, type: String, desc: 'This will be ignored' - optional :tags, type: Array, desc: 'This will be ignored' end post 'create', urgency: :low do - present experiment_repository.create!(params[:name]), with: Entities::Ml::Mlflow::NewExperiment + present experiment_repository.create!(params[:name], params[:tags]), + with: Entities::Ml::Mlflow::NewExperiment rescue ActiveRecord::RecordInvalid resource_already_exists! end + + desc 'Sets a tag for an experiment.' do + summary 'Sets a tag for an experiment. ' + + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-experiment-tag' + end + params do + requires :experiment_id, type: String, desc: 'ID of the experiment.' + requires :key, type: String, desc: 'Name for the tag.' + requires :value, type: String, desc: 'Value for the tag.' + end + post 'set-experiment-tag', urgency: :low do + bad_request! unless experiment_repository.add_tag!(experiment, params[:key], params[:value]) + + {} + end end resource :runs do @@ -148,10 +165,10 @@ module API desc: 'Unix timestamp in milliseconds of when the run started.', default: 0 optional :user_id, type: String, desc: 'This will be ignored' - optional :tags, type: Array, desc: 'This will be ignored' + optional :tags, type: Array, desc: 'Tags are stored, but not displayed' end post 'create', urgency: :low do - present candidate_repository.create!(experiment, params[:start_time]), + present candidate_repository.create!(experiment, params[:start_time], params[:tags]), with: Entities::Ml::Mlflow::Run, packages_url: packages_url end @@ -229,6 +246,22 @@ module API {} end + desc 'Sets a tag for a run.' do + summary 'Sets a tag for a run. ' + + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#set-tag' + end + params do + requires :run_id, type: String, desc: 'UUID of the run.' + requires :key, type: String, desc: 'Name for the tag.' + requires :value, type: String, desc: 'Value for the tag.' + end + post 'set-tag', urgency: :low do + bad_request! unless candidate_repository.add_tag!(candidate, params[:key], params[:value]) + + {} + end + desc 'Logs multiple parameters and metrics.' do summary 'Log a batch of metrics and params for a run. Validation errors will block the entire batch, '\ 'duplicate errors will be ignored.' @@ -251,6 +284,7 @@ module API post 'log-batch', urgency: :low do candidate_repository.add_metrics(candidate, params[:metrics]) candidate_repository.add_params(candidate, params[:params]) + candidate_repository.add_tags(candidate, params[:tags]) {} end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index eeb66c86b3b..2b1007e715a 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + NAMESPACES_TAGS = %w[namespaces].freeze + helpers do params :optional_list_params_ee do # EE::API::Namespaces would override this helper @@ -20,12 +22,18 @@ module API prepend_mod_with('API::Namespaces') # rubocop: disable Cop/InjectEnterpriseEditionModule resource :namespaces do - desc 'Get a namespaces list' do + desc 'List namespaces' do + detail 'Get a list of the namespaces of the authenticated user. If the user is an administrator, a list of all namespaces in the GitLab instance is shown.' success Entities::Namespace + failure [ + { code: 401, message: 'Unauthorized' } + ] + is_array true + tags NAMESPACES_TAGS end params do - optional :search, type: String, desc: "Search query for namespaces" - optional :owned_only, type: Boolean, desc: "Owned namespaces only" + optional :search, type: String, desc: 'Returns a list of namespaces the user is authorized to view based on the search criteria' + optional :owned_only, type: Boolean, desc: 'In GitLab 14.2 and later, returns a list of owned namespaces only' use :pagination use :optional_list_params_ee @@ -46,11 +54,17 @@ module API present paginate(namespaces), options.reverse_merge(custom_namespace_present_options) end - desc 'Get a namespace by ID' do + desc 'Get namespace by ID' do + detail 'Get a namespace by ID' success Entities::Namespace + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags NAMESPACES_TAGS end params do - requires :id, type: String, desc: "Namespace's ID or path" + requires :id, types: [String, Integer], desc: 'ID or URL-encoded path of the namespace' end get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do user_namespace = find_namespace!(params[:id]) @@ -58,12 +72,17 @@ module API present user_namespace, with: Entities::Namespace, current_user: current_user end - desc 'Get existence of a namespace including alternative suggestions' do + desc 'Get existence of a namespace' do + detail 'Get existence of a namespace by path. Suggests a new namespace path that does not already exist.' success Entities::NamespaceExistence + failure [ + { code: 401, message: 'Unauthorized' } + ] + tags NAMESPACES_TAGS end params do - requires :namespace, type: String, desc: "Namespace's path" - optional :parent_id, type: Integer, desc: "The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered." + requires :namespace, type: String, desc: "Namespace’s path" + optional :parent_id, type: Integer, desc: 'The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered.' end get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do check_rate_limit!(:namespace_exists, scope: current_user) diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 494b493f5e0..f42ded5ac09 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -16,6 +16,12 @@ module API namespace 'projects/:id/packages/npm' do desc 'Download the NPM tarball' do detail 'This feature was introduced in GitLab 11.8' + success code: 200 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[npm_packages] end params do requires :package_name, type: String, desc: 'Package name' @@ -33,13 +39,21 @@ module API package_file = ::Packages::PackageFileFinder .new(package, params[:file_name]).execute! - track_package_event('pull_package', package, category: 'API::NpmPackages', project: project, namespace: project.namespace) + track_package_event('pull_package', :npm, category: 'API::NpmPackages', project: project, namespace: project.namespace) present_package_file!(package_file) end desc 'Create NPM package' do detail 'This feature was introduced in GitLab 11.8' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[npm_packages] end params do requires :package_name, type: String, desc: 'Package name' diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb index eb55e4cbf70..c93b24ee544 100644 --- a/lib/api/nuget_group_packages.rb +++ b/lib/api/nuget_group_packages.rb @@ -45,7 +45,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a group', regexp: ::API::Concerns::Packages::NugetEndpoints::POSITIVE_INTEGER_REGEX + requires :id, types: [Integer, String], desc: 'The group ID or full group path.', regexp: ::API::Concerns::Packages::NugetEndpoints::POSITIVE_INTEGER_REGEX end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index d549a8be035..aa517661791 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -39,18 +39,19 @@ module API end def project_or_group - authorized_user_project + authorized_user_project(action: :read_package) end def snowplow_gitlab_standard_context - { project: authorized_user_project, namespace: authorized_user_project.namespace } + { project: project_or_group, namespace: project_or_group.namespace } end def authorize_nuget_upload + project = project_or_group authorize_workhorse!( - subject: project_or_group, + subject: project, has_length: false, - maximum_size: project_or_group.actual_limits.nuget_max_file_size + maximum_size: project.actual_limits.nuget_max_file_size ) end @@ -67,8 +68,9 @@ module API end def upload_nuget_package_file(symbol_package: false) - authorize_upload!(project_or_group) - bad_request!('File is too large') if project_or_group.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size) + project = project_or_group + authorize_upload!(project) + bad_request!('File is too large') if project.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size) file_params = params.merge( file: params[:package], @@ -76,7 +78,7 @@ module API ) package = ::Packages::CreateTemporaryPackageService.new( - project_or_group, current_user, declared_params.merge(build: current_authenticated_job) + project, current_user, declared_params.merge(build: current_authenticated_job) ).execute(:nuget, name: temp_file_name(symbol_package)) package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)) @@ -100,6 +102,14 @@ module API # https://docs.microsoft.com/en-us/nuget/api/package-publish-resource desc 'The NuGet Package Publish endpoint' do detail 'This feature was introduced in GitLab 12.6' + success code: 201 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end params do @@ -121,6 +131,17 @@ module API forbidden! end + + desc 'The NuGet Package Authorize endpoint' do + detail 'This feature was introduced in GitLab 14.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end put 'authorize', urgency: :low do authorize_nuget_upload end @@ -128,8 +149,15 @@ module API # https://docs.microsoft.com/en-us/nuget/api/symbol-package-publish-resource desc 'The NuGet Symbol Package Publish endpoint' do detail 'This feature was introduced in GitLab 14.1' + success code: 201 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end - params do use :file_params end @@ -149,13 +177,24 @@ module API forbidden! end + + desc 'The NuGet Symbol Package Authorize endpoint' do + detail 'This feature was introduced in GitLab 14.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end put 'symbolpackage/authorize', urgency: :low do authorize_nuget_upload end # https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource params do - requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX + requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'mynugetpkg.1.3.0.17.nupkg' } end namespace '/download/*package_name' do after_validation do @@ -164,6 +203,13 @@ module API desc 'The NuGet Content Service - index request' do detail 'This feature was introduced in GitLab 12.8' + success code: 200, model: ::API::Entities::Nuget::PackagesVersions + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end get 'index', format: :json, urgency: :low do present ::Packages::Nuget::PackagesVersionsPresenter.new(find_packages(params[:package_name])), @@ -172,10 +218,17 @@ module API desc 'The NuGet Content Service - content request' do detail 'This feature was introduced in GitLab 12.8' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] end params do - requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX - requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX + requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: '1.3.0.17' } + requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'mynugetpkg.1.3.0.17.nupkg' } end get '*package_version/*package_filename', format: [:nupkg, :snupkg], urgency: :low do filename = "#{params[:package_filename]}.#{params[:format]}" diff --git a/lib/api/pages.rb b/lib/api/pages.rb index 7e230bd3c67..0cedf7d975f 100644 --- a/lib/api/pages.rb +++ b/lib/api/pages.rb @@ -10,11 +10,18 @@ module API end params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' + requires :id, types: [String, Integer], + desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Unpublish pages' do - detail 'This feature was introduced in GitLab 12.6' + detail 'Remove pages. The user must have administrator access. This feature was introduced in GitLab 12.6' + success code: 204 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pages] end delete ':id/pages' do authorize! :remove_pages, user_project diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index c5add42decc..e5e6ccdf025 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -20,9 +20,15 @@ module API end route_setting :authentication, job_token_allowed: true, job_token_scope: :project resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get a project container repositories' do + desc 'List container repositories within a project' do detail 'This feature was introduced in GitLab 11.8.' success Entities::ContainerRegistry::Repository + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + is_array true + tags %w[container_registry] end params do use :pagination @@ -41,6 +47,13 @@ module API desc 'Delete repository' do detail 'This feature was introduced in GitLab 11.8.' + success status: :accepted, message: 'Success' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + is_array true + tags %w[container_registry] end params do requires :repository_id, type: Integer, desc: 'The ID of the repository' @@ -49,18 +62,20 @@ module API authorize_admin_container_image! repository.delete_scheduled! - unless Feature.enabled?(:container_registry_delete_repository_with_cron_worker) - DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker - end - track_package_event('delete_repository', :container, user: current_user, project: user_project, namespace: user_project.namespace) status :accepted end - desc 'Get a list of repositories tags' do + desc 'List tags of a repository' do detail 'This feature was introduced in GitLab 11.8.' success Entities::ContainerRegistry::Tag + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + is_array true + tags %w[container_registry] end params do requires :repository_id, type: Integer, desc: 'The ID of the repository' @@ -77,6 +92,13 @@ module API desc 'Delete repository tags (in bulk)' do detail 'This feature was introduced in GitLab 11.8.' + success status: :accepted, message: 'Success' + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[container_registry] end params do requires :repository_id, type: Integer, desc: 'The ID of the repository' @@ -104,9 +126,15 @@ module API status :accepted end - desc 'Get a details about repository tag' do + desc 'Get details about a repository tag' do detail 'This feature was introduced in GitLab 11.8.' success Entities::ContainerRegistry::TagDetails + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[container_registry] end params do requires :repository_id, type: Integer, desc: 'The ID of the repository' @@ -121,6 +149,13 @@ module API desc 'Delete repository tag' do detail 'This feature was introduced in GitLab 11.8.' + success status: :ok, message: 'Success' + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[container_registry] end params do requires :repository_id, type: Integer, desc: 'The ID of the repository' diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index d09c481403f..158ba7465f4 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -17,9 +17,15 @@ module API requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get all project packages' do + desc 'Get a list of project packages' do detail 'This feature was introduced in GitLab 11.8' - success ::API::Entities::Package + success code: 200, model: ::API::Entities::Package + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Project Not Found' } + ] + is_array true + tags %w[project_packages] end params do use :pagination @@ -48,7 +54,12 @@ module API desc 'Get a single project package' do detail 'This feature was introduced in GitLab 11.9' - success ::API::Entities::Package + success code: 200, model: ::API::Entities::Package + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[project_packages] end params do requires :package_id, type: Integer, desc: 'The ID of a package' @@ -58,11 +69,19 @@ module API package = ::Packages::PackageFinder .new(user_project, params[:package_id]).execute + render_api_error!('Package not found', 404) unless package.default? + present package, with: ::API::Entities::Package, user: current_user, namespace: user_project.namespace end - desc 'Remove a package' do + desc 'Delete a project package' do detail 'This feature was introduced in GitLab 11.9' + success code: 204 + failure [ + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[project_packages] end params do requires :package_id, type: Integer, desc: 'The ID of a package' diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 93ffb23fea8..7ef722301ca 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -6,7 +6,7 @@ module API before { check_snippets_enabled } - feature_category :snippets + feature_category :source_code_management params do requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index fc898c30a71..de39419b70b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -207,7 +207,10 @@ module API resource :users, requirements: API::USER_REQUIREMENTS do desc 'Get a user projects' do - success Entities::BasicProjectDetails + success code: 200, model: Entities::BasicProjectDetails + failure [{ code: 404, message: '404 User Not Found' }] + tags %w[projects] + is_array true end params do requires :user_id, type: String, desc: 'The ID or username of the user' @@ -225,7 +228,10 @@ module API end desc 'Get projects starred by a user' do - success Entities::BasicProjectDetails + success code: 200, model: Entities::BasicProjectDetails + failure [{ code: 404, message: '404 User Not Found' }] + tags %w[projects] + is_array true end params do requires :user_id, type: String, desc: 'The ID or username of the user' @@ -245,7 +251,9 @@ module API include CustomAttributesEndpoints desc 'Get a list of visible projects for authenticated user' do - success Entities::BasicProjectDetails + success code: 200, model: Entities::BasicProjectDetails + tags %w[projects] + is_array true end params do use :collection_params @@ -258,12 +266,18 @@ module API end desc 'Create new project' do - success Entities::Project + success code: 201, model: Entities::Project + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' }, + { code: 400, message: 'Bad request' } + ] + tags %w[projects] end params do - optional :name, type: String, desc: 'The name of the project' - optional :path, type: String, desc: 'The path of the repository' - optional :default_branch, type: String, desc: 'The default branch of the project' + optional :name, type: String, desc: 'The name of the project', documentation: { example: 'New Project' } + optional :path, type: String, desc: 'The path of the repository', documentation: { example: 'new_project' } + optional :default_branch, type: String, desc: 'The default branch of the project', documentation: { example: 'main' } at_least_one_of :name, :path use :optional_create_project_params use :create_params @@ -295,13 +309,19 @@ module API end desc 'Create new project for a specified user. Only available to admin users.' do - success Entities::Project + success code: 201, model: Entities::Project + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' }, + { code: 400, message: 'Bad request' } + ] + tags %w[projects] end params do - requires :name, type: String, desc: 'The name of the project' - requires :user_id, type: Integer, desc: 'The ID of a user' - optional :path, type: String, desc: 'The path of the repository' - optional :default_branch, type: String, desc: 'The default branch of the project' + requires :name, type: String, desc: 'The name of the project', documentation: { example: 'New Project' } + requires :user_id, type: Integer, desc: 'The ID of a user', documentation: { example: 1 } + optional :path, type: String, desc: 'The path of the repository', documentation: { example: 'new_project' } + optional :default_branch, type: String, desc: 'The default branch of the project', documentation: { example: 'main' } use :optional_project_params use :optional_create_project_params use :create_params @@ -339,7 +359,8 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a single project' do - success Entities::ProjectWithAccess + success code: 200, model: Entities::ProjectWithAccess + tags %w[projects] end params do use :statistics_params @@ -364,15 +385,21 @@ module API end desc 'Fork new project for the current user or provided namespace.' do - success Entities::Project + success code: 201, model: Entities::Project + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' }, + { code: 409, message: 'Conflict' } + ] + tags %w[projects] end params do - optional :namespace, type: String, desc: '(deprecated) The ID or name of the namespace that the project will be forked into' - optional :namespace_id, type: Integer, desc: 'The ID of the namespace that the project will be forked into' - optional :namespace_path, type: String, desc: 'The path of the namespace that the project will be forked into' - optional :path, type: String, desc: 'The path that will be assigned to the fork' - optional :name, type: String, desc: 'The name that will be assigned to the fork' - optional :description, type: String, desc: 'The description that will be assigned to the fork' + optional :namespace, type: String, desc: '(deprecated) The ID or name of the namespace that the project will be forked into', documentation: { example: 'gitlab' } + optional :namespace_id, type: Integer, desc: 'The ID of the namespace that the project will be forked into', documentation: { example: 1 } + optional :namespace_path, type: String, desc: 'The path of the namespace that the project will be forked into', documentation: { example: 'new_path/gitlab' } + optional :path, type: String, desc: 'The path that will be assigned to the fork', documentation: { example: 'fork' } + optional :name, type: String, desc: 'The name that will be assigned to the fork', documentation: { example: 'Fork' } + optional :description, type: String, desc: 'The description that will be assigned to the fork', documentation: { example: 'Description' } optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork' optional :mr_default_target_self, type: Boolean, desc: 'Merge requests of this forked project targets itself by default' end @@ -410,7 +437,9 @@ module API end desc 'List forks of this project' do - success Entities::Project + success code: 200, model: Entities::Project + tags %w[projects] + is_array true end params do use :collection_params @@ -422,19 +451,30 @@ module API present_projects forks, request_scope: user_project end - desc 'Check pages access of this project' + desc 'Check pages access of this project' do + success code: 200 + failure [ + { code: 403, message: 'Unauthenticated' } + ] + tags %w[projects] + end get ':id/pages_access', urgency: :low, feature_category: :pages do authorize! :read_pages_content, user_project unless user_project.public_pages? status 200 end desc 'Update an existing project' do - success Entities::Project + success code: 200, model: Entities::Project + failure [ + { code: 400, message: 'Bad request' }, + { code: 403, message: 'Unauthenticated' } + ] + tags %w[projects] end params do - optional :name, type: String, desc: 'The name of the project' - optional :default_branch, type: String, desc: 'The default branch of the project' - optional :path, type: String, desc: 'The path of the repository' + optional :name, type: String, desc: 'The name of the project', documentation: { example: 'project' } + optional :default_branch, type: String, desc: 'The default branch of the project', documentation: { example: 'main' } + optional :path, type: String, desc: 'The path of the repository', documentation: { example: 'group/project' } use :optional_project_params use :optional_update_params @@ -466,7 +506,11 @@ module API end desc 'Archive a project' do - success Entities::Project + success code: 201, model: Entities::Project + failure [ + { code: 403, message: 'Unauthenticated' } + ] + tags %w[projects] end post ':id/archive', feature_category: :projects do authorize!(:archive_project, user_project) @@ -477,7 +521,11 @@ module API end desc 'Unarchive a project' do - success Entities::Project + success code: 201, model: Entities::Project + failure [ + { code: 403, message: 'Unauthenticated' } + ] + tags %w[projects] end post ':id/unarchive', feature_category: :projects, urgency: :default do authorize!(:archive_project, user_project) @@ -488,7 +536,12 @@ module API end desc 'Star a project' do - success Entities::Project + success code: 201, model: Entities::Project + failure [ + { code: 304, message: 'Not modified' }, + { code: 403, message: 'Unauthenticated' } + ] + tags %w[projects] end post ':id/star', feature_category: :projects do if current_user.starred?(user_project) @@ -502,7 +555,12 @@ module API end desc 'Unstar a project' do - success Entities::Project + success code: 201, model: Entities::Project + failure [ + { code: 304, message: 'Not modified' }, + { code: 403, message: 'Unauthenticated' } + ] + tags %w[projects] end post ':id/unstar', feature_category: :projects do if current_user.starred?(user_project) @@ -516,10 +574,16 @@ module API end desc 'Get the users who starred a project' do - success Entities::UserBasic + success code: 200, model: Entities::UserBasic + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags %w[projects] end params do - optional :search, type: String, desc: 'Return list of users matching the search criteria' + optional :search, type: String, desc: 'Return list of users matching the search criteria', documentation: { example: 'user' } use :pagination end get ':id/starrers', feature_category: :projects do @@ -528,23 +592,44 @@ module API present paginate(starrers), with: Entities::UserStarsProject end - desc 'Get languages in project repository' + desc 'Get languages in project repository' do + success code: 200 + failure [ + { code: 404, message: 'Not found' } + ] + is_array true + tags %w[projects] + end get ':id/languages', feature_category: :source_code_management, urgency: :medium do ::Projects::RepositoryLanguagesService .new(user_project, current_user) .execute.to_h { |lang| [lang.name, lang.share] } end - desc 'Delete a project' + desc 'Delete a project' do + success code: 202 + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[projects] + end delete ":id", feature_category: :projects do authorize! :remove_project, user_project delete_project(user_project) end - desc 'Mark this project as forked from another' + desc 'Mark this project as forked from another' do + success code: 201, model: Entities::Project + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[projects] + end params do - requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' + requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from', documentation: { example: 'gitlab' } end post ":id/fork/:forked_from_id", feature_category: :source_code_management do authorize! :admin_project, user_project @@ -559,12 +644,20 @@ module API if result present_project user_project.reset, with: Entities::Project, current_user: current_user - else - render_api_error!("Project already forked", 409) if user_project.forked? + elsif user_project.forked? + render_api_error!("Project already forked", 409) end end - desc 'Remove a forked_from relationship' + desc 'Remove a forked_from relationship' do + success code: 204 + failure [ + { code: 304, message: 'Not modified' }, + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[projects] + end delete ":id/fork", feature_category: :source_code_management do authorize! :remove_fork_project, user_project @@ -576,10 +669,16 @@ module API end desc 'Share the project with a group' do - success Entities::ProjectGroupLink + success code: 201, model: Entities::ProjectGroupLink + failure [ + { code: 400, message: 'Bad request' }, + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[projects] end params do - requires :group_id, type: Integer, desc: 'The ID of a group' + requires :group_id, type: Integer, desc: 'The ID of a group', documentation: { example: 1 } requires :group_access, type: Integer, values: Gitlab::Access.values, as: :link_group_access, desc: 'The group access level' optional :expires_at, type: Date, desc: 'Share expiration date' end @@ -601,6 +700,14 @@ module API end end + desc 'Remove a group share' do + success code: 204 + failure [ + { code: 400, message: 'Bad request' }, + { code: 404, message: 'Not found' } + ] + tags %w[projects] + end params do requires :group_id, type: Integer, desc: 'The ID of the group' end @@ -619,6 +726,12 @@ module API desc 'Import members from another project' do detail 'This feature was introduced in GitLab 14.2' + success code: 201 + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[projects] end params do requires :project_id, type: Integer, desc: 'The ID of the source project to import the members from.' @@ -642,6 +755,11 @@ module API desc 'Workhorse authorize the file upload' do detail 'This feature was introduced in GitLab 13.11' + success code: 200 + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[projects] end post ':id/uploads/authorize', feature_category: :not_owned do # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned require_gitlab_workhorse! @@ -651,7 +769,13 @@ module API FileUploader.workhorse_authorize(has_length: false, maximum_size: project_attachment_size(user_project)) end - desc 'Upload a file' + desc 'Upload a file' do + success code: 201, model: Entities::ProjectUpload + failure [ + { code: 404, message: 'Not found' } + ] + tags %w[projects] + end params do requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded', documentation: { type: 'file' } end @@ -666,10 +790,16 @@ module API end desc 'Get the users list of a project' do - success Entities::UserBasic + success code: 200, model: Entities::UserBasic + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags %w[projects] end params do - optional :search, type: String, desc: 'Return list of users matching the search criteria' + optional :search, type: String, desc: 'Return list of users matching the search criteria', documentation: { example: 'user' } optional :skip_users, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Filter out users with the specified IDs' use :pagination end @@ -683,10 +813,16 @@ module API end desc 'Get ancestor and shared groups for a project' do - success Entities::PublicGroupDetails + success code: 200, model: Entities::PublicGroupDetails + failure [ + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + is_array true + tags %w[projects] end params do - optional :search, type: String, desc: 'Return list of groups matching the search criteria' + optional :search, type: String, desc: 'Return list of groups matching the search criteria', documentation: { example: 'group' } optional :skip_groups, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Array of group ids to exclude from list' optional :with_shared, type: Boolean, default: false, desc: 'Include shared groups' @@ -705,6 +841,13 @@ module API desc 'Start the housekeeping task for a project' do detail 'This feature was introduced in GitLab 9.0.' + success code: 201 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Unauthenticated' }, + { code: 409, message: 'Conflict' } + ] + tags %w[projects] end post ':id/housekeeping', feature_category: :source_code_management do authorize_admin_project @@ -718,6 +861,12 @@ module API desc 'Start a task to recalculate repository size for a project' do detail 'This feature was introduced in GitLab 15.0.' + success code: 201 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Unauthenticated' } + ] + tags %w[projects] end post ':id/repository_size', feature_category: :source_code_management do authorize_admin_project @@ -727,9 +876,17 @@ module API ::Projects::UpdateStatisticsService.new(user_project, nil, statistics: [:repository_size, :lfs_objects_size]).execute end - desc 'Transfer a project to a new namespace' + desc 'Transfer a project to a new namespace' do + success code: 200, model: Entities::Project + failure [ + { code: 400, message: 'Bad request' }, + { code: 403, message: 'Unauthenticated' }, + { code: 404, message: 'Not found' } + ] + tags %w[projects] + end params do - requires :namespace, type: String, desc: 'The ID or path of the new namespace' + requires :namespace, type: String, desc: 'The ID or path of the new namespace', documentation: { example: 'gitlab' } end put ":id/transfer", feature_category: :projects do authorize! :change_namespace, user_project @@ -744,9 +901,16 @@ module API end end - desc 'Get the namespaces to where the project can be transferred' + desc 'Get the namespaces to where the project can be transferred' do + success code: 200, model: Entities::PublicGroupDetails + failure [ + { code: 403, message: 'Unauthenticated' } + ] + is_array true + tags %w[projects] + end params do - optional :search, type: String, desc: 'Return list of namespaces matching the search criteria' + optional :search, type: String, desc: 'Return list of namespaces matching the search criteria', documentation: { example: 'search' } use :pagination end get ":id/transfer_locations", feature_category: :projects do @@ -761,7 +925,11 @@ module API end desc 'Show the storage information' do - success Entities::ProjectRepositoryStorage + success code: 200, model: Entities::ProjectRepositoryStorage + failure [ + { code: 403, message: 'Unauthenticated' } + ] + tags %w[projects] end params do requires :id, type: String, desc: 'ID of a project' diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 6c649483da1..f9470ce1cb6 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -32,12 +32,12 @@ module API helpers do params :package_download do - requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true - requires :sha256, type: String, desc: 'The PyPi package sha256 check sum' + requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true, documentation: { example: 'my.pypi.package-0.0.1.tar.gz' } + requires :sha256, type: String, desc: 'The PyPi package sha256 check sum', documentation: { example: '5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff' } end params :package_name do - requires :package_name, type: String, file_path: true, desc: 'The PyPi package name' + requires :package_name, type: String, file_path: true, desc: 'The PyPi package name', documentation: { example: 'my.pypi.package' } end def present_simple_index(group_or_project) @@ -102,7 +102,7 @@ module API end params do - requires :id, type: String, desc: 'The ID of a group' + requires :id, types: [Integer, String], desc: 'The ID or full path of the group.' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do after_validation do @@ -110,6 +110,16 @@ module API end namespace ':id/-/packages/pypi' do + desc 'Download a package file from a group' do + detail 'This feature was introduced in GitLab 13.12' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pypi_packages] + end params do use :package_download end @@ -123,13 +133,20 @@ module API package = Packages::Pypi::PackageFinder.new(current_user, group, { filename: filename, sha256: params[:sha256] }).execute package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute - track_package_event('pull_package', :pypi) + track_package_event('pull_package', :pypi, namespace: group, project: package.project) present_package_file!(package_file, supports_direct_download: true) end desc 'The PyPi Simple Group Index Endpoint' do detail 'This feature was introduced in GitLab 15.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pypi_packages] end # An API entry point but returns an HTML file instead of JSON. @@ -141,6 +158,13 @@ module API desc 'The PyPi Simple Group Package Endpoint' do detail 'This feature was introduced in GitLab 12.10' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pypi_packages] end params do @@ -164,6 +188,13 @@ module API namespace ':id/packages/pypi' do desc 'The PyPi package download endpoint' do detail 'This feature was introduced in GitLab 12.10' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pypi_packages] end params do @@ -185,6 +216,13 @@ module API desc 'The PyPi Simple Project Index Endpoint' do detail 'This feature was introduced in GitLab 15.1' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pypi_packages] end # An API entry point but returns an HTML file instead of JSON. @@ -196,6 +234,13 @@ module API desc 'The PyPi Simple Project Package Endpoint' do detail 'This feature was introduced in GitLab 12.10' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pypi_packages] end params do @@ -211,15 +256,24 @@ module API desc 'The PyPi Package upload endpoint' do detail 'This feature was introduced in GitLab 12.10' + success code: 201 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' }, + { code: 422, message: 'Unprocessable Entity' } + ] + tags %w[pypi_packages] end params do requires :content, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } - requires :name, type: String - requires :version, type: String - optional :requires_python, type: String - optional :md5_digest, type: String - optional :sha256_digest, type: String, regexp: Gitlab::Regex.sha256_regex + requires :name, type: String, documentation: { example: 'my.pypi.package' } + requires :version, type: String, documentation: { example: '1.3.7' } + optional :requires_python, type: String, documentation: { example: '>=3.7' } + optional :md5_digest, type: String, documentation: { example: '900150983cd24fb0d6963f7d28e17f72' } + optional :sha256_digest, type: String, regexp: Gitlab::Regex.sha256_regex, documentation: { example: 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad' } end route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth @@ -243,6 +297,17 @@ module API forbidden! end + desc 'Authorize the PyPi package upload from workhorse' do + detail 'This feature was introduced in GitLab 12.10' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pypi_packages] + end + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth post 'authorize' do project = project!(action: :read_project) diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index c72f90dfdf3..0e83d086a6e 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -10,13 +10,13 @@ module API RELEASE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) - before { authorize! :read_release, user_project } + after_validation { authorize! :read_release, user_project } feature_category :release_orchestration urgency :low params do - requires :id, type: [String, Integer], desc: 'The ID or URL-encoded path of the project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do diff --git a/lib/api/rpm_project_packages.rb b/lib/api/rpm_project_packages.rb index 40b8d022c6c..f02d288982a 100644 --- a/lib/api/rpm_project_packages.rb +++ b/lib/api/rpm_project_packages.rb @@ -25,7 +25,16 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/rpm' do - desc 'Download repository metadata files' + desc 'Download repository metadata files' do + detail 'This feature was introduced in GitLab 15.7' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[rpm_packages] + end params do requires :file_name, type: String, desc: 'Repository metadata file name' end @@ -40,7 +49,15 @@ module API present_carrierwave_file!(repository_file.file) end - desc 'Download RPM package files' + desc 'Download RPM package files' do + detail 'This feature was introduced in GitLab 15.7' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[rpm_packages] + end params do requires :package_file_id, type: Integer, desc: 'RPM package file id' requires :file_name, type: String, desc: 'RPM package file name' @@ -56,7 +73,16 @@ module API not_found! end - desc 'Upload a RPM package' + desc 'Upload a RPM package' do + detail 'This feature was introduced in GitLab 15.7' + failure [ + { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[rpm_packages] + end post do authorize_create_package!(authorized_user_project) @@ -64,6 +90,10 @@ module API bad_request!('File is too large') end + if Packages::Rpm::RepositoryFile.has_oversized_filelists?(project_id: authorized_user_project.id) + bad_request!('Repository packages limit exceeded') + end + track_package_event( 'push_package', :rpm, @@ -76,7 +106,15 @@ module API not_found! end - desc 'Authorize package upload from workhorse' + desc 'Authorize package upload from workhorse' do + detail 'This feature was introduced in GitLab 15.7' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[rpm_packages] + end post 'authorize' do not_found! end diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 87cf1f66223..af0ceb1acfc 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -28,19 +28,27 @@ module API before do require_packages_enabled! authenticate_non_get! + end + + after_validation do not_found! unless Feature.enabled?(:rubygem_packages, user_project) end params do - requires :id, type: String, desc: 'The ID or full path of a project' + requires :id, types: [Integer, String], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/rubygems' do desc 'Download the spec index file' do detail 'This feature was introduced in GitLab 13.9' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[rubygem_packages] end params do - requires :file_name, type: String, desc: 'Spec file name' + requires :file_name, type: String, desc: 'Spec file name', documentation: { type: 'file' } end get ":file_name", requirements: FILE_NAME_REQUIREMENTS do # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299267 @@ -49,9 +57,14 @@ module API desc 'Download the gemspec file' do detail 'This feature was introduced in GitLab 13.9' + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[rubygem_packages] end params do - requires :file_name, type: String, desc: 'Gemspec file name' + requires :file_name, type: String, desc: 'Gemspec file name', documentation: { type: 'file' } end get "quick/Marshal.#{MARSHAL_VERSION}/:file_name", requirements: FILE_NAME_REQUIREMENTS do # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299284 @@ -60,9 +73,16 @@ module API desc 'Download the .gem package' do detail 'This feature was introduced in GitLab 13.9' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[rubygem_packages] end params do - requires :file_name, type: String, desc: 'Package file name' + requires :file_name, type: String, desc: 'Package file name', documentation: { type: 'file' } end get "gems/:file_name", requirements: FILE_NAME_REQUIREMENTS do authorize_read_package!(user_project) @@ -80,6 +100,12 @@ module API namespace 'api/v1' do desc 'Authorize a gem upload from workhorse' do detail 'This feature was introduced in GitLab 13.9' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' } + ] + tags %w[rubygem_packages] end post 'gems/authorize' do authorize_workhorse!( @@ -91,6 +117,13 @@ module API desc 'Upload a gem' do detail 'This feature was introduced in GitLab 13.9' + success code: 201 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[rubygem_packages] end params do requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } @@ -133,6 +166,14 @@ module API desc 'Fetch a list of dependencies' do detail 'This feature was introduced in GitLab 13.9' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + is_array true + tags %w[rubygem_packages] end params do optional :gems, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma delimited gem names' diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 26b7e58bc7a..8b47604fe86 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -66,6 +66,7 @@ module API requires :eks_secret_access_key, type: String, desc: 'Secret access key for the EKS integration IAM user' end optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' + optional :email_confirmation_setting, type: String, values: ApplicationSetting.email_confirmation_settings.keys, desc: "Email confirmation setting, possible values: `off`, `soft`, and `hard`" optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' optional :gitpod_enabled, type: Boolean, desc: 'Enable Gitpod' given gitpod_enabled: ->(val) { val } do @@ -90,7 +91,7 @@ module API end optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' optional :import_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, - values: %w[github bitbucket bitbucket_server gitlab google_code fogbugz git gitlab_project gitea manifest phabricator], + values: %w[github bitbucket bitbucket_server gitlab fogbugz git gitlab_project gitea manifest phabricator], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :in_product_marketing_emails_enabled, type: Boolean, desc: 'By default, in-product marketing emails are enabled. To disable these emails, disable this option.' optional :invisible_captcha_enabled, type: Boolean, desc: 'Enable Invisible Captcha spam detection during signup.' @@ -100,6 +101,7 @@ module API optional :max_import_size, type: Integer, desc: 'Maximum import size in MB' optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' optional :max_pages_custom_domains_per_project, type: Integer, desc: 'Maximum number of GitLab Pages custom domains per project' + optional :max_terraform_state_size_bytes, type: Integer, desc: "Maximum size in bytes of the Terraform state file. Set this to 0 for unlimited file size." optional :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' @@ -139,7 +141,6 @@ module API requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' end optional :restricted_visibility_levels, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' - optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' given shared_runners_enabled: ->(val) { val } do @@ -184,6 +185,9 @@ module API optional :group_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for group runners, in seconds' optional :project_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for project runners, in seconds' optional :pipeline_limit_per_project_user_sha, type: Integer, desc: "Maximum number of pipeline creation requests allowed per minute per user and commit. Set to 0 for unlimited requests per minute." + optional :jira_connect_application_key, type: String, desc: "Application ID of the OAuth application that should be used to authenticate with the GitLab.com for Jira Cloud app" + optional :jira_connect_proxy_url, type: String, desc: "URL of the GitLab instance that should be used as a proxy for the GitLab.com for Jira Cloud app" + optional :bulk_import_enabled, type: Boolean, desc: 'Enable migrating GitLab groups and projects by direct transfer' Gitlab::SSHPublicKey.supported_types.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 36698a220bd..104848206a3 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -5,7 +5,7 @@ module API class Snippets < ::API::Base include PaginationParams - feature_category :snippets + feature_category :source_code_management urgency :low resource :snippets do diff --git a/lib/api/support/git_access_actor.rb b/lib/api/support/git_access_actor.rb index 16861a146ae..7a4e6f3e14c 100644 --- a/lib/api/support/git_access_actor.rb +++ b/lib/api/support/git_access_actor.rb @@ -16,7 +16,7 @@ module API def self.from_params(params) if params[:key_id] - new(key: Key.find_by_id(params[:key_id])) + new(key: Key.auth.find_by_id(params[:key_id])) elsif params[:user_id] new(user: UserFinder.new(params[:user_id]).find_by_id) elsif params[:username] diff --git a/lib/api/tags.rb b/lib/api/tags.rb index b412a17bc6f..4ddf22c726f 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -129,6 +129,24 @@ module API end end end + + desc "Get a tag's signature" do + success code: 200, model: Entities::TagSignature + tags %w[tags] + failure [ + { code: 404, message: 'Not found' } + ] + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end + get ':id/repository/tags/:tag_name/signature', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :source_code_management do + tag = user_project.repository.find_tag(params[:tag_name]) + not_found! 'Tag' unless tag + not_found! 'Signature' unless tag.has_signature? + + present tag, with: Entities::TagSignature + end end end end diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index 577d011ebad..bdc9f975970 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -20,6 +20,8 @@ module API render_api_error!(e.message, 422) end + STATE_NAME_URI_REQUIREMENTS = { name: API::NO_SLASH_URL_PART_REGEX }.freeze + before do authenticate! authorize! :read_terraform_state, user_project @@ -45,7 +47,7 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - namespace ':id/terraform/state/:name' do + namespace ':id/terraform/state/:name', requirements: STATE_NAME_URI_REQUIREMENTS do params do requires :name, type: String, desc: 'The name of a Terraform state' optional :ID, type: String, limit: 255, desc: 'Terraform state lock ID' @@ -55,6 +57,17 @@ module API def remote_state_handler ::Terraform::RemoteStateHandler.new(user_project, current_user, name: params[:name], lock_id: params[:ID]) end + + def not_found_for_dots? + Feature.disabled?(:allow_dots_on_tf_state_names) && params[:name].include?(".") + end + + # Change the state name to behave like before, https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105674 + # has been introduced. This behavior can be controlled via `allow_dots_on_tf_state_names` FF. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106861 + def legacy_state_name! + params[:name] = params[:name].split('.').first + end end desc 'Get a Terraform state by its name' do @@ -72,6 +85,8 @@ module API end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get do + legacy_state_name! if not_found_for_dots? + remote_state_handler.find_with_lock do |state| no_content! unless state.latest_file && state.latest_file.exists? @@ -88,17 +103,22 @@ module API ] failure [ { code: 403, message: 'Forbidden' }, - { code: 422, message: 'Validation failure' } + { code: 422, message: 'Validation failure' }, + { code: 413, message: 'Request Entity Too Large' } ] tags %w[terraform_state] end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth post do authorize! :admin_terraform_state, user_project + legacy_state_name! if not_found_for_dots? data = request.body.read no_content! if data.empty? + max_state_size = Gitlab::CurrentSettings.max_terraform_state_size_bytes + file_too_large! if max_state_size > 0 && data.size > max_state_size + remote_state_handler.handle_with_lock do |state| state.update_file!(CarrierWaveStringFile.new(data), version: params[:serial], build: current_authenticated_job) end @@ -120,6 +140,7 @@ module API route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth delete do authorize! :admin_terraform_state, user_project + legacy_state_name! if not_found_for_dots? remote_state_handler.find_with_lock do |state| ::Terraform::States::TriggerDestroyService.new(state, current_user: current_user).execute @@ -151,6 +172,8 @@ module API requires :Path, type: String, desc: 'Terraform path' end post '/lock' do + not_found! if not_found_for_dots? + authorize! :admin_terraform_state, user_project status_code = :ok @@ -194,6 +217,8 @@ module API optional :ID, type: String, limit: 255, desc: 'Terraform state lock ID' end delete '/lock' do + not_found! if not_found_for_dots? + authorize! :admin_terraform_state, user_project remote_state_handler.unlock! diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index b8323304957..dd8ad2cc144 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -23,14 +23,12 @@ module API end def load_issuable - @issuable ||= begin - case issuable_name - when 'issue' - find_project_issue(params.delete(issuable_key)) - when 'merge_request' - find_project_merge_request(params.delete(issuable_key)) - end - end + @issuable ||= case issuable_name + when 'issue' + find_project_issue(params.delete(issuable_key)) + when 'merge_request' + find_project_merge_request(params.delete(issuable_key)) + end end def update_issuable(attrs) @@ -54,10 +52,18 @@ module API issuable_collection_name = issuable_name.pluralize issuable_key = "#{issuable_name}_iid".to_sym - desc "Set a time estimate for a project #{issuable_name}" + desc "Set a time estimate for a #{issuable_name}" do + detail " Sets an estimated time of work for this #{issuable_name}." + success Entities::IssuableTimeStats + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags [issuable_collection_name] + end params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" - requires :duration, type: String, desc: 'The duration to be parsed' + requires issuable_key, type: Integer, desc: "The internal ID of the #{issuable_name}." + requires :duration, type: String, desc: 'The duration in human format.', documentation: { example: '3h30m' } end post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do authorize! admin_issuable_key, load_issuable @@ -66,9 +72,17 @@ module API update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration))) end - desc "Reset the time estimate for a project #{issuable_name}" + desc "Reset the time estimate for a project #{issuable_name}" do + detail "Resets the estimated time for this #{issuable_name} to 0 seconds." + success Entities::IssuableTimeStats + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags [issuable_collection_name] + end params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires issuable_key, type: Integer, desc: "The internal ID of the #{issuable_name}." end post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do authorize! admin_issuable_key, load_issuable @@ -77,10 +91,18 @@ module API update_issuable(time_estimate: 0) end - desc "Add spent time for a project #{issuable_name}" + desc "Add spent time for a #{issuable_name}" do + detail "Adds spent time for this #{issuable_name}." + success Entities::IssuableTimeStats + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags [issuable_collection_name] + end params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" - requires :duration, type: String, desc: 'The duration to be parsed' + requires issuable_key, type: Integer, desc: "The internal ID of the #{issuable_name}." + requires :duration, type: String, desc: 'The duration in human format.' end post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do authorize! admin_issuable_key, load_issuable @@ -97,9 +119,17 @@ module API update_issuable(update_params) end - desc "Reset spent time for a project #{issuable_name}" + desc "Reset spent time for a #{issuable_name}" do + detail "Resets the total spent time for this #{issuable_name} to 0 seconds." + success Entities::IssuableTimeStats + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags [issuable_collection_name] + end params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires issuable_key, type: Integer, desc: "The internal ID of the #{issuable_name}" end post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do authorize! admin_issuable_key, load_issuable @@ -108,9 +138,17 @@ module API update_issuable(spend_time: { duration: :reset, user_id: current_user.id }) end - desc "Show time stats for a project #{issuable_name}" + desc "Get time tracking stats" do + detail "Get time tracking stats" + success Entities::IssuableTimeStats + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + tags [issuable_collection_name] + end params do - requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires issuable_key, type: Integer, desc: "The internal ID of the #{issuable_name}" end get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do authorize! read_issuable_key, load_issuable diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb index 38ce4bd7f32..b2133dc743a 100644 --- a/lib/api/unleash.rb +++ b/lib/api/unleash.rb @@ -63,13 +63,9 @@ module API cache_context: -> (client) { client.unleash_api_cache_key } end - def project - @project ||= find_project(params[:project_id]) - end - def feature_flags_client strong_memoize(:feature_flags_client) do - client = Operations::FeatureFlagsClient.find_for_project_and_token(project, unleash_instance_id) + client = Operations::FeatureFlagsClient.find_for_project_and_token(params[:project_id], unleash_instance_id) client.unleash_app_name = unleash_app_name if client client end @@ -86,12 +82,6 @@ module API def authorize_by_unleash_instance_id! unauthorized! unless feature_flags_client end - - def feature_flags - return [] unless unleash_app_name.present? - - Operations::FeatureFlag.for_unleash_client(project, unleash_app_name) - end end end end diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index 9e446aff605..3e2023d769f 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -12,11 +12,18 @@ module API forbidden!('Invalid CSRF token is provided') unless verified_request? end - desc 'Track usage data events' do + desc 'Track usage data event' do detail 'This feature was introduced in GitLab 13.4.' + success code: 200 + failure [ + { code: 403, message: 'Invalid CSRF token is provided' }, + { code: 404, message: 'Not found' } + ] + tags %w[usage_data] end params do - requires :event, type: String, desc: 'The event name that should be tracked' + requires :event, type: String, desc: 'The event name that should be tracked', + documentation: { example: 'i_quickactions_page' } end post 'increment_counter' do event_name = params[:event] @@ -26,8 +33,17 @@ module API status :ok end + desc 'Track usage data event for the current user' do + success code: 200 + failure [ + { code: 403, message: 'Invalid CSRF token is provided' }, + { code: 404, message: 'Not found' } + ] + tags %w[usage_data] + end params do - requires :event, type: String, desc: 'The event name that should be tracked' + requires :event, type: String, desc: 'The event name that should be tracked', + documentation: { example: 'i_quickactions_page' } end post 'increment_unique_users', urgency: :low do event_name = params[:event] @@ -39,6 +55,13 @@ module API desc 'Get a list of all metric definitions' do detail 'This feature was introduced in GitLab 13.11.' + success code: 200 + failure [ + { code: 403, message: 'Invalid CSRF token is provided' }, + { code: 404, message: 'Not found' } + ] + produces ['application/yaml'] + tags %w[usage_data metrics] end get 'metric_definitions', urgency: :low do content_type 'application/yaml' diff --git a/lib/api/usage_data_non_sql_metrics.rb b/lib/api/usage_data_non_sql_metrics.rb index 41f369a43b8..81f96a7958b 100644 --- a/lib/api/usage_data_non_sql_metrics.rb +++ b/lib/api/usage_data_non_sql_metrics.rb @@ -14,6 +14,12 @@ module API desc 'Get Non SQL usage ping metrics' do detail 'This feature was introduced in GitLab 13.11.' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] end get 'non_sql_metrics' do diff --git a/lib/api/usage_data_queries.rb b/lib/api/usage_data_queries.rb index fe972942111..8e85fca4ba9 100644 --- a/lib/api/usage_data_queries.rb +++ b/lib/api/usage_data_queries.rb @@ -14,6 +14,12 @@ module API desc 'Get raw SQL queries for usage data SQL metrics' do detail 'This feature was introduced in GitLab 13.11.' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] end get 'queries' do diff --git a/lib/api/users.rb b/lib/api/users.rb index 72c121bca03..d2d45c94291 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -133,7 +133,7 @@ module API get feature_category: :users, urgency: :low do authenticated_as_admin! if params[:extern_uid].present? && params[:provider].present? - unless current_user&.admin? + unless current_user&.can_read_all_resources? params.except!(:created_after, :created_before, :order_by, :sort, :two_factor, :without_projects) end @@ -151,7 +151,7 @@ module API users = UsersFinder.new(current_user, params).execute users = reorder_users(users) - entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic + entity = current_user&.can_read_all_resources? ? Entities::UserWithAdmin : Entities::UserBasic if entity == Entities::UserWithAdmin users = users.preload(:identities, :u2f_registrations, :webauthn_registrations, :namespace, :followers, :followees, :user_preference) @@ -177,7 +177,7 @@ module API get ":id", feature_category: :users, urgency: :low do forbidden!('Not authorized!') unless current_user - unless current_user.admin? + unless current_user.can_read_all_resources? check_rate_limit!(:users_get_by_id, scope: current_user, users_allowlist: Gitlab::CurrentSettings.current_application_settings.users_get_by_id_limit_allowlist @@ -188,7 +188,7 @@ module API not_found!('User') unless user && can?(current_user, :read_user, user) - opts = { with: current_user.admin? ? Entities::UserDetailsWithAdmin : Entities::User, current_user: current_user } + opts = { with: current_user.can_read_all_resources? ? Entities::UserDetailsWithAdmin : Entities::User, current_user: current_user } user, opts = with_custom_attributes(user, opts) present user, opts @@ -333,12 +333,12 @@ module API not_found!('User') unless user conflict!('Email has already been taken') if params[:email] && - User.by_any_email(params[:email].downcase) - .where.not(id: user.id).exists? + User.by_any_email(params[:email].downcase) + .where.not(id: user.id).exists? conflict!('Username has already been taken') if params[:username] && - User.by_username(params[:username]) - .where.not(id: user.id).exists? + User.by_username(params[:username]) + .where.not(id: user.id).exists? user_params = declared_params(include_missing: false) admin_making_changes_for_another_user = (current_user != user) @@ -373,7 +373,8 @@ module API user = User.find_by_id(params[:id]) not_found!('User') unless user - forbidden!('Two-factor authentication for admins cannot be disabled via the API. Use the Rails console') if user.admin? + # We're disabling Cop/UserAdmin because it checks if the given user (not the current user) is an admin. + forbidden!('Two-factor authentication for admins cannot be disabled via the API. Use the Rails console') if user.admin? # rubocop:disable Cop/UserAdmin result = TwoFactor::DestroyService.new(current_user, user: user).execute @@ -437,6 +438,8 @@ module API requires :key, type: String, desc: 'The new SSH key' requires :title, type: String, desc: 'The title of the new SSH key' optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)' + optional :usage_type, type: String, values: Key.usage_types.keys, default: 'auth_and_signing', + desc: 'Scope of usage for the SSH key' end # rubocop: disable CodeReuse/ActiveRecord post ":user_id/keys", feature_category: :authentication_and_authorization do @@ -1006,7 +1009,8 @@ module API end get feature_category: :users, urgency: :low do entity = - if current_user.admin? + # We're disabling Cop/UserAdmin because it checks if the given user is an admin. + if current_user.admin? # rubocop:disable Cop/UserAdmin Entities::UserWithAdmin else Entities::UserPublic @@ -1050,6 +1054,8 @@ module API requires :key, type: String, desc: 'The new SSH key' requires :title, type: String, desc: 'The title of the new SSH key' optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)' + optional :usage_type, type: String, values: Key.usage_types.keys, default: 'auth_and_signing', + desc: 'Scope of usage for the SSH key' end post "keys", feature_category: :authentication_and_authorization do key = ::Keys::CreateService.new(current_user, declared_params(include_missing: false)).execute diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index e4a26838746..db71e823b1d 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -78,13 +78,13 @@ module API # rubocop: enable CodeReuse/ActiveRecord def authorized_merge_requests - MergeRequestsFinder.new(current_user, authorized_only: !current_user.admin?) + MergeRequestsFinder.new(current_user, authorized_only: !current_user.can_read_all_resources?) .execute.with_jira_integration_associations end def authorized_merge_requests_for_project(project) MergeRequestsFinder - .new(current_user, authorized_only: !current_user.admin?, project_id: project.id) + .new(current_user, authorized_only: !current_user.can_read_all_resources?, project_id: project.id) .execute.with_jira_integration_associations end diff --git a/lib/api/validations/validators/array_none_any.rb b/lib/api/validations/validators/array_none_any.rb index 3732c1f575c..8c064eefbf2 100644 --- a/lib/api/validations/validators/array_none_any.rb +++ b/lib/api/validations/validators/array_none_any.rb @@ -8,7 +8,7 @@ module API value = params[attr_name] return if value.is_a?(Array) || - [IssuableFinder::Params::FILTER_NONE, IssuableFinder::Params::FILTER_ANY].include?(value.to_s.downcase) + [IssuableFinder::Params::FILTER_NONE, IssuableFinder::Params::FILTER_ANY].include?(value.to_s.downcase) raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], diff --git a/lib/assets/images/bot_avatars/admin-bot.png b/lib/assets/images/bot_avatars/admin-bot.png Binary files differnew file mode 100644 index 00000000000..8418676d9b1 --- /dev/null +++ b/lib/assets/images/bot_avatars/admin-bot.png diff --git a/lib/atlassian/jira_connect.rb b/lib/atlassian/jira_connect.rb index 595cf0ac465..3ebed08280a 100644 --- a/lib/atlassian/jira_connect.rb +++ b/lib/atlassian/jira_connect.rb @@ -17,8 +17,16 @@ module Atlassian private def gitlab_host + return host_from_settings if Gitlab::CurrentSettings.jira_connect_proxy_url? + Gitlab.config.gitlab.host end + + def host_from_settings + uri = URI(Gitlab::CurrentSettings.jira_connect_proxy_url) + + uri.hostname + uri.path + end end end end diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index 3c999920d39..96e285f4eea 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -76,7 +76,7 @@ module Atlassian return if items.empty? r = post('/rest/deployments/0.1/bulk', { deployments: items }) - handle_response(r, 'deployments') { |data| errors(data, 'rejectedDeployments') } + handle_response(r, 'deployments') { |data| errors(data, 'rejectedDeployments', r) } end def store_build_info(project:, pipelines:, update_sequence_id: nil) @@ -92,7 +92,7 @@ module Atlassian return if builds.empty? r = post('/rest/builds/0.1/bulk', { builds: builds }) - handle_response(r, 'builds') { |data| errors(data, 'rejectedBuilds') } + handle_response(r, 'builds') { |data| errors(data, 'rejectedBuilds', r) } end def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil) @@ -132,22 +132,20 @@ module Atlassian if [200, 202].include?(response.code) yield data else - message = case response.code - when 400 then { 'errorMessages' => data.map { |e| e['message'] } } - when 401 then { 'errorMessages' => ['Invalid JWT'] } - when 403 then { 'errorMessages' => ["App does not support #{name}"] } - when 413 then { 'errorMessages' => ['Data too large'] + data.map { |e| e['message'] } } - when 429 then { 'errorMessages' => ['Rate limit exceeded'] } - when 503 then { 'errorMessages' => ['Service unavailable'] } - else - { 'errorMessages' => ['Unknown error'], 'response' => data } - end - - message.merge('responseCode' => response.code) + case response.code + when 400 then { 'errorMessages' => data.map { |e| e['message'] } } + when 401 then { 'errorMessages' => ['Invalid JWT'] } + when 403 then { 'errorMessages' => ["App does not support #{name}"] } + when 413 then { 'errorMessages' => ['Data too large'] + data.map { |e| e['message'] } } + when 429 then { 'errorMessages' => ['Rate limit exceeded'] } + when 503 then { 'errorMessages' => ['Service unavailable'] } + else + { 'errorMessages' => ['Unknown error'], 'response' => data } + end.merge('responseCode' => response.code) end end - def errors(data, key) + def errors(data, key, response) messages = if data[key].present? data[key].flat_map do |rejection| rejection['errors'].map { |e| e['message'] } @@ -156,7 +154,13 @@ module Atlassian [] end - { 'errorMessages' => messages } + { 'errorMessages' => messages, 'responseCode' => response.code, 'requestBody' => request_body_schema(response) } + end + + def request_body_schema(response) + Oj.load(response.request.raw_body).deep_transform_values! {} + rescue Oj::ParseError, EncodingError, Encoding::UndefinedConversionError + 'Request body includes invalid JSON' end def user_notes_count(merge_requests) diff --git a/lib/atlassian/jira_connect/jwt/asymmetric.rb b/lib/atlassian/jira_connect/jwt/asymmetric.rb index 573a8022752..7c1cf1cabb6 100644 --- a/lib/atlassian/jira_connect/jwt/asymmetric.rb +++ b/lib/atlassian/jira_connect/jwt/asymmetric.rb @@ -77,11 +77,7 @@ module Atlassian end def public_key_cdn_url - if public_key_cdn_url_setting.blank? || Feature.disabled?(:jira_connect_oauth_self_managed) - return DEFAULT_PUBLIC_KEY_CDN_URL - end - - public_key_cdn_url_setting + public_key_cdn_url_setting.presence || DEFAULT_PUBLIC_KEY_CDN_URL end def public_key_cdn_url_setting diff --git a/lib/atlassian/jira_connect/serializers/build_entity.rb b/lib/atlassian/jira_connect/serializers/build_entity.rb index 10e4bb0e709..aa864cb268f 100644 --- a/lib/atlassian/jira_connect/serializers/build_entity.rb +++ b/lib/atlassian/jira_connect/serializers/build_entity.rb @@ -24,12 +24,10 @@ module Atlassian def issue_keys # extract Jira issue keys from either the source branch/ref or the # merge request title. - @issue_keys ||= begin - pipeline.all_merge_requests.flat_map do |mr| - src = "#{mr.source_branch} #{mr.title} #{mr.description}" - JiraIssueKeyExtractor.new(src).issue_keys - end.uniq - end + @issue_keys ||= pipeline.all_merge_requests.flat_map do |mr| + src = "#{mr.source_branch} #{mr.title} #{mr.description}" + JiraIssueKeyExtractor.new(src).issue_keys + end.uniq end private diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 55b10c008fb..9b019f16ddd 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -157,7 +157,7 @@ module Backup end def backup_files_realpath - @backup_files_realpath ||= File.join(Gitlab.config.backup.path, File.basename(@app_files_dir) ) + @backup_files_realpath ||= File.join(Gitlab.config.backup.path, File.basename(@app_files_dir)) end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index a8b3c12a2a2..f8424f6250e 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -218,7 +218,7 @@ module Backup build_backup_information - definitions.keys.each do |task_name| + definitions.each_key do |task_name| run_create_task(task_name) end @@ -239,7 +239,7 @@ module Backup read_backup_information verify_backup_version - definitions.keys.each do |task_name| + definitions.each_key do |task_name| if !skipped?(task_name) && enabled_task?(task_name) run_restore_task(task_name) end @@ -263,7 +263,7 @@ module Backup def write_backup_information # Make sure there is a connection - ::Gitlab::Database.database_base_models.values.each do |base_model| + ::Gitlab::Database.database_base_models.each_value do |base_model| base_model.connection.reconnect! end @@ -277,7 +277,7 @@ module Backup def build_backup_information @backup_information ||= { db_version: ActiveRecord::Migrator.current_version.to_s, - backup_created_at: Time.now, + backup_created_at: Time.current, gitlab_version: Gitlab::VERSION, tar_version: tar_version, installation_type: Gitlab::INSTALLATION_TYPE, @@ -291,7 +291,7 @@ module Backup @backup_information.merge!( full_backup_id: full_backup_id, db_version: ActiveRecord::Migrator.current_version.to_s, - backup_created_at: Time.zone.now, + backup_created_at: Time.current, gitlab_version: Gitlab::VERSION, tar_version: tar_version, installation_type: Gitlab::INSTALLATION_TYPE, @@ -396,7 +396,7 @@ module Backup timestamp = matched[1].to_i - next unless Time.at(timestamp) < (Time.now - keep_time) + next unless Time.zone.at(timestamp) < (Time.current - keep_time) begin FileUtils.rm(file) @@ -523,9 +523,7 @@ module Backup end def object_storage_config - @object_storage_config ||= begin - ObjectStorage::Config.new(Gitlab.config.backup.upload) - end + @object_storage_config ||= ObjectStorage::Config.new(Gitlab.config.backup.upload) end def connect_to_remote_directory @@ -613,7 +611,7 @@ module Backup end def puts_time(msg) - progress.puts "#{Time.now} -- #{msg}" + progress.puts "#{Time.current} -- #{msg}" Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") end end diff --git a/lib/banzai/filter/attributes_filter.rb b/lib/banzai/filter/attributes_filter.rb new file mode 100644 index 00000000000..ab50b3d6858 --- /dev/null +++ b/lib/banzai/filter/attributes_filter.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Looks for attributes that are specified for an element. Follows the basic syntax laid out + # in https://github.com/jgm/commonmark-hs/blob/master/commonmark-extensions/test/attributes.md + # For example, + # {width=50%} + # + # However we currently have the following limitations: + # - only support images + # - only support the `width` and `height` attributes + # - attributes can not span multiple lines + # - unsupported attributes are thrown away + class AttributesFilter < HTML::Pipeline::Filter + CSS = 'img' + XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze + + ATTRIBUTES_PATTERN = %r{\A(?<matched>\{(?<attributes>.{1,100})\})}.freeze + WIDTH_HEIGHT_REGEX = %r{\A(?<name>height|width)="?(?<size>[\w%]{1,10})"?\z}.freeze + VALID_SIZE_REGEX = %r{\A\d{1,4}(%|px)?\z}.freeze + + def call + doc.xpath(XPATH).each do |img| + sibling = img.next + next unless sibling && sibling.text? && sibling.content.first == '{' + + match = sibling.content.match(ATTRIBUTES_PATTERN) + next unless match && match[:attributes] + + match[:attributes].split(' ').each do |attribute| + next unless attribute.match?(WIDTH_HEIGHT_REGEX) + + attribute_match = attribute.match(WIDTH_HEIGHT_REGEX) + img[attribute_match[:name].to_sym] = attribute_match[:size] if valid_size?(attribute_match[:size]) + end + + sibling.content = sibling.content.sub(match[:matched], '') + end + + doc + end + + private + + def valid_size?(size) + size.match?(VALID_SIZE_REGEX) + end + end + end +end diff --git a/lib/banzai/filter/inline_observability_filter.rb b/lib/banzai/filter/inline_observability_filter.rb new file mode 100644 index 00000000000..27b89073a0e --- /dev/null +++ b/lib/banzai/filter/inline_observability_filter.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class InlineObservabilityFilter < ::Banzai::Filter::InlineEmbedsFilter + # Placeholder element for the frontend to use as an + # injection point for observability. + def create_element(url) + doc.document.create_element( + 'div', + class: 'js-render-observability', + 'data-frame-url': url + ) + end + + # Search params for selecting observability links. + def xpath_search + "descendant-or-self::a[starts-with(@href, '#{Gitlab::Observability.observability_url}')]" + end + + # Creates a new element based on the parameters + # obtained from the target link + def element_to_embed(node) + url = node['href'] + + create_element(url) + end + end + end +end diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index e95da735647..86beeae01b7 100644 --- a/lib/banzai/filter/repository_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -60,7 +60,7 @@ module Banzai def get_uri_types(paths) return {} if paths.empty? - uri_types = paths.to_h { |name| [name, nil] } + uri_types = paths.index_with { nil } get_blob_types(paths).each do |name, type| if type == :blob diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index fe189b1b0c9..de9b846efe9 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -10,6 +10,8 @@ module Banzai TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze def customize_allowlist(allowlist) + allowlist[:allow_comments] = context[:allow_comments] + # Allow table alignment; we allow specific text-align values in a # transformer below allowlist[:attributes]['th'] = %w[style] diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 766715d9e39..489b4d21300 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -9,7 +9,7 @@ module Banzai module Filter # HTML Filter to highlight fenced code blocks # - class SyntaxHighlightFilter < HTML::Pipeline::Filter + class SyntaxHighlightFilter < TimeoutHtmlPipelineFilter include OutputSafety LANG_PARAMS_DELIMITER = ':' @@ -19,7 +19,7 @@ module Banzai CSS = 'pre:not([data-kroki-style]) > code:only-child' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze - def call + def call_with_timeout doc.xpath(XPATH).each do |node| highlight_node(node) end diff --git a/lib/banzai/filter/timeout_html_pipeline_filter.rb b/lib/banzai/filter/timeout_html_pipeline_filter.rb new file mode 100644 index 00000000000..b9b71163ab1 --- /dev/null +++ b/lib/banzai/filter/timeout_html_pipeline_filter.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML Filter that wraps a filter in a Gitlab::RenderTimeout. + # This way partial results can be returned, and the entire pipeline + # is not killed. + # + # This should not be used for any filter that must be allowed to complete, + # like a `ReferenceRedactorFilter` + # + class TimeoutHtmlPipelineFilter < HTML::Pipeline::Filter + RENDER_TIMEOUT = 10.seconds + + def call + Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { call_with_timeout } + rescue Timeout::Error => e + class_name = self.class.name.demodulize + timeout_counter.increment(source: class_name) + Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name) + + # we've timed out, but some work may have already been completed, + # so go ahead and return the document + doc + end + + def call_with_timeout + raise NotImplementedError + end + + private + + def timeout_counter + Gitlab::Metrics.counter(:banzai_filter_timeouts_total, 'Count of the Banzai filters that time out') + end + end + end +end diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index afd5802de22..8764367426c 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -13,7 +13,7 @@ module Banzai Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, Filter::WikiLinkFilter, - Filter::SyntaxHighlightFilter, + Filter::SyntaxHighlightFilter, # this filter should remain next to last Filter::AsciiDocPostProcessingFilter ] end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 9b73e413d44..73065571849 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -22,6 +22,7 @@ module Banzai Filter::MermaidFilter, Filter::VideoLinkFilter, Filter::AudioLinkFilter, + Filter::AttributesFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, *metrics_filters, @@ -36,8 +37,9 @@ module Banzai Filter::CustomEmojiFilter, Filter::TaskListFilter, Filter::InlineDiffFilter, + Filter::InlineObservabilityFilter, Filter::SetDirectionFilter, - Filter::SyntaxHighlightFilter + Filter::SyntaxHighlightFilter # this filter should remain at the end ] end diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb index 330914f7238..635d4c0884e 100644 --- a/lib/banzai/pipeline/markup_pipeline.rb +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -10,7 +10,7 @@ module Banzai Filter::ExternalLinkFilter, Filter::PlantumlFilter, Filter::KrokiFilter, - Filter::SyntaxHighlightFilter + Filter::SyntaxHighlightFilter # this filter should remain at the end ] end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index caba9570ab9..96b6b9699f4 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -4,10 +4,8 @@ module Banzai module Pipeline class WikiPipeline < FullPipeline def self.filters - @filters ||= begin - super.insert_before(Filter::ImageLazyLoadFilter, Filter::GollumTagsFilter) + @filters ||= super.insert_before(Filter::ImageLazyLoadFilter, Filter::GollumTagsFilter) .insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter) - end end end end diff --git a/lib/banzai/reference_parser/alert_parser.rb b/lib/banzai/reference_parser/alert_parser.rb index 7b864d26f67..676c0ac40ef 100644 --- a/lib/banzai/reference_parser/alert_parser.rb +++ b/lib/banzai/reference_parser/alert_parser.rb @@ -5,14 +5,18 @@ module Banzai class AlertParser < BaseParser self.reference_type = :alert + def self.reference_class + AlertManagement::Alert + end + def references_relation AlertManagement::Alert end private - def can_read_reference?(user, alert, node) - can?(user, :read_alert_management_alert, alert) + def can_read_reference?(user, project, node) + can?(user, :read_alert_management_alert, project) end end end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 19d91876892..88ce63a63fe 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -47,6 +47,14 @@ module Banzai @data_attribute ||= "data-#{reference_type.to_s.dasherize}" end + # Returns a model class to use as a reference. + # By default, the method does not take namespaces into account, + # thus parser classes can customize the reference class to use + # a model name with a namespace + def self.reference_class + reference_type.to_s.classify.constantize + end + # context - An instance of `Banzai::RenderContext`. def initialize(context) @context = context diff --git a/lib/bitbucket_server/connection.rb b/lib/bitbucket_server/connection.rb index fbd451efb23..845acf034a5 100644 --- a/lib/bitbucket_server/connection.rb +++ b/lib/bitbucket_server/connection.rb @@ -15,6 +15,7 @@ module BitbucketServer Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout, + URI::InvalidURIError, Gitlab::HTTP::BlockedUrlError ].freeze diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 4c36f226006..8129ff6151c 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -55,20 +55,11 @@ module BulkImports end def instance_version - strong_memoize(:instance_version) do - response = begin - with_error_handling do - Gitlab::HTTP.get(resource_url(:version), default_options) - end - rescue BulkImports::NetworkError - # `version` endpoint is not available, try `metadata` endpoint instead - with_error_handling do - Gitlab::HTTP.get(resource_url(:metadata), default_options) - end - end + Gitlab::VersionInfo.parse(metadata['version']) + end - Gitlab::VersionInfo.parse(response.parsed_response['version']) - end + def instance_enterprise + Gitlab::Utils.to_boolean(metadata['enterprise'], default: true) end def compatible_for_project_migration? @@ -87,6 +78,22 @@ module BulkImports end end + def metadata + response = begin + with_error_handling do + Gitlab::HTTP.get(resource_url(:version), default_options) + end + rescue BulkImports::NetworkError + # `version` endpoint is not available, try `metadata` endpoint instead + with_error_handling do + Gitlab::HTTP.get(resource_url(:metadata), default_options) + end + end + + response.parsed_response + end + strong_memoize_attr :metadata + # rubocop:disable GitlabSecurity/PublicSend def request(method, resource, options = {}, &block) validate_instance_version! diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb index d7b9d6920ea..a1b338aeb9f 100644 --- a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -21,15 +21,16 @@ module BulkImports end def load(context, file_path) - avatar_path = AVATAR_PATTERN.match(file_path) + # Validate that the path is OK to load + Gitlab::Utils.check_allowed_absolute_path_and_path_traversal!(file_path, [Dir.tmpdir]) + return if File.directory?(file_path) + return if File.lstat(file_path).symlink? + avatar_path = AVATAR_PATTERN.match(file_path) return save_avatar(file_path) if avatar_path dynamic_path = file_uploader.extract_dynamic_path(file_path) - return unless dynamic_path - return if File.directory?(file_path) - return if File.lstat(file_path).symlink? named_captures = dynamic_path.named_captures.symbolize_keys diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 6928ce43191..0db2b1f0698 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -21,7 +21,7 @@ module BulkImports # instance version is 15.2.0, 15.2.1, 16.0.0, etc. def config - @config ||= { + { group: { pipeline: BulkImports::Groups::Pipelines::GroupPipeline, stage: 0 diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 54d5d3209f8..681d6e9aad6 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -13,6 +13,7 @@ module BulkImports CACHE_KEY_EXPIRATION = 2.hours NDJSON_EXPORT_TIMEOUT = 90.minutes + EMPTY_EXPORT_STATUS_TIMEOUT = 5.minutes def initialize(context) @context = context diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index 2fefdb9055e..73e102696fa 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -21,7 +21,7 @@ module BulkImports # instance version is 15.2.0, 15.2.1, 16.0.0, etc. def config - @config ||= { + { project: { pipeline: BulkImports::Projects::Pipelines::ProjectPipeline, stage: 0 diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index b45ac139385..5c622db3b01 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -19,6 +19,8 @@ module BulkImports private + attr_reader :bulk_import, :bulk_import_entity + def config # To be implemented in a sub-class NotImplementedError diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb index 9799116038e..f22996df0a5 100644 --- a/lib/extracts_ref.rb +++ b/lib/extracts_ref.rb @@ -64,10 +64,16 @@ module ExtractsRef def assign_ref_vars @id, @ref, @path = extract_ref_path @repo = repository_container.repository - raise InvalidPathError if @ref.match?(/\s/) - @commit = @repo.commit(@ref) if @ref.present? + return unless @ref.present? + + @commit = if ref_type && Feature.enabled?(:use_ref_type_parameter, @repo.project) + @fully_qualified_ref = %(refs/#{ref_type}/#{@ref}) + @repo.commit(@fully_qualified_ref) + else + @repo.commit(@ref) + end end # rubocop:enable Gitlab/ModuleWithInstanceVariables @@ -82,6 +88,12 @@ module ExtractsRef [id, ref, path] end + def ref_type + return unless params[:ref_type].present? + + params[:ref_type] == 'tags' ? 'tags' : 'heads' + end + private def extract_raw_ref(id) diff --git a/lib/feature.rb b/lib/feature.rb index 5841828da0e..d012639d489 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -16,6 +16,16 @@ module Feature end end + class OptOut + def initialize(inner) + @inner = inner + end + + def flipper_id + "#{@inner.flipper_id}:opt_out" + end + end + class FlipperGate < Flipper::Adapters::ActiveRecord::Gate superclass.table_name = 'feature_gates' end @@ -25,6 +35,7 @@ module Feature end InvalidFeatureFlagError = Class.new(Exception) # rubocop:disable Lint/InheritException + InvalidOperation = Class.new(ArgumentError) # rubocop:disable Lint/InheritException class << self delegate :group, to: :flipper @@ -78,7 +89,7 @@ module Feature # and should not be used outside of Gitaly's `lib/feature/gitaly.rb` def enabled?(key, thing = nil, type: :development, default_enabled_if_undefined: nil) if check_feature_flags_definition? - if thing && !thing.respond_to?(:flipper_id) + if thing && !thing.respond_to?(:flipper_id) && !thing.is_a?(Flipper::Types::Group) raise InvalidFeatureFlagError, "The thing '#{thing.class.name}' for feature flag '#{key}' needs to include `FeatureGate` or implement `flipper_id`" end @@ -87,10 +98,7 @@ module Feature end default_enabled = Feature::Definition.default_enabled?(key, default_enabled_if_undefined: default_enabled_if_undefined) - - feature_value = with_feature(key) do |feature| - feature_value = current_feature_value(feature, thing, default_enabled: default_enabled) - end + feature_value = current_feature_value(key, thing, default_enabled: default_enabled) # If not yielded, then either recursion is happening, or the database does not exist yet, so use default_enabled. feature_value = default_enabled if feature_value.nil? @@ -108,6 +116,7 @@ module Feature def enable(key, thing = true) log(key: key, action: __method__, thing: thing) + return_value = with_feature(key) { _1.enable(thing) } # rubocop:disable Gitlab/RailsLogger @@ -120,12 +129,45 @@ module Feature def disable(key, thing = false) log(key: key, action: __method__, thing: thing) + with_feature(key) { _1.disable(thing) } end + def opted_out?(key, thing) + return false unless thing.respond_to?(:flipper_id) # Ignore Feature::Types::Group + return false unless persisted_name?(key) + + opt_out = OptOut.new(thing) + + with_feature(key) { _1.actors_value.include?(opt_out.flipper_id) } + end + + def opt_out(key, thing) + return unless thing.respond_to?(:flipper_id) # Ignore Feature::Types::Group + + log(key: key, action: __method__, thing: thing) + opt_out = OptOut.new(thing) + + with_feature(key) { _1.enable(opt_out) } + end + + def remove_opt_out(key, thing) + return unless thing.respond_to?(:flipper_id) # Ignore Feature::Types::Group + return unless persisted_name?(key) + + log(key: key, action: __method__, thing: thing) + opt_out = OptOut.new(thing) + + with_feature(key) { _1.disable(opt_out) } + end + def enable_percentage_of_time(key, percentage) log(key: key, action: __method__, percentage: percentage) - with_feature(key) { _1.enable_percentage_of_time(percentage) } + with_feature(key) do |flag| + raise InvalidOperation, 'Cannot enable percentage of time for a fully-enabled flag' if flag.state == :on + + flag.enable_percentage_of_time(percentage) + end end def disable_percentage_of_time(key) @@ -135,7 +177,11 @@ module Feature def enable_percentage_of_actors(key, percentage) log(key: key, action: __method__, percentage: percentage) - with_feature(key) { _1.enable_percentage_of_actors(percentage) } + with_feature(key) do |flag| + raise InvalidOperation, 'Cannot enable percentage of actors for a fully-enabled flag' if flag.state == :on + + flag.enable_percentage_of_actors(percentage) + end end def disable_percentage_of_actors(key) @@ -147,6 +193,7 @@ module Feature return unless persisted_name?(key) log(key: key, action: __method__) + with_feature(key, &:remove) end @@ -189,14 +236,26 @@ module Feature private + # Compute if thing is enabled, taking opt-out overrides into account # Evaluate if `default enabled: false` or the feature has been persisted. # `persisted_name?` can potentially generate DB queries and also checks for inclusion # in an array of feature names (177 at last count), possibly reducing performance by half. # So we only perform the `persisted` check if `default_enabled: true` - def current_feature_value(feature, thing, default_enabled:) - return true if default_enabled && !Feature.persisted_name?(feature.name) - - feature.enabled?(thing) + def current_feature_value(key, thing, default_enabled:) + with_feature(key) do |feature| + if default_enabled && !Feature.persisted_name?(feature.name) + true + else + enabled = feature.enabled?(thing) + + if enabled && !thing.nil? + opt_out = OptOut.new(thing) + feature.actors_value.exclude?(opt_out.flipper_id) + else + enabled + end + end + end end # NOTE: it is not safe to call `Flipper::Feature#enabled?` outside the block @@ -292,7 +351,7 @@ module Feature end class Target - UnknowTargetError = Class.new(StandardError) + UnknownTargetError = Class.new(StandardError) attr_reader :params @@ -322,7 +381,7 @@ module Feature return unless params.key?(:user) params[:user].split(',').map do |arg| - UserFinder.new(arg).find_by_username || (raise UnknowTargetError, "#{arg} is not found!") + UserFinder.new(arg).find_by_username || (raise UnknownTargetError, "#{arg} is not found!") end end @@ -330,7 +389,7 @@ module Feature return unless params.key?(:project) params[:project].split(',').map do |arg| - Project.find_by_full_path(arg) || (raise UnknowTargetError, "#{arg} is not found!") + Project.find_by_full_path(arg) || (raise UnknownTargetError, "#{arg} is not found!") end end @@ -338,7 +397,7 @@ module Feature return unless params.key?(:group) params[:group].split(',').map do |arg| - Group.find_by_full_path(arg) || (raise UnknowTargetError, "#{arg} is not found!") + Group.find_by_full_path(arg) || (raise UnknownTargetError, "#{arg} is not found!") end end @@ -347,7 +406,7 @@ module Feature params[:namespace].split(',').map do |arg| # We are interested in Group or UserNamespace - Namespace.without_project_namespaces.find_by_full_path(arg) || (raise UnknowTargetError, "#{arg} is not found!") + Namespace.without_project_namespaces.find_by_full_path(arg) || (raise UnknownTargetError, "#{arg} is not found!") end end @@ -356,7 +415,7 @@ module Feature params[:repository].split(',').map do |arg| container, _project, _type, _path = Gitlab::RepoPath.parse(arg) - raise UnknowTargetError, "#{arg} is not found!" if container.nil? + raise UnknownTargetError, "#{arg} is not found!" if container.nil? container.repository end diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index 270bf46221d..2bad7cfd33d 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -7,6 +7,8 @@ module Feature attr_reader :path attr_reader :attributes + VALID_FEATURE_NAME = %r{^#{Gitlab::Regex.sep_by_1('_', /[a-z0-9]+/)}$}.freeze + PARAMS.each do |param| define_method(param) do attributes[param] @@ -38,6 +40,10 @@ module Feature raise Feature::InvalidFeatureFlagError, "Feature flag is missing name" end + unless VALID_FEATURE_NAME =~ name + raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' is invalid" + end + unless path.present? raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' is missing path" end diff --git a/lib/flowdock/git.rb b/lib/flowdock/git.rb deleted file mode 100644 index 897ee647d87..00000000000 --- a/lib/flowdock/git.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true -require 'flowdock' -require 'flowdock/git/builder' - -module Flowdock - class Git - TokenError = Class.new(StandardError) - - DEFAULT_PERMANENT_REFS = [ - Regexp.new('refs/heads/master') - ].freeze - - class << self - def post(ref, from, to, options = {}) - Git.new(ref, from, to, options).post - end - end - - def initialize(ref, from, to, options = {}) - raise TokenError, "Flowdock API token not found" unless options[:token] - - @ref = ref - @from = from - @to = to - @options = options - @token = options[:token] - @commit_url = options[:commit_url] - @diff_url = options[:diff_url] - @repo_url = options[:repo_url] - @repo_name = options[:repo_name] - @permanent_refs = options.fetch(:permanent_refs, DEFAULT_PERMANENT_REFS) - end - - # Send git push notification to Flowdock - def post - messages.each do |message| - ::Flowdock::Client.new(flow_token: @token).post_to_thread(message) - end - end - - def repo - @options[:repo] - end - - private - - def messages - Git::Builder.new(repo: repo, - ref: @ref, - before: @from, - after: @to, - commit_url: @commit_url, - branch_url: @branch_url, - diff_url: @diff_url, - repo_url: @repo_url, - repo_name: @repo_name, - permanent_refs: @permanent_refs, - tags: tags - ).to_hashes - end - - # Flowdock tags attached to the push notification - def tags - Array(@options[:tags]).map { |tag| CGI.escape(tag) } - end - end -end diff --git a/lib/flowdock/git/builder.rb b/lib/flowdock/git/builder.rb deleted file mode 100644 index 88d9814950a..00000000000 --- a/lib/flowdock/git/builder.rb +++ /dev/null @@ -1,145 +0,0 @@ -# frozen_string_literal: true -module Flowdock - class Git - class Commit - def initialize(external_thread_id, thread, tags, commit) - @commit = commit - @external_thread_id = external_thread_id - @thread = thread - @tags = tags - end - - def to_hash - hash = { - external_thread_id: @external_thread_id, - event: "activity", - author: { - name: @commit[:author][:name], - email: @commit[:author][:email] - }, - title: title, - thread: @thread, - body: body - } - hash[:tags] = @tags if @tags - encode(hash) - end - - private - - def encode(hash) - return hash unless "".respond_to?(:encode) - - encode_as_utf8(hash) - end - - # This only works on Ruby 1.9 - def encode_as_utf8(obj) - if obj.is_a? Hash - obj.each_pair do |key, val| - encode_as_utf8(val) - end - elsif obj.is_a?(Array) - obj.each do |val| - encode_as_utf8(val) - end - elsif obj.is_a?(String) && obj.encoding != Encoding::UTF_8 - unless obj.force_encoding("UTF-8").valid_encoding? - obj.force_encoding("ISO-8859-1").encode!(Encoding::UTF_8, invalid: :replace, undef: :replace) - end - end - end - - def body - content = @commit[:message][first_line.size..] - content.strip! if content - "<pre>#{content}</pre>" unless content.empty? - end - - def first_line - @first_line ||= (@commit[:message].split("\n")[0] || @commit[:message]) - end - - def title - commit_id = @commit[:id][0, 7] - if @commit[:url] - "<a href=\"#{@commit[:url]}\">#{commit_id}</a> #{message_title}" - else - "#{commit_id} #{message_title}" - end - end - - def message_title - CGI.escape_html(first_line.strip) - end - end - - # Class used to build Git payload - class Builder - include ::Gitlab::Utils::StrongMemoize - - def initialize(opts) - @repo = opts[:repo] - @ref = opts[:ref] - @before = opts[:before] - @after = opts[:after] - @opts = opts - end - - def commits - @repo.commits_between(@before, @after).map do |commit| - { - url: @opts[:commit_url] ? @opts[:commit_url] % [commit.sha] : nil, - id: commit.sha, - message: commit.message, - author: { - name: commit.author_name, - email: commit.author_email - } - } - end - end - - def ref_name - @ref.to_s.sub(%r{\Arefs/(heads|tags)/}, '') - end - - def to_hashes - commits.map do |commit| - Commit.new(external_thread_id, thread, @opts[:tags], commit).to_hash - end - end - - private - - def thread - @thread ||= { - title: thread_title, - external_url: @opts[:repo_url] - } - end - - def permanent? - strong_memoize(:permanent) do - @opts[:permanent_refs].any? { |regex| regex.match(@ref) } - end - end - - def thread_title - action = "updated" if permanent? - type = @ref =~ %r(^refs/heads/) ? "branch" : "tag" - - [@opts[:repo_name], type, ref_name, action].compact.join(" ") - end - - def external_thread_id - @external_thread_id ||= - if permanent? - SecureRandom.hex - else - @ref - end - end - end - end -end diff --git a/lib/gem_extensions/active_record/association.rb b/lib/gem_extensions/active_record/association.rb index c6634a0524a..0969b803ca4 100644 --- a/lib/gem_extensions/active_record/association.rb +++ b/lib/gem_extensions/active_record/association.rb @@ -23,13 +23,12 @@ module GemExtensions def association_scope if klass - @association_scope ||= begin # rubocop:disable Gitlab/ModuleWithInstanceVariables + @association_scope ||= # rubocop:disable Gitlab/ModuleWithInstanceVariables if disable_joins ::GemExtensions::ActiveRecord::DisableJoins::Associations::AssociationScope.scope(self) else super end - end end end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index d33120575a2..1190c92ce17 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -29,19 +29,19 @@ module Gitlab end def self.revision - @_revision ||= begin - if File.exist?(root.join("REVISION")) - File.read(root.join("REVISION")).strip.freeze - else - result = Gitlab::Popen.popen_with_detail(%W[#{config.git.bin_path} log --pretty=format:%h --abbrev=11 -n 1]) - - if result.status.success? - result.stdout.chomp.freeze - else - "Unknown" - end - end - end + @_revision ||= if File.exist?(root.join("REVISION")) + File.read(root.join("REVISION")).strip.freeze + else + result = Gitlab::Popen.popen_with_detail( + %W[#{config.git.bin_path} log --pretty=format:%h --abbrev=11 -n 1] + ) + + if result.status.success? + result.stdout.chomp.freeze + else + "Unknown" + end + end end APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}.freeze diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index b7a11bc0418..9ac433944a8 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -80,6 +80,10 @@ module Gitlab def self.internal_events INTERNAL_EVENTS end + + def self.selectable_events + (events - internal_events).sort_by(&:name) + end end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index b6ad25e700b..06ce1dbdc77 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -164,7 +164,11 @@ module Gitlab end def include_client? - set_values.include?(:user) || set_values.include?(:runner) || set_values.include?(:remote_ip) + # Don't overwrite an existing more specific client id with an `ip/` one. + original_client_id = self.class.current_context_attribute(:client_id).to_s + return false if original_client_id.starts_with?('user/') || original_client_id.starts_with?('runner/') + + include_user? || set_values.include?(:runner) || set_values.include?(:remote_ip) end def include_user? @@ -178,8 +182,8 @@ module Gitlab def client if runner "runner/#{runner.id}" - elsif user - "user/#{user.id}" + elsif user_id + "user/#{user_id}" else "ip/#{remote_ip}" end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 507f94d87a5..5b1bf99e297 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -51,8 +51,11 @@ module Gitlab project_testing_integration: { threshold: 5, interval: 1.minute }, email_verification: { threshold: 10, interval: 10.minutes }, email_verification_code_send: { threshold: 10, interval: 1.hour }, + phone_verification_send_code: { threshold: 10, interval: 1.hour }, + phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, namespace_exists: { threshold: 20, interval: 1.minute }, - fetch_google_ip_list: { threshold: 10, interval: 1.minute } + fetch_google_ip_list: { threshold: 10, interval: 1.minute }, + jobs_index: { threshold: 600, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index 4a6e4e2e06e..fddc1f830aa 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -59,7 +59,8 @@ module Gitlab @context = context @name = @context.fetch(:name, 'audit_operation') - @stream_only = @context.fetch(:stream_only, false) + @is_audit_event_yaml_defined = Gitlab::Audit::Type::Definition.defined?(@name) + @stream_only = stream_only? @author = @context.fetch(:author) @scope = @context.fetch(:scope) @target = @context.fetch(:target) @@ -70,6 +71,14 @@ module Gitlab @target_details = @context[:target_details] @authentication_event = @context.fetch(:authentication_event, false) @authentication_provider = @context[:authentication_provider] + + # TODO: Remove this code once we close https://gitlab.com/gitlab-org/gitlab/-/issues/367870 + return unless @is_audit_event_yaml_defined + + # rubocop:disable Gitlab/RailsLogger + Rails.logger.warn('WARNING: Logging audit events without an event type definition will be deprecated soon.') + Rails.logger.warn('See https://docs.gitlab.com/ee/development/audit_event_guide/#event-type-definitions') + # rubocop:enable Gitlab/RailsLogger end def single_audit @@ -84,14 +93,23 @@ module Gitlab end def record(events) - log_events(events) unless @stream_only - send_to_stream(events) + @stream_only ? send_to_stream(events) : log_events_and_stream(events) end - def log_events(events) + def log_events_and_stream(events) log_authentication_event - log_to_database(events) + saved_events = log_to_database(events) + + # we only want to override events with saved_events when it successfully saves into database. + # we are doing so to ensure events in memory reflects events saved in database and have id column. + events = saved_events if saved_events.present? + + log_to_file_and_stream(events) + end + + def log_to_file_and_stream(events) log_to_file(events) + send_to_stream(events) end def audit_enabled? @@ -102,6 +120,14 @@ module Gitlab @authentication_event end + def stream_only? + if @is_audit_event_yaml_defined + Gitlab::Audit::Type::Definition.stream_only?(@name) + else + @context.fetch(:stream_only, false) + end + end + def log_authentication_event return unless Gitlab::Database.read_write? && authentication_event? @@ -145,7 +171,13 @@ module Gitlab end def log_to_database(events) - AuditEvent.bulk_insert!(events) + if events.one? + events.first.save! + events + else + event_ids = AuditEvent.bulk_insert!(events, returns: :ids) + AuditEvent.id_in(event_ids) + end rescue ActiveRecord::RecordInvalid => e ::Gitlab::ErrorTracking.track_exception(e, audit_operation: @name) end diff --git a/lib/gitlab/audit/type/definition.rb b/lib/gitlab/audit/type/definition.rb index f64f66f4ca4..81c88a3a0ae 100644 --- a/lib/gitlab/audit/type/definition.rb +++ b/lib/gitlab/audit/type/definition.rb @@ -59,19 +59,36 @@ module Gitlab end class << self + include ::Gitlab::Utils::StrongMemoize + def paths @paths ||= [Rails.root.join('config', 'audit_events', 'types', '*.yml')] end def definitions - # We lazily load all definitions - @definitions ||= load_all! + load_all! end + strong_memoize_attr :definitions def get(key) definitions[key.to_sym] end + def event_names + definitions.keys.map(&:to_s) + end + + def defined?(key) + get(key).present? + end + + def stream_only?(key) + event_definition = get(key) + return false unless event_definition + + event_definition.streamed && !event_definition.saved_to_database + end + private def load_all! diff --git a/lib/gitlab/audit/type/shared.rb b/lib/gitlab/audit/type/shared.rb index 999b7de13e2..1e3e26d3735 100644 --- a/lib/gitlab/audit/type/shared.rb +++ b/lib/gitlab/audit/type/shared.rb @@ -14,7 +14,7 @@ module Gitlab description introduced_by_issue introduced_by_mr - group + feature_category milestone saved_to_database streamed diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c567df8e133..7e8f9c76dea 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -135,15 +135,13 @@ module Gitlab # it is important to reset the ban counter once the client has proven # they are not a 'bad guy'. rate_limiter.reset! - else + elsif rate_limiter.register_fail! # Register a login failure so that Rack::Attack can block the next # request from this IP if needed. # This returns true when the failures are over the threshold and the IP # is banned. - if rate_limiter.register_fail! - Gitlab::AppLogger.info "IP #{rate_limiter.ip} failed to login " \ + Gitlab::AppLogger.info "IP #{rate_limiter.ip} failed to login " \ "as #{login} but has been temporarily banned from Git auth" - end end end diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb index fc391543f4d..9bd4711c4bb 100644 --- a/lib/gitlab/auth/current_user_mode.rb +++ b/lib/gitlab/auth/current_user_mode.rb @@ -106,8 +106,8 @@ module Gitlab end def enable_admin_mode!(password: nil, skip_password_validation: false) - return unless user&.admin? - return unless skip_password_validation || user&.valid_password?(password) + return false unless user&.admin? + return false unless skip_password_validation || user&.valid_password?(password) raise NotRequestedError unless admin_mode_requested? @@ -115,6 +115,10 @@ module Gitlab current_session_data[ADMIN_MODE_REQUESTED_TIME_KEY] = nil current_session_data[ADMIN_MODE_START_TIME_KEY] = Time.now + + audit_user_enable_admin_mode + + true end def disable_admin_mode! @@ -175,6 +179,10 @@ module Gitlab def privileged_runtime? Gitlab::Runtime.rake? || Gitlab::Runtime.rails_runner? || Gitlab::Runtime.console? end + + def audit_user_enable_admin_mode; end end end end + +Gitlab::Auth::CurrentUserMode.prepend_mod_with('Gitlab::Auth::CurrentUserMode') diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 62a817d7c4d..ea098ff8057 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -12,7 +12,7 @@ module Gitlab def self.open(user, &block) Gitlab::Auth::Ldap::Adapter.open(user.ldap_identity.provider) do |adapter| - block.call(self.new(user, adapter)) + yield(self.new(user, adapter)) end end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 47eca74aa5b..9aedc131e92 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -11,7 +11,7 @@ module Gitlab def self.open(provider, &block) Net::LDAP.open(config(provider).adapter_options) do |ldap| - block.call(self.new(provider, ldap)) + yield(self.new(provider, ldap)) end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 9dafd59561a..6c99b505797 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -82,7 +82,8 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption_options + encryption: encryption_options, + instrumentation_service: ActiveSupport::Notifications ) opts.merge!(auth_options) if has_auth? diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb index a188aa168c1..84bf455c98a 100644 --- a/lib/gitlab/auth/ldap/dn.rb +++ b/lib/gitlab/auth/ldap/dn.rb @@ -51,7 +51,7 @@ module Gitlab ## # Parse a DN into key value pairs using ASN from - # http://tools.ietf.org/html/rfc2253 section 3. + # https://www.rfc-editor.org/rfc/rfc2253 section 3. # rubocop:disable Metrics/AbcSize # rubocop:disable Metrics/CyclomaticComplexity # rubocop:disable Metrics/PerceivedComplexity @@ -231,7 +231,7 @@ module Gitlab self.class.new(*to_a).to_s.downcase end - # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # https://www.rfc-editor.org/rfc/rfc4514 section 2.4 lists these exceptions # for DN values. All of the following must be escaped in any normal string # using a single backslash ('\') as escape. The space character is left # out here because in a "normalized" string, spaces should only be escaped diff --git a/lib/gitlab/background_migration/backfill_environment_tiers.rb b/lib/gitlab/background_migration/backfill_environment_tiers.rb new file mode 100644 index 00000000000..6f381577274 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_environment_tiers.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class backfills the `environments.tier` column by using `guess_tier` logic. + # Environments created after 13.10 already have a value, however, environments created before 13.10 don't. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/300741 for more information. + class BackfillEnvironmentTiers < BatchedMigrationJob + operation_name :backfill_environment_tiers + + # Equivalent to `Environment#guess_tier` pattern matching. + PRODUCTION_TIER = 0 + STAGING_TIER = 1 + TESTING_TIER = 2 + DEVELOPMENT_TIER = 3 + OTHER_TIER = 4 + + TIER_REGEXP_PAIR = [ + { tier: DEVELOPMENT_TIER, regexp: '(dev|review|trunk)' }, + { tier: TESTING_TIER, regexp: '(test|tst|int|ac(ce|)pt|qa|qc|control|quality)' }, + { tier: STAGING_TIER, regexp: '(st(a|)g|mod(e|)l|pre|demo|non)' }, + { tier: PRODUCTION_TIER, regexp: '(pr(o|)d|live)' } + ].freeze + + def perform + TIER_REGEXP_PAIR.each do |pair| + each_sub_batch( + batching_scope: ->(relation) { relation.where(tier: nil).where("name ~* '#{pair[:regexp]}'") } # rubocop:disable GitlabSecurity/SqlInjection + ) do |sub_batch| + sub_batch.update_all(tier: pair[:tier]) + end + end + + each_sub_batch(batching_scope: ->(relation) { relation.where(tier: nil) }) do |sub_batch| + sub_batch.update_all(tier: OTHER_TIER) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_note_discussion_id.rb b/lib/gitlab/background_migration/backfill_note_discussion_id.rb index da2c31ebd11..ce2698b3cb8 100644 --- a/lib/gitlab/background_migration/backfill_note_discussion_id.rb +++ b/lib/gitlab/background_migration/backfill_note_discussion_id.rb @@ -33,8 +33,8 @@ module Gitlab private def update_discussion_ids(notes) - mapping = notes.each_with_object({}) do |note, hash| - hash[note] = { discussion_id: note.generate_discussion_id } + mapping = notes.index_with do |note| + { discussion_id: note.generate_discussion_id } end Gitlab::Database::BulkUpdate.execute(%i(discussion_id), mapping) diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb new file mode 100644 index 00000000000..1a3dd88ea31 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill storage_size for project_statistics + class BackfillProjectStatisticsStorageSizeWithoutUploadsSize < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + # no-op + end + end + end +end + +Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutUploadsSize.prepend_mod_with('Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutUploadsSize') # rubocop:disable Layout/LineLength diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 64401bc0674..973ab20f547 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -9,55 +9,68 @@ module Gitlab # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#job-arguments. class BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers + include Gitlab::ClassAttributes - def initialize( - start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: - ) + DEFAULT_FEATURE_CATEGORY = :database - @start_id = start_id - @end_id = end_id - @batch_table = batch_table - @batch_column = batch_column - @sub_batch_size = sub_batch_size - @pause_ms = pause_ms - @job_arguments = job_arguments - @connection = connection - end + class << self + def generic_instance(batch_table:, batch_column:, job_arguments: [], connection:) + new( + batch_table: batch_table, batch_column: batch_column, + job_arguments: job_arguments, connection: connection, + start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0 + ) + end - def self.generic_instance(batch_table:, batch_column:, job_arguments: [], connection:) - new( - batch_table: batch_table, batch_column: batch_column, - job_arguments: job_arguments, connection: connection, - start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0 - ) - end + def job_arguments_count + 0 + end - def self.job_arguments_count - 0 - end + def operation_name(operation) + define_method('operation_name') do + operation + end + end - def self.operation_name(operation) - define_method('operation_name') do - operation + def job_arguments(*args) + args.each.with_index do |arg, index| + define_method(arg) do + @job_arguments[index] + end + end + + define_singleton_method(:job_arguments_count) do + args.count + end end - end - def self.job_arguments(*args) - args.each.with_index do |arg, index| - define_method(arg) do - @job_arguments[index] + def scope_to(scope) + define_method(:filter_batch) do |relation| + instance_exec(relation, &scope) end end - define_singleton_method(:job_arguments_count) do - args.count + def feature_category(feature_category_name = nil) + if feature_category_name.present? + set_class_attribute(:feature_category, feature_category_name) + else + get_class_attribute(:feature_category) || DEFAULT_FEATURE_CATEGORY + end end end - def self.scope_to(scope) - define_method(:filter_batch) do |relation| - instance_exec(relation, &scope) - end + def initialize( + start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: + ) + + @start_id = start_id + @end_id = end_id + @batch_table = batch_table + @batch_column = batch_column + @sub_batch_size = sub_batch_size + @pause_ms = pause_ms + @job_arguments = job_arguments + @connection = connection end def filter_batch(relation) diff --git a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb new file mode 100644 index 00000000000..4b7b7d42c77 --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphans records whenever report_type equals to scan_finding (i.e., 4) + class DeleteOrphansApprovalMergeRequestRules < BatchedMigrationJob + scope_to ->(relation) { relation.where(report_type: 4) } + + operation_name :delete_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(security_orchestration_policy_configuration_id: nil).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb new file mode 100644 index 00000000000..33aa1a8d29d --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphans records whenever report_type equals to scan_finding (i.e., 4) + class DeleteOrphansApprovalProjectRules < BatchedMigrationJob + operation_name :delete_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(report_type: 4, security_orchestration_policy_configuration_id: nil).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb new file mode 100644 index 00000000000..dcef4f086e2 --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for projects less than 5 MB + class DisableLegacyOpenSourceLicenseForProjectsLessThanFiveMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) do + relation + .where(legacy_open_source_license_available: true) + end + + operation_name :disable_legacy_open_source_license_for_projects_less_than_five_mb + + def perform + each_sub_batch do |sub_batch| + updates = { legacy_open_source_license_available: false, updated_at: Time.current } + + sub_batch + .joins('INNER JOIN project_statistics ON project_statistics.project_id = project_settings.project_id') + .where('project_statistics.repository_size < ?', 5.megabyte) + .update_all(updates) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb new file mode 100644 index 00000000000..4b283bae79d --- /dev/null +++ b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't update approval project rules + # as this feature exists only in EE + class FixApprovalProjectRulesWithoutProtectedBranches < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::FixApprovalProjectRulesWithoutProtectedBranches.prepend_mod_with('Gitlab::BackgroundMigration::FixApprovalProjectRulesWithoutProtectedBranches') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/fix_security_scan_statuses.rb b/lib/gitlab/background_migration/fix_security_scan_statuses.rb new file mode 100644 index 00000000000..b60e739f870 --- /dev/null +++ b/lib/gitlab/background_migration/fix_security_scan_statuses.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fixes the `status` attribute of `security_scans` records + class FixSecurityScanStatuses < BatchedMigrationJob + def perform + # no-op. The logic is defined in EE module. + end + end + end +end + +::Gitlab::BackgroundMigration::FixSecurityScanStatuses.prepend_mod diff --git a/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb new file mode 100644 index 00000000000..81b29b5a6cd --- /dev/null +++ b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition < BatchedMigrationJob + def perform; end + end + end +end + +Gitlab::BackgroundMigration::MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition.prepend_mod +# rubocop:enable Style/Documentation diff --git a/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb new file mode 100644 index 00000000000..a91cda2c427 --- /dev/null +++ b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting stale project import jobs + class PruneStaleProjectExportJobs < BatchedMigrationJob + EXPIRES_IN = 7.days + + scope_to ->(relation) { relation.where("updated_at < ?", EXPIRES_IN.ago) } + operation_name :delete_all + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb new file mode 100644 index 00000000000..09cd3b1895f --- /dev/null +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job that: + # * pickup container repositories with delete_scheduled status. + # * check if there are tags linked to it. + # * if there are tags, reset the status to nil. + class ResetStatusOnContainerRepositories < BatchedMigrationJob + DELETE_SCHEDULED_STATUS = 0 + DUMMY_TAGS = %w[tag].freeze + MIGRATOR = 'ResetStatusOnContainerRepositories' + + scope_to ->(relation) { relation.where(status: DELETE_SCHEDULED_STATUS) } + operation_name :reset_status_on_container_repositories + + def perform + each_sub_batch do |sub_batch| + reset_status_if_tags(sub_batch) + end + end + + private + + def reset_status_if_tags(container_repositories) + container_repositories_with_tags = container_repositories.select { |cr| cr.becomes(ContainerRepository).tags? } # rubocop:disable Cop/AvoidBecomes + + ContainerRepository.where(id: container_repositories_with_tags.map(&:id)) + .update_all(status: nil) + end + + # rubocop:disable Style/Documentation + module Routable + extend ActiveSupport::Concern + + included do + has_one :route, + as: :source, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route' + end + + def full_path + route&.path || build_full_path + end + + def build_full_path + if parent && path + "#{parent.full_path}/#{path}" + else + path + end + end + end + + class Route < ::ApplicationRecord + self.table_name = 'routes' + end + + class Namespace < ::ApplicationRecord + include ::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Routable + include ::Namespaces::Traversal::Recursive + include ::Namespaces::Traversal::Linear + include ::Gitlab::Utils::StrongMemoize + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + belongs_to :parent, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + + def self.polymorphic_name + 'Namespace' + end + end + + class Project < ::ApplicationRecord + include ::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Routable + + self.table_name = 'projects' + + belongs_to :namespace, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + + alias_method :parent, :namespace + alias_attribute :parent_id, :namespace_id + + delegate :root_ancestor, to: :namespace, allow_nil: true + end + + class ContainerRepository < ::ApplicationRecord + self.table_name = 'container_repositories' + + belongs_to :project, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project' + + def tags? + result = ContainerRegistry.tags_for(path).any? + ::Gitlab::BackgroundMigration::Logger.info( + migrator: MIGRATOR, + has_tags: result, + container_repository_id: id, + container_repository_path: path + ) + result + end + + def path + @path ||= [project.full_path, name].select(&:present?).join('/').downcase + end + end + + class ContainerRegistry + class << self + def tags_for(path) + response = ContainerRegistryClient.repository_tags(path, page_size: 1) + return DUMMY_TAGS unless response + + response['tags'] || [] + rescue StandardError + DUMMY_TAGS + end + end + end + + class ContainerRegistryClient + def self.repository_tags(path, page_size:) + registry_config = ::Gitlab.config.registry + + return { 'tags' => DUMMY_TAGS } unless registry_config.enabled && registry_config.api_url.present? + + pull_token = ::Auth::ContainerRegistryAuthenticationService.pull_access_token(path) + client = ::ContainerRegistry::Client.new(registry_config.api_url, token: pull_token) + client.repository_tags(path, page_size: page_size) + end + end + # rubocop:enable Style/Documentation + end + end +end diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb index e9a38916999..8aab7d13b45 100644 --- a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -37,16 +37,16 @@ module Gitlab end end - cloud_mappings = cloud.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 2 } + cloud_mappings = cloud.index_with do + { deployment_type: 2 } end - server_mappings = server.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 1 } + server_mappings = server.index_with do + { deployment_type: 1 } end - unknown_mappings = unknown.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 0 } + unknown_mappings = unknown.index_with do + { deployment_type: 0 } end mappings = cloud_mappings.merge(server_mappings, unknown_mappings) diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 7de6be45349..49b8ab760f3 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -98,7 +98,7 @@ module Gitlab create_labels - issue_type_id = WorkItems::Type.default_issue_type.id + issue_type_id = ::WorkItems::Type.default_issue_type.id client.issues(repo).each do |issue| import_issue(issue, issue_type_id) diff --git a/lib/gitlab/bullet.rb b/lib/gitlab/bullet.rb index f5f8a316855..9759a82be0c 100644 --- a/lib/gitlab/bullet.rb +++ b/lib/gitlab/bullet.rb @@ -10,7 +10,7 @@ module Gitlab alias_method :extra_logging_enabled?, :enabled? def configure_bullet? - defined?(::Bullet) && (enabled? || Rails.env.development?) + defined?(::Bullet) && (enabled? || Gitlab.config.bullet.enabled) end end end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index fb75a78a978..999d2ee4356 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -15,14 +15,12 @@ module Gitlab end def changes - @changes ||= begin - @raw_changes.map do |change| - next if change.blank? + @changes ||= @raw_changes.map do |change| + next if change.blank? - oldrev, newrev, ref = change.strip.split(' ') - { oldrev: oldrev, newrev: newrev, ref: ref } - end.compact - end + oldrev, newrev, ref = change.strip.split(' ') + { oldrev: oldrev, newrev: newrev, ref: ref } + end.compact end end end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 19819ff7275..42a8b561d34 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -448,8 +448,8 @@ module Gitlab end def state - state = STATE_PARAMS.each_with_object({}) do |param, h| - h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend + state = STATE_PARAMS.index_with do |param| + send(param) # rubocop:disable GitlabSecurity/PublicSend end Base64.urlsafe_encode64(state.to_json) end diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb index 375e6b4a96f..1cddc9fcc98 100644 --- a/lib/gitlab/ci/build/cache.rb +++ b/lib/gitlab/ci/build/cache.rb @@ -8,9 +8,9 @@ module Gitlab def initialize(cache, pipeline) cache = Array.wrap(cache) - @cache = cache.map do |cache| + @cache = cache.map.with_index do |cache, index| Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache) + .new(pipeline, cache, index) end end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index a1a8e9288c7..1025e1cc2d7 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -9,25 +9,29 @@ module Gitlab attr_reader :attributes - def initialize(pipeline, attributes = {}) + def initialize(pipeline, attributes = {}, build = nil) super(pipeline) + @build = build @attributes = attributes end def variables - strong_memoize(:variables) do - # This is a temporary piece of technical debt to allow us access - # to the CI variables to evaluate rules before we persist a Build - # with the result. We should refactor away the extra Build.new, - # but be able to get CI Variables directly from the Seed::Build. - stub_build.scoped_variables - end + build.scoped_variables end + strong_memoize_attr :variables private + def build + @build || stub_build + end + def stub_build + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate rules before we persist a Build + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. ::Ci::Build.new(build_attributes) end diff --git a/lib/gitlab/ci/build/hook.rb b/lib/gitlab/ci/build/hook.rb new file mode 100644 index 00000000000..b731228678c --- /dev/null +++ b/lib/gitlab/ci/build/hook.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Hook + attr_reader :name, :script + + class << self + def from_hooks(job) + job.options[:hooks].to_a.map do |name, script| + new(name.to_s, script) + end + end + end + + def initialize(name, script) + @name = name + @script = script + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ee537f4efe5..142f0b8dfd8 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -10,7 +10,7 @@ module Gitlab ConfigError = Class.new(StandardError) TIMEOUT_SECONDS = 30.seconds - TIMEOUT_MESSAGE = 'Resolving config took longer than expected' + TIMEOUT_MESSAGE = 'Request timed out when fetching configuration files.' RESCUE_ERRORS = [ Gitlab::Config::Loader::FormatError, @@ -26,7 +26,7 @@ module Gitlab @source_ref_path = pipeline&.source_ref_path @project = project - @context = self.logger.instrument(:config_build_context) do + @context = self.logger.instrument(:config_build_context, once: true) do pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) end @@ -35,12 +35,16 @@ module Gitlab @source = source - @config = self.logger.instrument(:config_expand) do + @config = self.logger.instrument(:config_expand, once: true) do expand_config(config) end - @root = self.logger.instrument(:config_compose) do - Entry::Root.new(@config, project: project, user: user).tap(&:compose!) + @root = self.logger.instrument(:config_root, once: true) do + Entry::Root.new(@config, project: project, user: user, logger: self.logger) + end + + self.logger.instrument(:config_root_compose, once: true) do + @root.compose! end rescue *rescue_errors => e raise Config::ConfigError, e.message @@ -123,23 +127,23 @@ module Gitlab end def build_config(config) - initial_config = logger.instrument(:config_yaml_load) do + initial_config = logger.instrument(:config_yaml_load, once: true) do Config::Yaml.load!(config) end - initial_config = logger.instrument(:config_external_process) do + initial_config = logger.instrument(:config_external_process, once: true) do Config::External::Processor.new(initial_config, @context).perform end - initial_config = logger.instrument(:config_yaml_extend) do + initial_config = logger.instrument(:config_yaml_extend, once: true) do Config::Extendable.new(initial_config).to_hash end - initial_config = logger.instrument(:config_tags_resolve) do + initial_config = logger.instrument(:config_tags_resolve, once: true) do Config::Yaml::Tags::Resolver.new(initial_config).to_hash end - logger.instrument(:config_stages_inject) do + logger.instrument(:config_stages_inject, once: true) do Config::EdgeStagesInjector.new(initial_config).to_hash end end @@ -163,7 +167,7 @@ module Gitlab end def build_variables(pipeline:) - logger.instrument(:config_build_variables) do + logger.instrument(:config_build_variables, once: true) do pipeline .variables_builder .config_variables diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 3b0cbc6b69e..27206d7e3a8 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,6 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable + ALLOWED_WHEN = %w[on_success on_failure always].freeze ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude public].freeze EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" @@ -38,10 +39,10 @@ module Gitlab validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present? validates :exclude, array_of_strings: true validates :reports, type: Hash - validates :when, - inclusion: { in: %w[on_success on_failure always], - message: 'should be on_success, on_failure ' \ - 'or always' } + validates :when, type: String, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser } end end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index ab79add688b..a5481071fc5 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -17,16 +17,16 @@ module Gitlab validations do validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, - inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, - allow_blank: true + validates :policy, type: String, allow_blank: true, inclusion: { + in: ALLOWED_POLICY, + message: "should be one of: #{ALLOWED_POLICY.join(', ')}" + } with_options allow_nil: true do - validates :when, - inclusion: { - in: ALLOWED_WHEN, - message: 'should be on_success, on_failure or always' - } + validates :when, type: String, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } end end diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index 12d68b755b3..e996b6b1312 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -13,9 +13,8 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Inheritable - ALLOWED_KEYS = %i[before_script image services - after_script cache interruptible - timeout retry tags artifacts].freeze + ALLOWED_KEYS = %i[before_script after_script hooks cache image services + interruptible timeout retry tags artifacts].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -25,22 +24,27 @@ module Gitlab description: 'Script that will be executed before each job.', inherit: true - entry :image, Entry::Image, - description: 'Docker image that will be used to execute jobs.', - inherit: true - - entry :services, Entry::Services, - description: 'Docker images that will be linked to the container.', - inherit: true - entry :after_script, Entry::Commands, description: 'Script that will be executed after each job.', inherit: true + entry :hooks, Entry::Hooks, + description: 'Commands that will be executed on Runner before/after some events ' \ + 'such as `clone` and `build-script`.', + inherit: false + entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', inherit: true + entry :image, Entry::Image, + description: 'Docker image that will be used to execute jobs.', + inherit: true + + entry :services, Entry::Services, + description: 'Docker images that will be linked to the container.', + inherit: true + entry :interruptible, ::Gitlab::Config::Entry::Boolean, description: 'Set jobs interruptible default value.', inherit: false diff --git a/lib/gitlab/ci/config/entry/hooks.rb b/lib/gitlab/ci/config/entry/hooks.rb new file mode 100644 index 00000000000..28bc2e4e7ce --- /dev/null +++ b/lib/gitlab/ci/config/entry/hooks.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Hooks < ::Gitlab::Config::Entry::Node + # `Configurable` alreadys adds `Validatable` + include ::Gitlab::Config::Entry::Configurable + + # NOTE: If a new hook is added, inheriting should be changed because a `job:hooks` overrides all + # `default:hooks` now. We should implement merging; each hook must be overridden individually. + ALLOWED_HOOKS = %i[pre_get_sources_script].freeze + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_HOOKS + end + + entry :pre_get_sources_script, Entry::Commands, + description: 'Commands that will be executed on Runner before cloning/fetching the Git repository.' + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/id_token.rb b/lib/gitlab/ci/config/entry/id_token.rb new file mode 100644 index 00000000000..12e0975d1b1 --- /dev/null +++ b/lib/gitlab/ci/config/entry/id_token.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a JWT definition. + # + class IdToken < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Validatable + + attributes %i[aud] + + validations do + validates :config, required_keys: %i[aud], allowed_keys: %i[aud] + validates :aud, array_of_strings_or_string: true + end + + def value + { aud: aud } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 8e7f6ba4326..7c49b59a7f0 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -12,9 +12,9 @@ module Gitlab ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script image services start_in artifacts - cache dependencies before_script after_script + cache dependencies before_script after_script hooks environment coverage retry parallel interruptible timeout - release].freeze + release id_tokens].freeze validations do validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS @@ -59,6 +59,10 @@ module Gitlab description: 'Commands that will be executed when finishing job.', inherit: true + entry :hooks, Entry::Hooks, + description: 'Commands that will be executed on Runner before/after some events; clone, build-script.', + inherit: true + entry :cache, Entry::Caches, description: 'Cache definition for this job.', inherit: true @@ -116,6 +120,11 @@ module Gitlab description: 'Indicates whether this job is allowed to fail or not.', inherit: false + entry :id_tokens, ::Gitlab::Config::Entry::ComposableHash, + description: 'Configured JWTs for this job', + inherit: false, + metadata: { composable_class: ::Gitlab::Ci::Config::Entry::IdToken } + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, @@ -155,10 +164,12 @@ module Gitlab artifacts: artifacts_value, release: release_value, after_script: after_script_value, + hooks: hooks_pre_get_sources_script_enabled? ? hooks_value : nil, ignore: ignored?, allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, - scheduling_type: needs_defined? ? :dag : :stage + scheduling_type: needs_defined? ? :dag : :stage, + id_tokens: id_tokens_value ).compact end @@ -183,6 +194,10 @@ module Gitlab allow_failure_value end + + def hooks_pre_get_sources_script_enabled? + YamlProcessor::FeatureFlags.enabled?(:ci_hooks_pre_get_sources_script) + end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index f77876cc926..16844fa88db 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -16,8 +16,8 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif dotenv terraform accessibility - requirements coverage_fuzzing api_fuzzing cluster_image_scanning - coverage_report cyclonedx].freeze + coverage_fuzzing api_fuzzing cluster_image_scanning + requirements requirements_v2 coverage_report cyclonedx].freeze attributes ALLOWED_KEYS @@ -48,6 +48,7 @@ module Gitlab validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true validates :requirements, array_of_strings_or_string: true + validates :requirements_v2, array_of_strings_or_string: true validates :cyclonedx, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index a30e6a0d9c3..a3d57ab6ac6 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -50,7 +50,7 @@ module Gitlab entry :variables, Entry::Variables, description: 'Environment variables that will be used.', - metadata: { allowed_value_data: %i[value description expand], allow_array_value: true }, + metadata: { allowed_value_data: %i[value description expand options] }, reserved: true entry :stages, Entry::Stages, @@ -103,12 +103,16 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def compose_jobs! - factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs) - .value(jobs_config) - .with(key: :jobs, parent: self, - description: 'Jobs definition for this pipeline') + factory = logger.instrument(:config_root_compose_jobs_factory, once: true) do + ::Gitlab::Config::Entry::Factory.new(Entry::Jobs) + .value(jobs_config) + .with(key: :jobs, parent: self, + description: 'Jobs definition for this pipeline') + end - @entries[:jobs] = factory.create! + @entries[:jobs] = logger.instrument(:config_root_compose_jobs_create, once: true) do + factory.create! + end end # rubocop: enable CodeReuse/ActiveRecord @@ -123,6 +127,10 @@ module Gitlab @config = @config.except(*@jobs_config.keys) end + + def logger + metadata[:logger] + end end end end diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index 0f94b3f94fe..4c254a4fa07 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -41,7 +41,7 @@ module Gitlab validations do validates :config, presence: true validates :config, allowed_keys: ALLOWED_KEYS - validates :project, presence: true + validates :project, type: String, presence: true validates :branch, type: String, allow_nil: true validates :strategy, type: String, inclusion: { in: %w[depend], message: 'should be depend' }, allow_nil: true end diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index 16091758916..decb568ffc9 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -10,7 +10,6 @@ module Gitlab class Variable < ::Gitlab::Config::Entry::Simplifiable strategy :SimpleVariable, if: -> (config) { SimpleVariable.applies_to?(config) } strategy :ComplexVariable, if: -> (config) { ComplexVariable.applies_to?(config) } - strategy :ComplexArrayVariable, if: -> (config) { ComplexArrayVariable.applies_to?(config) } class SimpleVariable < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -41,20 +40,24 @@ module Gitlab class ComplexVariable < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable class << self def applies_to?(config) - config.is_a?(Hash) && !config[:value].is_a?(Array) + config.is_a?(Hash) end end + attributes :value, :description, :expand, :options, prefix: :config + validations do validates :key, alphanumeric: true - validates :config_value, alphanumeric: true, allow_nil: false, if: :config_value_defined? - validates :config_description, alphanumeric: true, allow_nil: false, if: :config_description_defined? - validates :config_expand, boolean: true, - allow_nil: false, - if: -> { ci_raw_variables_in_yaml_config_enabled? && config_expand_defined? } + validates :config_value, alphanumeric: true, allow_nil: true + validates :config_description, alphanumeric: true, allow_nil: true + validates :config_expand, boolean: true, allow_nil: true, if: -> { + ci_raw_variables_in_yaml_config_enabled? + } + validates :config_options, array_of_strings: true, allow_nil: true validate do allowed_value_data = Array(opt(:allowed_value_data)) @@ -66,91 +69,43 @@ module Gitlab else errors.add(:config, "must be a string") end + + if config_options.present? && config_options.exclude?(config_value) + errors.add(:config, 'value must be present in options') + end end end def value + # Needed since the `Entry::Node` provides `value` (which is current hash) config_value.to_s end def value_with_data if ci_raw_variables_in_yaml_config_enabled? { - value: value, - raw: (!config_expand if config_expand_defined?) + value: config_value.to_s, + raw: (!config_expand if has_config_expand?) }.compact else { - value: value + value: config_value.to_s }.compact end end def value_with_prefill_data value_with_data.merge( - description: config_description + description: config_description, + options: config_options ).compact end - def config_value - @config[:value] - end - - def config_description - @config[:description] - end - - def config_expand - @config[:expand] - end - - def config_value_defined? - config.key?(:value) - end - - def config_description_defined? - config.key?(:description) - end - - def config_expand_defined? - config.key?(:expand) - end - def ci_raw_variables_in_yaml_config_enabled? YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) end end - class ComplexArrayVariable < ComplexVariable - include ::Gitlab::Config::Entry::Validatable - - class << self - def applies_to?(config) - config.is_a?(Hash) && config[:value].is_a?(Array) - end - end - - validations do - validates :config_value, array_of_strings: true, allow_nil: false, if: :config_value_defined? - - validate do - next if opt(:allow_array_value) - - errors.add(:config, 'value must be an alphanumeric string') - end - end - - def value - config_value.first - end - - def value_with_prefill_data - super.merge( - value_options: config_value - ).compact - end - end - class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["variable definition must be either a string or a hash"] diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index ef4f74b9f56..e338bce3109 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -42,7 +42,7 @@ module Gitlab end def composable_metadata - { allowed_value_data: opt(:allowed_value_data), allow_array_value: opt(:allow_array_value) } + { allowed_value_data: opt(:allowed_value_data) } end end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 57ff606c9ee..65caf4ac47d 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,7 +47,7 @@ module Gitlab end def validate! - validate_execution_time! + context.check_execution_time! if ::Feature.disabled?(:ci_refactoring_external_mapper, context.project) validate_location! validate_context! if valid? fetch_and_validate_content! if valid? @@ -87,10 +87,6 @@ module Gitlab nil end - def validate_execution_time! - context.check_execution_time! - end - def validate_location! if invalid_location_type? errors.push("Included file `#{masked_location}` needs to be a string") diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index b0c540685d4..ed37357dc53 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -53,7 +53,7 @@ module Gitlab errors.push("Remote file `#{masked_location}` could not be fetched because of a timeout error!") rescue Gitlab::HTTP::Error errors.push("Remote file `#{masked_location}` could not be fetched because of HTTP error!") - rescue Gitlab::HTTP::BlockedUrlError => e + rescue Errno::ECONNREFUSED, Gitlab::HTTP::BlockedUrlError => e errors.push("Remote file could not be fetched because #{e}!") end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index fc03ac125fd..a41bc2b39f2 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,6 +7,7 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize + # Will be removed with FF ci_refactoring_external_mapper FILE_CLASSES = [ External::File::Local, External::File::Project, @@ -15,6 +16,7 @@ module Gitlab External::File::Artifact ].freeze + # Will be removed with FF ci_refactoring_external_mapper FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze Error = Class.new(StandardError) @@ -22,27 +24,43 @@ module Gitlab TooManyIncludesError = Class.new(Error) def initialize(values, context) - @locations = Array.wrap(values.fetch(:include, [])) + @locations = Array.wrap(values.fetch(:include, [])).compact @context = context end def process - return [] if locations.empty? + return [] if @locations.empty? - logger.instrument(:config_mapper_process) do - process_without_instrumentation + context.logger.instrument(:config_mapper_process) do + if ::Feature.enabled?(:ci_refactoring_external_mapper, context.project) + process_without_instrumentation + else + legacy_process_without_instrumentation + end end end private - attr_reader :locations, :context + attr_reader :context delegate :expandset, :logger, to: :context def process_without_instrumentation - locations - .compact + locations = Normalizer.new(context).process(@locations) + locations = Filter.new(context).process(locations) + locations = LocationExpander.new(context).process(locations) + locations = VariablesExpander.new(context).process(locations) + + files = Matcher.new(context).process(locations) + Verifier.new(context).process(files) + + files + end + + # This and the following methods will be removed with FF ci_refactoring_external_mapper + def legacy_process_without_instrumentation + @locations .map(&method(:normalize_location)) .filter_map(&method(:verify_rules)) .flat_map(&method(:expand_project_files)) @@ -52,14 +70,8 @@ module Gitlab .each(&method(:verify!)) end - def normalize_location(location) - logger.instrument(:config_mapper_normalize) do - normalize_location_without_instrumentation(location) - end - end - # convert location if String to canonical form - def normalize_location_without_instrumentation(location) + def normalize_location(location) if location.is_a?(String) expanded_location = expand_variables(location) normalize_location_string(expanded_location) diff --git a/lib/gitlab/ci/config/external/mapper/base.rb b/lib/gitlab/ci/config/external/mapper/base.rb new file mode 100644 index 00000000000..d2f56d0b8f6 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/base.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Base class for mapper classes + class Base + def initialize(context) + @context = context + end + + def process(*args) + context.logger.instrument(mapper_instrumentation_key) do + process_without_instrumentation(*args) + end + end + + private + + attr_reader :context + + def process_without_instrumentation + raise NotImplementedError + end + + def mapper_instrumentation_key + "config_mapper_#{self.class.name.demodulize.downcase}".to_sym + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/filter.rb b/lib/gitlab/ci/config/external/mapper/filter.rb new file mode 100644 index 00000000000..4d2b26c7d98 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/filter.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Filters locations according to rules + class Filter < Base + private + + def process_without_instrumentation(locations) + locations.select do |location| + Rules.new(location[:rules]).evaluate(context).pass? + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/location_expander.rb b/lib/gitlab/ci/config/external/mapper/location_expander.rb new file mode 100644 index 00000000000..a4ca058f0d9 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/location_expander.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Expands locations to include all files matching the pattern + class LocationExpander < Base + private + + def process_without_instrumentation(locations) + locations.flat_map do |location| + if location[:project] + expand_project_files(location) + elsif location[:local] + expand_wildcard_paths(location) + else + location + end + end + end + + def expand_project_files(location) + Array.wrap(location[:file]).map do |file| + location.merge(file: file) + end + end + + def expand_wildcard_paths(location) + return location unless location[:local].include?('*') + + context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| + { local: path } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb new file mode 100644 index 00000000000..85e19ff1ced --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Matches the first file type that matches the given location + class Matcher < Base + FILE_CLASSES = [ + External::File::Local, + External::File::Project, + External::File::Remote, + External::File::Template, + External::File::Artifact + ].freeze + + FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze + + private + + def process_without_instrumentation(locations) + locations.map do |location| + matching = FILE_CLASSES.map do |file_class| + file_class.new(location, context) + end.select(&:matching?) + + if matching.one? + matching.first + elsif matching.empty? + raise Mapper::AmbigiousSpecificationError, + "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ + "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + else + raise Mapper::AmbigiousSpecificationError, + "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + end + end + end + + def masked_location(location) + context.mask_variables_from(location) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/normalizer.rb b/lib/gitlab/ci/config/external/mapper/normalizer.rb new file mode 100644 index 00000000000..8fc798e78a0 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/normalizer.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Converts locations to canonical form (local:/remote:) if String + class Normalizer < Base + def initialize(context) + super + + @variables_expander = VariablesExpander.new(context) + end + + private + + attr_reader :variables_expander + + def process_without_instrumentation(locations) + locations.map do |location| + if location.is_a?(String) + # We need to expand before normalizing because the information of + # whether if it's a remote or local path may be hidden inside the variable. + location = variables_expander.expand(location) + + normalize_location_string(location) + else + location.deep_symbolize_keys + end + end + end + + def normalize_location_string(location) + if ::Gitlab::UrlSanitizer.valid?(location) + { remote: location } + else + { local: location } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/variables_expander.rb b/lib/gitlab/ci/config/external/mapper/variables_expander.rb new file mode 100644 index 00000000000..fddf04984d8 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/variables_expander.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Handles variable expansion + class VariablesExpander < Base + def expand(data) + if data.is_a?(String) + expand_variable(data) + else + transform_and_expand_variable(data) + end + end + + private + + def process_without_instrumentation(locations) + locations.map { |location| expand(location) } + end + + def transform_and_expand_variable(data) + data.transform_values do |values| + case values + when Array + values.map { |value| expand_variable(value.to_s) } + when String + expand_variable(values) + else + values + end + end + end + + def expand_variable(data) + ExpandVariables.expand(data, -> { variables }) + end + + def variables + @variables ||= context.variables_hash + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb new file mode 100644 index 00000000000..6d6f227b940 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Fetches file contents and verifies them + class Verifier < Base + private + + def process_without_instrumentation(files) + files.select do |file| + verify_max_includes! + verify_execution_time! + + file.validate! + + context.expandset.add(file) + end + end + + def verify_max_includes! + return if context.expandset.count < context.max_includes + + raise Mapper::TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" + end + + def verify_execution_time! + context.check_execution_time! + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index 6a4aee26d80..e15b51fbff4 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -32,9 +32,7 @@ module Gitlab def validate_external_files! @external_files.each do |file| - logger.instrument(:config_external_verify) do - raise IncludeError, file.error_message unless file.valid? - end + raise IncludeError, file.error_message unless file.valid? end end diff --git a/lib/gitlab/ci/environment_matcher.rb b/lib/gitlab/ci/environment_matcher.rb new file mode 100644 index 00000000000..7d7a7742f68 --- /dev/null +++ b/lib/gitlab/ci/environment_matcher.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class EnvironmentMatcher + def initialize(pattern) + @pattern = pattern + end + + def match?(environment) + return false if pattern.blank? + + exact_match?(environment) || wildcard_match?(environment) + end + + private + + attr_reader :pattern, :match_type + + def exact_match?(environment) + pattern == environment + end + + def wildcard_match?(environment) + return false unless wildcard? + + wildcard_regex.match?(environment) + end + + def wildcard? + pattern.include?('*') + end + + def wildcard_regex + @wildcard_regex ||= Regexp.new(pattern.gsub('*', '.*')) + end + end + end +end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 51743a1f273..e0112a1b1c2 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -73,7 +73,7 @@ module Gitlab end def yaml_processor_result(content, logger) - logger.instrument(:yaml_process) do + logger.instrument(:yaml_process, once: true) do Gitlab::Ci::YamlProcessor.new(content, project: @project, user: @current_user, sha: @sha, @@ -119,7 +119,7 @@ module Gitlab environment: job[:environment], when: job[:when], allow_failure: job[:allow_failure], - needs: job.dig(:needs_attributes) + needs: job[:needs_attributes] } end end @@ -130,10 +130,10 @@ module Gitlab def build_logger Gitlab::Ci::Pipeline::Logger.new(project: @project) do |l| l.log_when do |observations| - values = observations['yaml_process_duration_s'] - next false if values.empty? + duration = observations['yaml_process_duration_s'] + next false unless duration - values.max >= LOG_MAX_DURATION_THRESHOLD + duration >= LOG_MAX_DURATION_THRESHOLD end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 0ac012b9fd1..67817c9f832 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -65,8 +65,14 @@ module Gitlab ) end + # New Oj parsers are not thread safe, therefore, + # we need to initialize them for each thread. + def introspect_parser + Thread.current[:introspect_parser] ||= Oj::Introspect.new(filter: "remediations") + end + def report_data - @report_data ||= Gitlab::Json.parse!(json_data) + @report_data ||= introspect_parser.parse(json_data) end def report_version diff --git a/lib/gitlab/ci/pipeline/chain/build/associations.rb b/lib/gitlab/ci/pipeline/chain/build/associations.rb index b5d63691849..b484a88a381 100644 --- a/lib/gitlab/ci/pipeline/chain/build/associations.rb +++ b/lib/gitlab/ci/pipeline/chain/build/associations.rb @@ -31,7 +31,8 @@ module Gitlab source_pipeline: @command.bridge.pipeline, source_project: @command.bridge.project, source_bridge: @command.bridge, - project: @command.project + project: @command.project, + source_partition_id: @command.bridge.partition_id ) end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 07a3aff1862..53c8a7ac122 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -11,11 +11,10 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def perform! - ff_enabled = Feature.enabled?(:ci_skip_auto_cancelation_on_child_pipelines, project) - return if ff_enabled && pipeline.parent_pipeline? # skip if child pipeline + return if pipeline.parent_pipeline? # skip if child pipeline return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines(ff_enabled), name: 'cancel_pending_pipelines') do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| auto_cancel_interruptible_pipelines(cancelables_batch.ids) end @@ -29,19 +28,14 @@ module Gitlab private - def auto_cancelable_pipelines(ff_enabled) - relation = project.all_pipelines + def auto_cancelable_pipelines + project.all_pipelines .created_after(1.week.ago) .ci_and_parent_sources .for_ref(pipeline.ref) .where_not_sha(project.commit(pipeline.ref).try(:id)) .alive_or_scheduled - - if ff_enabled - relation.id_not_in(pipeline.id) - else - relation.id_not_in(pipeline.same_family_pipeline_ids) - end + .id_not_in(pipeline.id) end def auto_cancel_interruptible_pipelines(pipeline_ids) diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 5ec04b4889e..31b130b5ab7 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -98,7 +98,7 @@ module Gitlab def observe_step_duration(step_class, duration) step = step_class.name.underscore.parameterize(separator: '_') - logger.observe("pipeline_step_#{step}_duration_s", duration) + logger.observe("pipeline_step_#{step}_duration_s", duration, once: true) if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops) metrics.pipeline_creation_step_duration_histogram @@ -107,14 +107,14 @@ module Gitlab end def observe_creation_duration(duration) - logger.observe(:pipeline_creation_duration_s, duration) + logger.observe(:pipeline_creation_duration_s, duration, once: true) metrics.pipeline_creation_duration_histogram - .observe({}, duration.seconds) + .observe({ gitlab: gitlab_org_project?.to_s }, duration.seconds) end def observe_pipeline_size(pipeline) - logger.observe(:pipeline_size_count, pipeline.total_size) + logger.observe(:pipeline_size_count, pipeline.total_size, once: true) metrics.pipeline_size_histogram .observe({ source: pipeline.source.to_s, plan: project.actual_plan_name }, pipeline.total_size) @@ -157,6 +157,10 @@ module Gitlab def full_git_ref_name_unavailable? ref == origin_ref end + + def gitlab_org_project? + project.full_path == 'gitlab-org/gitlab' + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 5548fca320f..ad6b2fd3411 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,7 +11,7 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - result = logger.instrument(:pipeline_config_process) do + result = logger.instrument(:pipeline_config_process, once: true) do processor = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 207b4b5ff8b..d4c4f94c7d3 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - logger.instrument_with_sql(:pipeline_save) do + logger.instrument_once_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do ::Ci::BulkInsertableTags.with_bulk_insert_tags do pipeline.transaction do diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb index 1b9dd158733..ebea6a538ef 100644 --- a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -16,7 +16,7 @@ module Gitlab private def ensure_environment(build) - ::Environments::CreateForBuildService.new.execute(build, merge_request: @command.merge_request) + ::Environments::CreateForBuildService.new.execute(build) end end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index feae123f216..ae98c55e425 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,7 +13,7 @@ module Gitlab raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. - logger.instrument(:pipeline_allocate_seed_attributes) do + logger.instrument(:pipeline_allocate_seed_attributes, once: true) do pipeline.ensure_project_iid! pipeline.ensure_ci_ref! end @@ -25,7 +25,7 @@ module Gitlab ## # Gather all runtime build/stage errors # - seed_errors = logger.instrument(:pipeline_seed_evaluation) do + seed_errors = logger.instrument(:pipeline_seed_evaluation, once: true) do pipeline_seed.errors end @@ -44,7 +44,7 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do - logger.instrument(:pipeline_seed_initialization) do + logger.instrument(:pipeline_seed_initialization, once: true) do stages_attributes = @command.yaml_processor_result.stages_attributes Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) @@ -61,7 +61,7 @@ module Gitlab end def root_variables - logger.instrument(:pipeline_seed_merge_variables) do + strong_memoize(:root_variables) do ::Gitlab::Ci::Variables::Helpers.merge_variables( @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 4b7cbae5004..f393406b549 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -23,7 +23,7 @@ module Gitlab log_conditions.push(block) end - def instrument(operation) + def instrument(operation, once: false) return yield unless enabled? raise ArgumentError, 'block not given' unless block_given? @@ -32,63 +32,72 @@ module Gitlab result = yield - observe("#{operation}_duration_s", current_monotonic_time - op_started_at) + observe("#{operation}_duration_s", current_monotonic_time - op_started_at, once: once) result end - def instrument_with_sql(operation, &block) + def instrument_once_with_sql(operation, &block) op_start_db_counters = current_db_counter_payload - result = instrument(operation, &block) + result = instrument(operation, once: true, &block) - observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload) + observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload, once: true) result end - def observe(operation, value) + def observe(operation, value, once: false) return unless enabled? - observations[operation.to_s].push(value) + if once + observations[operation.to_s] = value + else + observations[operation.to_s] ||= [] + observations[operation.to_s].push(value) + end end def commit(pipeline:, caller:) return unless log? - attributes = { - class: self.class.name.to_s, - pipeline_creation_caller: caller, - project_id: project&.id, # project is not available when called from `/ci/lint` - pipeline_persisted: pipeline.persisted?, - pipeline_source: pipeline.source, - pipeline_creation_service_duration_s: age - } - - if pipeline.persisted? - attributes[:pipeline_builds_tags_count] = pipeline.tags_count - attributes[:pipeline_builds_distinct_tags_count] = pipeline.distinct_tags_count - attributes[:pipeline_id] = pipeline.id + Gitlab::ApplicationContext.with_context(project: project) do + attributes = Gitlab::ApplicationContext.current.merge( + class: self.class.name.to_s, + pipeline_creation_caller: caller, + project_id: project&.id, # project is not available when called from `/ci/lint` + pipeline_persisted: pipeline.persisted?, + pipeline_source: pipeline.source, + pipeline_creation_service_duration_s: age + ) + + if pipeline.persisted? + attributes[:pipeline_builds_tags_count] = pipeline.tags_count + attributes[:pipeline_builds_distinct_tags_count] = pipeline.distinct_tags_count + attributes[:pipeline_id] = pipeline.id + end + + attributes.compact! + attributes.stringify_keys! + attributes.merge!(observations_hash) + + destination.info(attributes) end - - attributes.compact! - attributes.stringify_keys! - attributes.merge!(observations_hash) - - destination.info(attributes) end def observations_hash - observations.transform_values do |values| - next if values.empty? - - { - 'count' => values.size, - 'min' => values.min, - 'max' => values.max, - 'sum' => values.sum, - 'avg' => values.sum / values.size - } + observations.transform_values do |observation| + next if observation.blank? + + if observation.is_a?(Array) + { + 'count' => observation.size, + 'max' => observation.max, + 'sum' => observation.sum + } + else + observation + end end.compact end @@ -110,21 +119,20 @@ module Gitlab end def enabled? - strong_memoize(:enabled) do - ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) - end + ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) end + strong_memoize_attr :enabled?, :enabled def observations - @observations ||= Hash.new { |hash, key| hash[key] = [] } + @observations ||= {} end - def observe_sql_counters(operation, start_db_counters, end_db_counters) + def observe_sql_counters(operation, start_db_counters, end_db_counters, once: false) end_db_counters.each do |key, value| result = value - start_db_counters.fetch(key, 0) next if result == 0 - observe("#{operation}_#{key}", result) + observe("#{operation}_#{key}", result, once: once) end end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index c3e0f043b44..04565beeecc 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -9,7 +9,8 @@ module Gitlab def self.pipeline_creation_duration_histogram name = :gitlab_ci_pipeline_creation_duration_seconds comment = 'Pipeline creation duration' - labels = {} + # @gitlab: boolean value - if project is gitlab-org/gitlab + labels = { gitlab: false } buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] ::Gitlab::Metrics.histogram(name, comment, labels, buckets) diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 2e4267e986b..b0b79b994c1 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -9,12 +9,13 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(context, attributes, stages_for_needs_lookup = []) + def initialize(context, attributes, stages_for_needs_lookup, stage) @context = context @pipeline = context.pipeline @seed_attributes = attributes @stages_for_needs_lookup = stages_for_needs_lookup.compact @needs_attributes = dig(:needs_attributes) + @stage = stage @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true } @@ -33,6 +34,8 @@ module Gitlab .new(attributes.delete(:cache), @pipeline) calculate_yaml_variables! + + @processable = initialize_processable end def name @@ -40,21 +43,20 @@ module Gitlab end def included? - strong_memoize(:inclusion) do - logger.instrument(:pipeline_seed_build_inclusion) do - if @using_rules - rules_result.pass? - elsif @using_only || @using_except - all_of_only? && none_of_except? - else - true - end + logger.instrument(:pipeline_seed_build_inclusion) do + if @using_rules + rules_result.pass? + elsif @using_only || @using_except + all_of_only? && none_of_except? + else + true end end end + strong_memoize_attr :included?, :inclusion def errors - strong_memoize(:errors) do + logger.instrument(:pipeline_seed_build_errors) do # We check rules errors before checking "included?" because rules affects its inclusion status. next rules_errors if rules_errors next unless included? @@ -62,14 +64,22 @@ module Gitlab [needs_errors, variable_expansion_errors].compact.flatten end end + strong_memoize_attr :errors + # TODO: Method used only in specs. Replace with `to_resource.attributes` when + # the feature flag ci_reuse_build_in_seed_context is removed. + # Then remove this method. def attributes - @seed_attributes - .deep_merge(pipeline_attributes) - .deep_merge(rules_attributes) - .deep_merge(allow_failure_criteria_attributes) - .deep_merge(@cache.cache_attributes) - .deep_merge(runner_tags) + if reuse_build_in_seed_context? + initial_attributes.deep_merge(evaluated_attributes) + else + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(rules_attributes) + .deep_merge(allow_failure_criteria_attributes) + .deep_merge(@cache.cache_attributes) + .deep_merge(runner_tags) + end end def bridge? @@ -80,12 +90,30 @@ module Gitlab end def to_resource - strong_memoize(:resource) do - initialize_processable + logger.instrument(:pipeline_seed_build_to_resource) do + if reuse_build_in_seed_context? + # The `options` attribute need to be entirely reassigned because they may + # be overridden by evaluated_attributes. + # We also don't want to reassign all the `initial_attributes` since those + # can affect performance. We only want to assign what's changed. + assignable_attributes = initial_attributes.slice(:options) + .deep_merge(evaluated_attributes) + processable.assign_attributes(assignable_attributes) + processable + else + legacy_initialize_processable + end end end + strong_memoize_attr :to_resource - def initialize_processable + private + + attr_reader :processable + + delegate :logger, to: :@context + + def legacy_initialize_processable if bridge? ::Ci::Bridge.new(attributes) else @@ -93,9 +121,28 @@ module Gitlab end end - private + def initialize_processable + return unless reuse_build_in_seed_context? - delegate :logger, to: :@context + if bridge? + ::Ci::Bridge.new(initial_attributes) + else + ::Ci::Build.new(initial_attributes) + end + end + + def initial_attributes + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(ci_stage: @stage) + .deep_merge(@cache.cache_attributes) + end + + def evaluated_attributes + rules_attributes + .deep_merge(allow_failure_criteria_attributes) + .deep_merge(runner_tags) + end def all_of_only? @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } @@ -155,40 +202,39 @@ module Gitlab end def rules_attributes - strong_memoize(:rules_attributes) do - next {} unless @using_rules + return {} unless @using_rules - rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( - @seed_attributes[:yaml_variables], rules_result.variables - ) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) - rules_result.build_attributes.merge(yaml_variables: rules_variables_result) - end + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) end + strong_memoize_attr :rules_attributes def rules_result - strong_memoize(:rules_result) do - @rules.evaluate(@pipeline, evaluate_context) - end + @rules.evaluate(@pipeline, evaluate_context) end + strong_memoize_attr :rules_result def rules_errors - strong_memoize(:rules_errors) do - ["Failed to parse rule for #{name}: #{rules_result.errors.join(', ')}"] if rules_result.errors.present? - end + ["Failed to parse rule for #{name}: #{rules_result.errors.join(', ')}"] if rules_result.errors.present? end + strong_memoize_attr :rules_errors def evaluate_context - strong_memoize(:evaluate_context) do + if reuse_build_in_seed_context? + Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes, processable) + else Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) end end + strong_memoize_attr :evaluate_context def runner_tags - strong_memoize(:runner_tags) do - { tag_list: evaluate_runner_tags }.compact - end + { tag_list: evaluate_runner_tags }.compact end + strong_memoize_attr :runner_tags def evaluate_runner_tags @seed_attributes.delete(:tag_list)&.map do |tag| @@ -211,6 +257,11 @@ module Gitlab from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance ) end + + def reuse_build_in_seed_context? + Feature.enabled?(:ci_reuse_build_in_seed_context, @pipeline.project) + end + strong_memoize_attr :reuse_build_in_seed_context?, :reuse_build_in_seed_context end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 78ffaaa7e81..781065a63db 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -6,7 +6,7 @@ module Gitlab module Seed class Build class Cache - def initialize(pipeline, cache) + def initialize(pipeline, cache, custom_key_prefix) @pipeline = pipeline local_cache = cache.to_h.deep_dup @key = local_cache.delete(:key) @@ -14,6 +14,7 @@ module Gitlab @policy = local_cache.delete(:policy) @untracked = local_cache.delete(:untracked) @when = local_cache.delete(:when) + @custom_key_prefix = custom_key_prefix raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? end @@ -45,6 +46,7 @@ module Gitlab def key_from_files return unless @key.is_a?(Hash) + @key[:prefix] ||= @custom_key_prefix.to_s [@key[:prefix], files_digest].select(&:present?).join('-') end diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb index 9e609debeed..57ad2546f1c 100644 --- a/lib/gitlab/ci/pipeline/seed/pipeline.rb +++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb @@ -38,8 +38,10 @@ module Gitlab private + delegate :logger, to: :@context + def stage_seeds - strong_memoize(:stage_seeds) do + logger.instrument(:pipeline_seed_stage_seeds) do seeds = @stages_attributes.inject([]) do |previous_stages, attributes| seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@context, attributes, previous_stages) previous_stages + [seed] @@ -48,6 +50,7 @@ module Gitlab seeds.select(&:included?) end end + strong_memoize_attr :stage_seeds end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 1c4247bd5ee..c3e94529634 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,54 +10,49 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - def initialize(context, attributes, previous_stages) - @context = context - @pipeline = context.pipeline - @attributes = attributes - @previous_stages = previous_stages - - @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(context, attributes, previous_stages + [self]) + attr_reader :attributes + + def initialize(context, stage_attributes, previous_stages) + pipeline = context.pipeline + @attributes = { + name: stage_attributes.fetch(:name), + position: stage_attributes.fetch(:index), + pipeline: pipeline, + project: pipeline.project, + partition_id: pipeline.partition_id + } + + @stage = ::Ci::Stage.new(@attributes) + + @builds = stage_attributes.fetch(:builds).map do |build_attributes| + Seed::Build.new(context, build_attributes, previous_stages + [self], @stage) end end - def attributes - { name: @attributes.fetch(:name), - position: @attributes.fetch(:index), - pipeline: @pipeline, - project: @pipeline.project, - partition_id: @pipeline.partition_id } - end - def seeds - strong_memoize(:seeds) do - @builds.select(&:included?) - end + @builds.select(&:included?) end + strong_memoize_attr :seeds def errors - strong_memoize(:errors) do - @builds.flat_map(&:errors).compact - end + @builds.flat_map(&:errors).compact end + strong_memoize_attr :errors def seeds_names - strong_memoize(:seeds_names) do - seeds.map(&:name).to_set - end + seeds.map(&:name).to_set end + strong_memoize_attr :seeds_names def included? seeds.any? end def to_resource - strong_memoize(:stage) do - ::Ci::Stage.new(attributes).tap do |stage| - stage.statuses = seeds.map(&:to_resource) - end - end + @stage.statuses = seeds.map(&:to_resource) + @stage end + strong_memoize_attr :to_resource end end end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index dd9b9cc6d55..92a91854358 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -83,8 +83,8 @@ module Gitlab message cve solution - ].each_with_object({}) do |key, hash| - hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + ].index_with do |key| + public_send(key) # rubocop:disable GitlabSecurity/PublicSend end end @@ -98,7 +98,7 @@ module Gitlab end def unsafe?(severity_levels, report_types) - severity.to_s.in?(severity_levels) && (report_types.blank? || report_type.to_s.in?(report_types) ) + severity.to_s.in?(severity_levels) && (report_types.blank? || report_type.to_s.in?(report_types)) end def eql?(other) diff --git a/lib/gitlab/ci/reports/security/finding_key.rb b/lib/gitlab/ci/reports/security/finding_key.rb index ad047fbf904..d42a0ea5b2e 100644 --- a/lib/gitlab/ci/reports/security/finding_key.rb +++ b/lib/gitlab/ci/reports/security/finding_key.rb @@ -15,7 +15,7 @@ module Gitlab has_fingerprints? && other.has_fingerprints? && location_fingerprint == other.location_fingerprint && - identifier_fingerprint == other.identifier_fingerprint + identifier_fingerprint == other.identifier_fingerprint end def hash diff --git a/lib/gitlab/ci/reports/security/identifier.rb b/lib/gitlab/ci/reports/security/identifier.rb index 4ba943cdcbc..0ff6be6acc4 100644 --- a/lib/gitlab/ci/reports/security/identifier.rb +++ b/lib/gitlab/ci/reports/security/identifier.rb @@ -31,8 +31,8 @@ module Gitlab fingerprint name url - ].each_with_object({}) do |key, hash| - hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + ].index_with do |key| + public_send(key) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb index 5c08381d5cc..8425881a4ab 100644 --- a/lib/gitlab/ci/reports/security/reports.rb +++ b/lib/gitlab/ci/reports/security/reports.rb @@ -21,29 +21,6 @@ module Gitlab def findings reports.values.flat_map(&:findings) end - - def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states, report_types = []) - if Feature.enabled?(:require_approval_on_scan_removal, pipeline.project) && scan_removed?(target_reports) - return true - end - - unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) > vulnerabilities_allowed - end - - def unsafe_findings_uuids(severity_levels, report_types) - findings.select { |finding| finding.unsafe?(severity_levels, report_types) }.map(&:uuid) - end - - private - - def unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) - new_uuids = unsafe_findings_uuids(severity_levels, report_types) - target_reports&.unsafe_findings_uuids(severity_levels, report_types).to_a - new_uuids.count - end - - def scan_removed?(target_reports) - (target_reports&.reports&.keys.to_a - reports.keys).any? - end end end end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index d0388c65f58..dcc593b4403 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -96,8 +96,8 @@ module Gitlab end def sort_by_execution_time_desc - @test_cases = @test_cases.keys.each_with_object({}) do |key, hash| - hash[key] = @test_cases[key].sort_by { |_key, test_case| -test_case.execution_time }.to_h + @test_cases = @test_cases.keys.index_with do |key| + @test_cases[key].sort_by { |_key, test_case| -test_case.execution_time }.to_h end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index 68c911d3dbb..bcda2fec5ba 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -22,7 +22,8 @@ module Gitlab osx: { human_readable_name: "macOS", download_locations: { - amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64" + amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64", + arm64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-arm64" }, install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/osx/install.sh", runner_executable: "gitlab-runner" @@ -61,7 +62,7 @@ module Gitlab def install_script with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError] do - raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Architecture not found for OS') unless environment[:download_locations].key?(@arch.to_sym) + raise Gitlab::Ci::RunnerInstructions::ArgumentError, _('Architecture not found for OS') unless environment[:download_locations].key?(@arch.to_sym) replace_variables(get_file(environment[:install_script_template_path])) end @@ -69,7 +70,7 @@ module Gitlab def register_command with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError, Gitlab::Access::AccessDeniedError] do - raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('No runner executable') unless environment[:runner_executable] + raise Gitlab::Ci::RunnerInstructions::ArgumentError, _('No runner executable') unless environment[:runner_executable] server_url = Gitlab::Routing.url_helpers.root_url(only_path: false) runner_executable = environment[:runner_executable] @@ -90,12 +91,12 @@ module Gitlab end def environment - @environment ||= OS[@os.to_sym] || ( raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Invalid OS') ) + @environment ||= OS[@os.to_sym] || (raise Gitlab::Ci::RunnerInstructions::ArgumentError, _('Invalid OS')) end def validate_params - @errors << s_('Missing OS') unless @os.present? - @errors << s_('Missing arch') unless @arch.present? + @errors << _('Missing OS') unless @os.present? + @errors << _('Missing arch') unless @arch.present? end def replace_variables(expression) diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index fddcc1492a8..11420b05dfb 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -177,11 +177,11 @@ include: - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - template: Jobs/Helm-2to3.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml + - template: Jobs/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml + - template: Jobs/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml + - template: Jobs/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml + - template: Jobs/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml # The latest build job generates a dotenv report artifact with a CI_APPLICATION_TAG # that also includes the image digest. This configures Auto Deploy to receive diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 671925c5df6..16ce85548df 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -11,13 +11,6 @@ image: gradle:alpine -# Disable the Gradle daemon for Continuous Integration servers as correctness -# is usually a priority over speed in CI environments. Using a fresh -# runtime for each build is more reliable since the runtime is completely -# isolated from any previous builds. -variables: - GRADLE_OPTS: "-Dorg.gradle.daemon=false" - before_script: - GRADLE_USER_HOME="$(pwd)/.gradle" - export GRADLE_USER_HOME diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index fcf2ac7de7a..026ddf4a17a 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -7,7 +7,7 @@ browser_performance: variables: DOCKER_TLS_CERTDIR: "" SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 14.1.0 + SITESPEED_VERSION: 26.1.0 SITESPEED_OPTIONS: '' services: - name: 'docker:20.10.12-dind' diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 04b7dacf2dd..218c2f79e6a 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -7,7 +7,7 @@ browser_performance: variables: DOCKER_TLS_CERTDIR: "" SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 14.1.0 + SITESPEED_VERSION: latest SITESPEED_OPTIONS: '' services: - name: 'docker:20.10.12-dind' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 23efed212f8..b4beeb60dfd 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,8 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:0.87.0" + CODE_QUALITY_IMAGE_TAG: "0.87.3" + CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml new file mode 100644 index 00000000000..fa609afc5a8 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml @@ -0,0 +1,54 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml + +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# CS_DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables + +variables: + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + +container_scanning: + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" + stage: test + variables: + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting + GIT_STRATEGY: none + allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + dependencies: [] + script: + - gtcs scan + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f750bda2a3f --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,68 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml + +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# CS_DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables + +variables: + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + +container_scanning: + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" + stage: test + variables: + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting + GIT_STRATEGY: none + allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + dependencies: [] + script: + - gtcs scan + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 936d8751fe1..12105e0e95d 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -4,8 +4,8 @@ load_performance: allow_failure: true variables: DOCKER_TLS_CERTDIR: "" - K6_IMAGE: loadimpact/k6 - K6_VERSION: 0.27.0 + K6_IMAGE: grafana/k6 + K6_VERSION: 0.41.0 K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index a6d47e31de2..2c5027cdb43 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -238,6 +238,8 @@ semgrep-sast: - '**/*.java' - '**/*.cs' - '**/*.html' + - '**/*.scala' + - '**/*.sc' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index 4600468ef30..58709d3ab62 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -299,6 +299,8 @@ semgrep-sast: - '**/*.java' - '**/*.html' - '**/*.cs' + - '**/*.scala' + - '**/*.sc' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. @@ -313,6 +315,8 @@ semgrep-sast: - '**/*.java' - '**/*.html' - '**/*.cs' + - '**/*.scala' + - '**/*.sc' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 79a08c33fdf..879d6a7a468 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,54 +1,5 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +# This template moved to Jobs/Container-Scanning.gitlab-ci.yml in GitLab 15.6 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381665 -# Use this template to enable container scanning in your project. -# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` -# keyword. -# The template should work without modifications but you can customize the template settings if -# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# -# Requirements: -# - A `test` stage to be present in the pipeline. -# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the -# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. -# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the -# image to be scanned is in a private registry. -# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the -# CS_DOCKERFILE_PATH variable. -# -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). -# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables - -variables: - CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" - -container_scanning: - image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" - stage: test - variables: - # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your - # `.gitlab-ci.yml` file and set it to `fetch`. - # For details, see the following links: - # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template - # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting - GIT_STRATEGY: none - allow_failure: true - artifacts: - reports: - container_scanning: gl-container-scanning-report.json - dependency_scanning: gl-dependency-scanning-report.json - paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] - dependencies: [] - script: - - gtcs scan - rules: - - if: $CONTAINER_SCANNING_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $CI_GITLAB_FIPS_MODE == "true" && - $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ - variables: - CS_IMAGE_SUFFIX: -fips - - if: $CI_COMMIT_BRANCH +include: + template: Jobs/Container-Scanning.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml index f7b1d12b3b3..7a4f451314e 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml @@ -1,68 +1,5 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +# This template moved to Jobs/Container-Scanning.latest.gitlab-ci.yml in GitLab 15.6 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381665 -# Use this template to enable container scanning in your project. -# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` -# keyword. -# The template should work without modifications but you can customize the template settings if -# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# -# Requirements: -# - A `test` stage to be present in the pipeline. -# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the -# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. -# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the -# image to be scanned is in a private registry. -# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the -# CS_DOCKERFILE_PATH variable. -# -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). -# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables - -variables: - CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" - -container_scanning: - image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" - stage: test - variables: - # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your - # `.gitlab-ci.yml` file and set it to `fetch`. - # For details, see the following links: - # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template - # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting - GIT_STRATEGY: none - allow_failure: true - artifacts: - reports: - container_scanning: gl-container-scanning-report.json - dependency_scanning: gl-dependency-scanning-report.json - paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] - dependencies: [] - script: - - gtcs scan - rules: - - if: $CONTAINER_SCANNING_DISABLED - when: never - - # Add the job to merge request pipelines if there's an open merge request. - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && - $CI_GITLAB_FIPS_MODE == "true" && - $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ - variables: - CS_IMAGE_SUFFIX: -fips - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - - if: $CI_OPEN_MERGE_REQUESTS - when: never - - # Add the job to branch pipelines. - - if: $CI_COMMIT_BRANCH && - $CI_GITLAB_FIPS_MODE == "true" && - $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ - variables: - CS_IMAGE_SUFFIX: -fips - - if: $CI_COMMIT_BRANCH +include: + template: Jobs/Container-Scanning.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index d933007ec61..89944e347f6 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -16,7 +16,7 @@ variables: COVFUZZ_VERSION: v3 # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries # to their own servers - COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + COVFUZZ_URL_PREFIX: "https://gitlab.com/security-products/gitlab-cov-fuzz/-/raw" coverage_fuzzing_unlicensed: diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml index feed4c47157..4f6ba427058 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml @@ -16,7 +16,7 @@ variables: COVFUZZ_VERSION: v3 # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries # to their own servers - COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + COVFUZZ_URL_PREFIX: "https://gitlab.com/security-products/gitlab-cov-fuzz/-/raw" coverage_fuzzing_unlicensed: diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 40060e96dff..c43296b5865 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -51,7 +51,4 @@ dast: $REVIEW_DISABLED when: never - if: $CI_COMMIT_BRANCH && - ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && - $GITLAB_FEATURES =~ /\bdast\b/ - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 50e9bb5431d..27bcc14bcf5 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -55,9 +55,6 @@ dast: # Add the job to merge request pipelines if there's an open merge request. - if: $CI_PIPELINE_SOURCE == "merge_request_event" && - ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && - $GITLAB_FEATURES =~ /\bdast\b/ - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $GITLAB_FEATURES =~ /\bdast\b/ # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -66,9 +63,6 @@ dast: # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && - ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && - $GITLAB_FEATURES =~ /\bdast\b/ - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ after_script: # Remove any debug.log files because they might contain secrets. diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index fd04c86e6c7..631f6cecddf 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -9,9 +9,9 @@ # Usage: # # include: -# - template: Secure-Binaries.gitlab-ci.yml +# - template: Security/Secure-Binaries.gitlab-ci.yml # -# Docs: https://docs.gitlab.com/ee/topics/airgap/ +# Docs: https://docs.gitlab.com/ee/user/application_security/offline_deployments/ variables: # Setting this variable will affect all Security templates @@ -38,7 +38,7 @@ variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" services: - - docker:stable-dind + - docker:dind script: - docker info - env diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index c3113ffebf3..c1a90955f7f 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -17,10 +17,10 @@ browser_performance: variables: URL: '' SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 14.1.0 + SITESPEED_VERSION: 26.1.0 SITESPEED_OPTIONS: '' services: - - docker:stable-dind + - docker:dind script: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index c9f0c173692..adc92fde5ae 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -17,10 +17,10 @@ browser_performance: variables: URL: '' SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 14.1.0 + SITESPEED_VERSION: latest SITESPEED_OPTIONS: '' services: - - docker:stable-dind + - docker:dind script: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index bf5cfbb519d..a907915587a 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -15,13 +15,13 @@ load_performance: stage: performance image: docker:git variables: - K6_IMAGE: loadimpact/k6 - K6_VERSION: 0.27.0 + K6_IMAGE: grafana/k6 + K6_VERSION: 0.41.0 K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: - - docker:stable-dind + - docker:dind script: - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_DOCKER_OPTIONS $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS artifacts: diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 8db8ea3a720..8e18d57b724 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -53,7 +53,7 @@ module Gitlab # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 # Until then, we need to make both the old and the new KUBECONFIG contexts available collection.concat(deployment_variables(environment: environment, job: job)) - template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.try(:token)).execute + template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.try(:token), environment: environment).execute kubeconfig_yaml = collection['KUBECONFIG']&.value template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? @@ -135,6 +135,9 @@ module Gitlab variables.append(key: 'CI_NODE_INDEX', value: job.options[:instance].to_s) if job.options&.include?(:instance) variables.append(key: 'CI_NODE_TOTAL', value: ci_node_total_value(job).to_s) + # Set environment name here so we can access it when evaluating the job's rules + variables.append(key: 'CI_ENVIRONMENT_NAME', value: job.environment) if job.environment + # legacy variables variables.append(key: 'CI_BUILD_NAME', value: job.name) variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index ff255543d3b..f2c1ad0575d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -107,6 +107,7 @@ module Gitlab cache: job[:cache], resource_group_key: job[:resource_group], scheduling_type: job[:scheduling_type], + id_tokens: job[:id_tokens], options: { image: job[:image], services: job[:services], @@ -118,6 +119,7 @@ module Gitlab before_script: job[:before_script], script: job[:script], after_script: job[:after_script], + hooks: job[:hooks], environment: job[:environment], resource_group_key: job[:resource_group], retry: job[:retry], diff --git a/lib/gitlab/cluster/rack_timeout_observer.rb b/lib/gitlab/cluster/rack_timeout_observer.rb index 5182b2be148..15dd6a59e19 100644 --- a/lib/gitlab/cluster/rack_timeout_observer.rb +++ b/lib/gitlab/cluster/rack_timeout_observer.rb @@ -3,6 +3,7 @@ module Gitlab module Cluster class RackTimeoutObserver + include ActionView::Helpers::SanitizeHelper TRANSITION_STATES = %i(ready active).freeze def initialize @@ -28,9 +29,9 @@ module Gitlab params = controller_params(env) || grape_params(env) || {} { - controller: params['controller'], - action: params['action'], - route: params['route'], + controller: sanitize(params['controller']), + action: sanitize(params['action']), + route: sanitize(params['route']), state: info.state } end diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb index 01c534c15a0..7d9280ddba2 100644 --- a/lib/gitlab/color.rb +++ b/lib/gitlab/color.rb @@ -215,13 +215,11 @@ module Gitlab def rgb return [] unless valid? - @rgb ||= begin - if @value.length == 4 - @value[1, 4].scan(/./).map { |v| (v * 2).hex } - else - @value[1, 7].scan(/.{2}/).map(&:hex) - end - end + @rgb ||= if @value.length == 4 + @value[1, 4].scan(/./).map { |v| (v * 2).hex } + else + @value[1, 7].scan(/.{2}/).map(&:hex) + end end end end diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index d266d5218de..c8ad2521574 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -7,19 +7,21 @@ module Gitlab extend ActiveSupport::Concern class_methods do - def attributes(*attributes) + def attributes(*attributes, prefix: nil) attributes.flatten.each do |attribute| - if method_defined?(attribute) - raise ArgumentError, "Method '#{attribute}' already defined in '#{name}'" + attribute_method = prefix ? "#{prefix}_#{attribute}" : attribute + + if method_defined?(attribute_method) + raise ArgumentError, "Method '#{attribute_method}' already defined in '#{name}'" end - define_method(attribute) do + define_method(attribute_method) do return unless config.is_a?(Hash) config[attribute] end - define_method("has_#{attribute}?") do + define_method("has_#{attribute_method}?") do config.is_a?(Hash) && config.key?(attribute) end end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index d40a6323d4f..7bcbcf84a4e 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -236,14 +236,12 @@ module Gitlab else :modified_target_removed_source end + elsif our_path.present? && their_path.present? + :both_added + elsif their_path.blank? + diff_file.renamed_file? ? :renamed_same_file : :removed_target_renamed_source else - if our_path.present? && their_path.present? - :both_added - elsif their_path.blank? - diff_file.renamed_file? ? :renamed_same_file : :removed_target_renamed_source - else - :removed_source_renamed_target - end + :removed_source_renamed_target end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 29e8e631fb7..8b1298d0561 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -43,7 +43,10 @@ module Gitlab allow_websocket_connections(directives) allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? - allow_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn + # Support for Sentry setup via configuration files will be removed in 16.0 + # in favor of Gitlab::CurrentSettings. + allow_legacy_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn + allow_sentry(directives) if Gitlab::CurrentSettings.try(:sentry_enabled) && Gitlab::CurrentSettings.try(:sentry_clientside_dsn) allow_framed_gitlab_paths(directives) allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? allow_review_apps(directives) if ENV['REVIEW_APPS_ENABLED'] @@ -135,13 +138,22 @@ module Gitlab append_to_directive(directives, 'frame_src', customersdot_host) end - def self.allow_sentry(directives) + def self.allow_legacy_sentry(directives) + # Support for Sentry setup via configuration files will be removed in 16.0 + # in favor of Gitlab::CurrentSettings. sentry_dsn = Gitlab.config.sentry.clientside_dsn sentry_uri = URI(sentry_dsn) append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") end + def self.allow_sentry(directives) + sentry_dsn = Gitlab::CurrentSettings.sentry_clientside_dsn + sentry_uri = URI(sentry_dsn) + + append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") + end + def self.allow_letter_opener(directives) append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, '/rails/letter_opener/')) end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index a45380aca6c..2068a9ae7d5 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -31,7 +31,7 @@ module Gitlab repo_events = events_created_between(start_time, end_time, :repository) .where(action: :pushed) issue_events = events_created_between(start_time, end_time, :issues) - .where(action: [:created, :closed], target_type: "Issue") + .where(action: [:created, :closed], target_type: %w[Issue WorkItem]) mr_events = events_created_between(start_time, end_time, :merge_requests) .where(action: [:merged, :created, :closed], target_type: "MergeRequest") note_events = events_created_between(start_time, end_time, :merge_requests) diff --git a/lib/gitlab/counters/buffered_counter.rb b/lib/gitlab/counters/buffered_counter.rb new file mode 100644 index 00000000000..56593b642a9 --- /dev/null +++ b/lib/gitlab/counters/buffered_counter.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +module Gitlab + module Counters + class BufferedCounter + include Gitlab::ExclusiveLeaseHelpers + + WORKER_DELAY = 10.minutes + WORKER_LOCK_TTL = 10.minutes + + LUA_FLUSH_INCREMENT_SCRIPT = <<~LUA + local increment_key, flushed_key = KEYS[1], KEYS[2] + local increment_value = redis.call("get", increment_key) or 0 + local flushed_value = redis.call("incrby", flushed_key, increment_value) + if flushed_value == 0 then + redis.call("del", increment_key, flushed_key) + else + redis.call("del", increment_key) + end + return flushed_value + LUA + + def initialize(counter_record, attribute) + @counter_record = counter_record + @attribute = attribute + end + + def get + redis_state do |redis| + redis.get(key).to_i + end + end + + def increment(amount) + result = redis_state do |redis| + redis.incrby(key, amount) + end + + FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) + + result + end + + def reset! + counter_record.update!(attribute => 0) + + redis_state do |redis| + redis.del(key) + end + end + + def commit_increment! + with_exclusive_lease do + flush_amount = amount_to_be_flushed + next if flush_amount == 0 + + counter_record.transaction do + counter_record.update_counters_with_lease({ attribute => flush_amount }) + remove_flushed_key + end + + counter_record.execute_after_commit_callbacks + end + + counter_record.reset.read_attribute(attribute) + end + + # amount_to_be_flushed returns the total value to be flushed. + # The total value is the sum of the following: + # - current value in the increment_key + # - any existing value in the flushed_key that has not been flushed + def amount_to_be_flushed + redis_state do |redis| + redis.eval(LUA_FLUSH_INCREMENT_SCRIPT, keys: [key, flushed_key]) + end + end + + def key + project_id = counter_record.project.id + record_name = counter_record.class + record_id = counter_record.id + + "project:{#{project_id}}:counters:#{record_name}:#{record_id}:#{attribute}" + end + + def flushed_key + "#{key}:flushed" + end + + private + + attr_reader :counter_record, :attribute + + def remove_flushed_key + redis_state do |redis| + redis.del(flushed_key) + end + end + + def redis_state(&block) + Gitlab::Redis::SharedState.with(&block) + end + + def with_exclusive_lease(&block) + lock_key = "#{key}:locked" + + in_lock(lock_key, ttl: WORKER_LOCK_TTL, &block) + rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError + # a worker is already updating the counters + end + end + end +end diff --git a/lib/gitlab/counters/legacy_counter.rb b/lib/gitlab/counters/legacy_counter.rb new file mode 100644 index 00000000000..06951514ec3 --- /dev/null +++ b/lib/gitlab/counters/legacy_counter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Counters + # This class is a wrapper over ActiveRecord counter + # for attributes that have not adopted Redis-backed BufferedCounter. + class LegacyCounter + def initialize(counter_record, attribute) + @counter_record = counter_record + @attribute = attribute + @current_value = counter_record.method(attribute).call + end + + def increment(amount) + updated = counter_record.class.update_counters(counter_record.id, { attribute => amount }) + + if updated == 1 + counter_record.execute_after_commit_callbacks + @current_value += amount + end + + @current_value + end + + def reset! + counter_record.update!(attribute => 0) + end + + private + + attr_reader :counter_record, :attribute + end + end +end diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index a9c69e3f997..7055f64937d 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -35,6 +35,8 @@ module Gitlab deployable_id: deployment.deployable_id, deployable_url: deployable_url, environment: deployment.environment.name, + environment_slug: deployment.environment.slug, + environment_external_url: deployment.environment.external_url, project: deployment.project.hook_attrs, short_sha: deployment.short_sha, user: deployment.deployed_by&.hook_attrs, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 04cf056199c..51d5bfcee38 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -101,7 +101,8 @@ module Gitlab gitlab_main: [self.database_base_models.fetch(:main)], gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main gitlab_shared: database_base_models_with_gitlab_shared.values, # all models - gitlab_internal: database_base_models.values # all models + gitlab_internal: database_base_models.values, # all models + gitlab_pm: [self.database_base_models.fetch(:main)] # package metadata models }.with_indifferent_access.freeze end diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index d68be19047e..4b4a9b38fd8 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -157,7 +157,7 @@ module Gitlab def self.execute(columns, mapping, &to_class) raise ArgumentError if mapping.blank? - entries_by_class = mapping.group_by { |k, v| to_class ? to_class.call(k) : k.class } + entries_by_class = mapping.group_by { |k, v| to_class ? yield(k) : k.class } entries_by_class.each do |model, entries| Setter.new(model, columns, entries).update! diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index 345c7e44b05..11c83786aa4 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -18,9 +18,7 @@ module Gitlab end def count - models.each_with_object({}) do |model, data| - data[model] = model.count - end + models.index_with(&:count) rescue *CONNECTION_ERRORS {} end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 365a4283d4c..0f848ed40fb 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -6,14 +6,16 @@ # Each table / view needs to have assigned gitlab_schema. Names supported today: # # - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection) -# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given database +# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given application database +# - gitlab_geo - defines a set of tables that can only exist on the geo database +# - gitlab_internal - defines all internal tables of Rails and PostgreSQL # # Tables for the purpose of tests should be prefixed with `_test_my_table_name` module Gitlab module Database module GitlabSchema - GITLAB_SCHEMAS_FILE = 'lib/gitlab/database/gitlab_schemas.yml' + DICTIONARY_PATH = 'db/docs/' # These tables are deleted/renamed, but still referenced by migrations. # This is needed for now, but should be removed in the future @@ -55,7 +57,7 @@ module Gitlab tables.map { |table| table_schema(table) }.to_set end - def self.table_schema(name) + def self.table_schema(name, undefined: true) schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.` # Most of names do not have schemas, ensure that this is table @@ -68,7 +70,7 @@ module Gitlab table_name.gsub!(/_[0-9]+$/, '') # Tables that are properly mapped - if gitlab_schema = tables_to_schema[table_name] + if gitlab_schema = views_and_tables_to_schema[table_name] return gitlab_schema end @@ -84,6 +86,8 @@ module Gitlab return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_') + return :gitlab_geo if table_name.start_with?('_test_gitlab_geo_') + # All tables that start with `_test_` without a following schema are shared and ignored return :gitlab_shared if table_name.start_with?('_test_') @@ -91,15 +95,39 @@ module Gitlab return :gitlab_internal if table_name.start_with?('pg_') # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database - :"undefined_#{table_name}" + undefined ? :"undefined_#{table_name}" : nil + end + + def self.dictionary_path_globs + [Rails.root.join(DICTIONARY_PATH, '*.yml')] + end + + def self.view_path_globs + [Rails.root.join(DICTIONARY_PATH, 'views', '*.yml')] + end + + def self.views_and_tables_to_schema + @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end def self.tables_to_schema - @tables_to_schema ||= YAML.load_file(Rails.root.join(GITLAB_SCHEMAS_FILE)) + @tables_to_schema ||= Dir.glob(self.dictionary_path_globs).each_with_object({}) do |file_path, dic| + data = YAML.load_file(file_path) + + dic[data['table_name']] = data['gitlab_schema'].to_sym + end + end + + def self.views_to_schema + @views_to_schema ||= Dir.glob(self.view_path_globs).each_with_object({}) do |file_path, dic| + data = YAML.load_file(file_path) + + dic[data['view_name']] = data['gitlab_schema'].to_sym + end end def self.schema_names - @schema_names ||= self.tables_to_schema.values.to_set + @schema_names ||= self.views_and_tables_to_schema.values.to_set end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml deleted file mode 100644 index bf6ebb21f7d..00000000000 --- a/lib/gitlab/database/gitlab_schemas.yml +++ /dev/null @@ -1,606 +0,0 @@ -abuse_reports: :gitlab_main -agent_activity_events: :gitlab_main -agent_group_authorizations: :gitlab_main -agent_project_authorizations: :gitlab_main -alert_management_alert_assignees: :gitlab_main -alert_management_alerts: :gitlab_main -alert_management_alert_metric_images: :gitlab_main -alert_management_alert_user_mentions: :gitlab_main -alert_management_http_integrations: :gitlab_main -allowed_email_domains: :gitlab_main -analytics_cycle_analytics_aggregations: :gitlab_main -analytics_cycle_analytics_group_stages: :gitlab_main -analytics_cycle_analytics_group_value_streams: :gitlab_main -analytics_cycle_analytics_issue_stage_events: :gitlab_main -analytics_cycle_analytics_merge_request_stage_events: :gitlab_main -analytics_cycle_analytics_project_stages: :gitlab_main -analytics_cycle_analytics_project_value_streams: :gitlab_main -analytics_cycle_analytics_stage_event_hashes: :gitlab_main -analytics_devops_adoption_segments: :gitlab_main -analytics_devops_adoption_snapshots: :gitlab_main -analytics_language_trend_repository_languages: :gitlab_main -analytics_usage_trends_measurements: :gitlab_main -appearances: :gitlab_main -application_settings: :gitlab_main -application_setting_terms: :gitlab_main -approval_merge_request_rules_approved_approvers: :gitlab_main -approval_merge_request_rules: :gitlab_main -approval_merge_request_rules_groups: :gitlab_main -approval_merge_request_rule_sources: :gitlab_main -approval_merge_request_rules_users: :gitlab_main -approval_project_rules: :gitlab_main -approval_project_rules_groups: :gitlab_main -approval_project_rules_protected_branches: :gitlab_main -approval_project_rules_users: :gitlab_main -approvals: :gitlab_main -approver_groups: :gitlab_main -approvers: :gitlab_main -ar_internal_metadata: :gitlab_internal -atlassian_identities: :gitlab_main -audit_events_external_audit_event_destinations: :gitlab_main -audit_events: :gitlab_main -audit_events_streaming_headers: :gitlab_main -audit_events_streaming_event_type_filters: :gitlab_main -authentication_events: :gitlab_main -award_emoji: :gitlab_main -aws_roles: :gitlab_main -background_migration_jobs: :gitlab_shared -badges: :gitlab_main -banned_users: :gitlab_main -batched_background_migration_jobs: :gitlab_shared -batched_background_migrations: :gitlab_shared -board_assignees: :gitlab_main -board_group_recent_visits: :gitlab_main -board_labels: :gitlab_main -board_project_recent_visits: :gitlab_main -boards_epic_board_labels: :gitlab_main -boards_epic_board_positions: :gitlab_main -boards_epic_board_recent_visits: :gitlab_main -boards_epic_boards: :gitlab_main -boards_epic_lists: :gitlab_main -boards_epic_list_user_preferences: :gitlab_main -boards_epic_user_preferences: :gitlab_main -boards: :gitlab_main -board_user_preferences: :gitlab_main -broadcast_messages: :gitlab_main -bulk_import_configurations: :gitlab_main -bulk_import_entities: :gitlab_main -bulk_import_exports: :gitlab_main -bulk_import_export_uploads: :gitlab_main -bulk_import_failures: :gitlab_main -bulk_imports: :gitlab_main -bulk_import_trackers: :gitlab_main -chat_names: :gitlab_main -chat_teams: :gitlab_main -ci_build_needs: :gitlab_ci -ci_build_pending_states: :gitlab_ci -ci_build_report_results: :gitlab_ci -ci_builds: :gitlab_ci -ci_builds_metadata: :gitlab_ci -ci_builds_runner_session: :gitlab_ci -ci_build_trace_chunks: :gitlab_ci -ci_build_trace_metadata: :gitlab_ci -ci_daily_build_group_report_results: :gitlab_ci -ci_deleted_objects: :gitlab_ci -ci_freeze_periods: :gitlab_ci -ci_group_variables: :gitlab_ci -ci_instance_variables: :gitlab_ci -ci_job_artifacts: :gitlab_ci -ci_job_token_project_scope_links: :gitlab_ci -ci_job_variables: :gitlab_ci -ci_job_artifact_states: :gitlab_ci -ci_minutes_additional_packs: :gitlab_ci -ci_namespace_monthly_usages: :gitlab_ci -ci_namespace_mirrors: :gitlab_ci -ci_partitions: :gitlab_ci -ci_pending_builds: :gitlab_ci -ci_pipeline_artifacts: :gitlab_ci -ci_pipeline_chat_data: :gitlab_ci -ci_pipeline_messages: :gitlab_ci -ci_pipeline_schedules: :gitlab_ci -ci_pipeline_schedule_variables: :gitlab_ci -ci_pipelines_config: :gitlab_ci -ci_pipeline_metadata: :gitlab_ci -ci_pipelines: :gitlab_ci -ci_pipeline_variables: :gitlab_ci -ci_platform_metrics: :gitlab_ci -ci_project_monthly_usages: :gitlab_ci -ci_project_mirrors: :gitlab_ci -ci_refs: :gitlab_ci -ci_resource_groups: :gitlab_ci -ci_resources: :gitlab_ci -ci_runner_namespaces: :gitlab_ci -ci_runner_projects: :gitlab_ci -ci_runner_versions: :gitlab_ci -ci_runners: :gitlab_ci -ci_running_builds: :gitlab_ci -ci_sources_pipelines: :gitlab_ci -ci_secure_files: :gitlab_ci -ci_secure_file_states: :gitlab_ci -ci_sources_projects: :gitlab_ci -ci_stages: :gitlab_ci -ci_subscriptions_projects: :gitlab_ci -ci_trigger_requests: :gitlab_ci -ci_triggers: :gitlab_ci -ci_unit_test_failures: :gitlab_ci -ci_unit_tests: :gitlab_ci -ci_variables: :gitlab_ci -cluster_agents: :gitlab_main -cluster_agent_tokens: :gitlab_main -cluster_enabled_grants: :gitlab_main -cluster_groups: :gitlab_main -cluster_platforms_kubernetes: :gitlab_main -cluster_projects: :gitlab_main -cluster_providers_aws: :gitlab_main -cluster_providers_gcp: :gitlab_main -clusters_applications_cert_managers: :gitlab_main -clusters_applications_cilium: :gitlab_main -clusters_applications_crossplane: :gitlab_main -clusters_applications_helm: :gitlab_main -clusters_applications_ingress: :gitlab_main -clusters_applications_jupyter: :gitlab_main -clusters_applications_knative: :gitlab_main -clusters_applications_prometheus: :gitlab_main -clusters_applications_runners: :gitlab_main -clusters: :gitlab_main -clusters_integration_prometheus: :gitlab_main -clusters_kubernetes_namespaces: :gitlab_main -commit_user_mentions: :gitlab_main -compliance_management_frameworks: :gitlab_main -container_expiration_policies: :gitlab_main -container_repositories: :gitlab_main -content_blocked_states: :gitlab_main -conversational_development_index_metrics: :gitlab_main -coverage_fuzzing_corpuses: :gitlab_main -csv_issue_imports: :gitlab_main -custom_emoji: :gitlab_main -customer_relations_contacts: :gitlab_main -customer_relations_organizations: :gitlab_main -dast_profile_schedules: :gitlab_main -dast_profiles: :gitlab_main -dast_profiles_pipelines: :gitlab_main -dast_scanner_profiles_builds: :gitlab_main -dast_scanner_profiles: :gitlab_main -dast_site_profiles_builds: :gitlab_main -dast_site_profile_secret_variables: :gitlab_main -dast_site_profiles: :gitlab_main -dast_site_profiles_pipelines: :gitlab_main -dast_sites: :gitlab_main -dast_site_tokens: :gitlab_main -dast_site_validations: :gitlab_main -dependency_proxy_blob_states: :gitlab_main -dependency_proxy_blobs: :gitlab_main -dependency_proxy_group_settings: :gitlab_main -dependency_proxy_image_ttl_group_policies: :gitlab_main -dependency_proxy_manifests: :gitlab_main -deploy_keys_projects: :gitlab_main -deployment_approvals: :gitlab_main -deployment_clusters: :gitlab_main -deployment_merge_requests: :gitlab_main -deployments: :gitlab_main -deploy_tokens: :gitlab_main -description_versions: :gitlab_main -design_management_designs: :gitlab_main -design_management_designs_versions: :gitlab_main -design_management_versions: :gitlab_main -design_user_mentions: :gitlab_main -detached_partitions: :gitlab_shared -diff_note_positions: :gitlab_main -dora_configurations: :gitlab_main -dora_daily_metrics: :gitlab_main -draft_notes: :gitlab_main -elastic_index_settings: :gitlab_main -elastic_reindexing_slices: :gitlab_main -elastic_reindexing_subtasks: :gitlab_main -elastic_reindexing_tasks: :gitlab_main -elasticsearch_indexed_namespaces: :gitlab_main -elasticsearch_indexed_projects: :gitlab_main -emails: :gitlab_main -environments: :gitlab_main -epic_issues: :gitlab_main -epic_metrics: :gitlab_main -epics: :gitlab_main -epic_user_mentions: :gitlab_main -error_tracking_client_keys: :gitlab_main -error_tracking_error_events: :gitlab_main -error_tracking_errors: :gitlab_main -events: :gitlab_main -evidences: :gitlab_main -experiments: :gitlab_main -experiment_subjects: :gitlab_main -external_approval_rules: :gitlab_main -external_approval_rules_protected_branches: :gitlab_main -external_pull_requests: :gitlab_ci -external_status_checks: :gitlab_main -external_status_checks_protected_branches: :gitlab_main -feature_gates: :gitlab_main -features: :gitlab_main -fork_network_members: :gitlab_main -fork_networks: :gitlab_main -geo_cache_invalidation_events: :gitlab_main -geo_container_repository_updated_events: :gitlab_main -geo_event_log: :gitlab_main -geo_events: :gitlab_main -geo_hashed_storage_attachments_events: :gitlab_main -geo_hashed_storage_migrated_events: :gitlab_main -geo_node_namespace_links: :gitlab_main -geo_nodes: :gitlab_main -geo_node_statuses: :gitlab_main -geo_repositories_changed_events: :gitlab_main -geo_repository_created_events: :gitlab_main -geo_repository_deleted_events: :gitlab_main -geo_repository_renamed_events: :gitlab_main -geo_repository_updated_events: :gitlab_main -geo_reset_checksum_events: :gitlab_main -ghost_user_migrations: :gitlab_main -gitlab_subscription_histories: :gitlab_main -gitlab_subscriptions: :gitlab_main -gpg_keys: :gitlab_main -gpg_key_subkeys: :gitlab_main -gpg_signatures: :gitlab_main -grafana_integrations: :gitlab_main -group_custom_attributes: :gitlab_main -group_crm_settings: :gitlab_main -group_deletion_schedules: :gitlab_main -group_deploy_keys: :gitlab_main -group_deploy_keys_groups: :gitlab_main -group_deploy_tokens: :gitlab_main -group_features: :gitlab_main -group_group_links: :gitlab_main -group_import_states: :gitlab_main -group_merge_request_approval_settings: :gitlab_main -group_repository_storage_moves: :gitlab_main -group_wiki_repositories: :gitlab_main -historical_data: :gitlab_main -identities: :gitlab_main -import_export_uploads: :gitlab_main -import_failures: :gitlab_main -incident_management_escalation_policies: :gitlab_main -incident_management_escalation_rules: :gitlab_main -incident_management_issuable_escalation_statuses: :gitlab_main -incident_management_oncall_participants: :gitlab_main -incident_management_oncall_rotations: :gitlab_main -incident_management_oncall_schedules: :gitlab_main -incident_management_oncall_shifts: :gitlab_main -incident_management_pending_alert_escalations: :gitlab_main -incident_management_pending_issue_escalations: :gitlab_main -incident_management_timeline_events: :gitlab_main -incident_management_timeline_event_tags: :gitlab_main -incident_management_timeline_event_tag_links: :gitlab_main -index_statuses: :gitlab_main -in_product_marketing_emails: :gitlab_main -insights: :gitlab_main -integrations: :gitlab_main -internal_ids: :gitlab_main -ip_restrictions: :gitlab_main -issuable_metric_images: :gitlab_main -issuable_resource_links: :gitlab_main -issuable_severities: :gitlab_main -issuable_slas: :gitlab_main -issue_assignees: :gitlab_main -issue_customer_relations_contacts: :gitlab_main -issue_emails: :gitlab_main -issue_email_participants: :gitlab_main -issue_links: :gitlab_main -issue_metrics: :gitlab_main -issue_search_data: :gitlab_main -issues: :gitlab_main -issues_prometheus_alert_events: :gitlab_main -issues_self_managed_prometheus_alert_events: :gitlab_main -issue_tracker_data: :gitlab_main -issue_user_mentions: :gitlab_main -iterations_cadences: :gitlab_main -jira_connect_installations: :gitlab_main -jira_connect_subscriptions: :gitlab_main -jira_imports: :gitlab_main -jira_tracker_data: :gitlab_main -keys: :gitlab_main -label_links: :gitlab_main -label_priorities: :gitlab_main -labels: :gitlab_main -ldap_group_links: :gitlab_main -lfs_file_locks: :gitlab_main -lfs_objects: :gitlab_main -lfs_objects_projects: :gitlab_main -lfs_object_states: :gitlab_main -licenses: :gitlab_main -lists: :gitlab_main -list_user_preferences: :gitlab_main -loose_foreign_keys_deleted_records: :gitlab_shared -member_roles: :gitlab_main -member_tasks: :gitlab_main -members: :gitlab_main -merge_request_assignees: :gitlab_main -merge_request_blocks: :gitlab_main -merge_request_cleanup_schedules: :gitlab_main -merge_requests_compliance_violations: :gitlab_main -merge_request_context_commit_diff_files: :gitlab_main -merge_request_context_commits: :gitlab_main -merge_request_diff_commits: :gitlab_main -merge_request_diff_commit_users: :gitlab_main -merge_request_diff_details: :gitlab_main -merge_request_diff_files: :gitlab_main -merge_request_diffs: :gitlab_main -merge_request_metrics: :gitlab_main -merge_request_predictions: :gitlab_main -merge_request_reviewers: :gitlab_main -merge_requests_closing_issues: :gitlab_main -merge_requests: :gitlab_main -merge_request_user_mentions: :gitlab_main -merge_trains: :gitlab_main -metrics_dashboard_annotations: :gitlab_main -metrics_users_starred_dashboards: :gitlab_main -milestone_releases: :gitlab_main -milestones: :gitlab_main -ml_candidates: :gitlab_main -ml_experiments: :gitlab_main -ml_candidate_metrics: :gitlab_main -ml_candidate_params: :gitlab_main -namespace_admin_notes: :gitlab_main -namespace_aggregation_schedules: :gitlab_main -namespace_bans: :gitlab_main -namespace_limits: :gitlab_main -namespace_package_settings: :gitlab_main -namespace_root_storage_statistics: :gitlab_main -namespace_ci_cd_settings: :gitlab_main -namespace_commit_emails: :gitlab_main -namespace_settings: :gitlab_main -namespace_details: :gitlab_main -namespaces: :gitlab_main -namespaces_sync_events: :gitlab_main -namespace_statistics: :gitlab_main -note_diff_files: :gitlab_main -notes: :gitlab_main -notification_settings: :gitlab_main -oauth_access_grants: :gitlab_main -oauth_access_tokens: :gitlab_main -oauth_applications: :gitlab_main -oauth_openid_requests: :gitlab_main -onboarding_progresses: :gitlab_main -operations_feature_flags_clients: :gitlab_main -operations_feature_flag_scopes: :gitlab_main -operations_feature_flags: :gitlab_main -operations_feature_flags_issues: :gitlab_main -operations_scopes: :gitlab_main -operations_strategies: :gitlab_main -operations_strategies_user_lists: :gitlab_main -operations_user_lists: :gitlab_main -p_ci_builds_metadata: :gitlab_ci -packages_build_infos: :gitlab_main -packages_cleanup_policies: :gitlab_main -packages_composer_cache_files: :gitlab_main -packages_composer_metadata: :gitlab_main -packages_conan_file_metadata: :gitlab_main -packages_conan_metadata: :gitlab_main -packages_debian_file_metadata: :gitlab_main -packages_debian_group_architectures: :gitlab_main -packages_debian_group_component_files: :gitlab_main -packages_debian_group_components: :gitlab_main -packages_debian_group_distribution_keys: :gitlab_main -packages_debian_group_distributions: :gitlab_main -packages_debian_project_architectures: :gitlab_main -packages_debian_project_component_files: :gitlab_main -packages_debian_project_components: :gitlab_main -packages_debian_project_distribution_keys: :gitlab_main -packages_debian_project_distributions: :gitlab_main -packages_debian_publications: :gitlab_main -packages_dependencies: :gitlab_main -packages_dependency_links: :gitlab_main -packages_events: :gitlab_main -packages_helm_file_metadata: :gitlab_main -packages_maven_metadata: :gitlab_main -packages_npm_metadata: :gitlab_main -packages_rpm_metadata: :gitlab_main -packages_nuget_dependency_link_metadata: :gitlab_main -packages_nuget_metadata: :gitlab_main -packages_package_file_build_infos: :gitlab_main -packages_package_files: :gitlab_main -packages_rpm_repository_files: :gitlab_main -packages_packages: :gitlab_main -packages_pypi_metadata: :gitlab_main -packages_rubygems_metadata: :gitlab_main -packages_tags: :gitlab_main -pages_deployments: :gitlab_main -pages_deployment_states: :gitlab_main -pages_domain_acme_orders: :gitlab_main -pages_domains: :gitlab_main -path_locks: :gitlab_main -personal_access_tokens: :gitlab_main -plan_limits: :gitlab_main -plans: :gitlab_main -pool_repositories: :gitlab_main -postgres_async_indexes: :gitlab_shared -postgres_autovacuum_activity: :gitlab_shared -postgres_constraints: :gitlab_shared -postgres_foreign_keys: :gitlab_shared -postgres_index_bloat_estimates: :gitlab_shared -postgres_indexes: :gitlab_shared -postgres_partitioned_tables: :gitlab_shared -postgres_partitions: :gitlab_shared -postgres_reindex_actions: :gitlab_shared -postgres_reindex_queued_actions: :gitlab_shared -product_analytics_events_experimental: :gitlab_main -programming_languages: :gitlab_main -project_access_tokens: :gitlab_main -project_alerting_settings: :gitlab_main -project_aliases: :gitlab_main -project_authorizations: :gitlab_main -project_auto_devops: :gitlab_main -project_build_artifacts_size_refreshes: :gitlab_main -project_ci_cd_settings: :gitlab_main -project_ci_feature_usages: :gitlab_main -project_compliance_framework_settings: :gitlab_main -project_custom_attributes: :gitlab_main -project_daily_statistics: :gitlab_main -project_deploy_tokens: :gitlab_main -project_error_tracking_settings: :gitlab_main -project_export_jobs: :gitlab_main -project_features: :gitlab_main -project_feature_usages: :gitlab_main -project_group_links: :gitlab_main -project_import_data: :gitlab_main -project_incident_management_settings: :gitlab_main -project_metrics_settings: :gitlab_main -project_mirror_data: :gitlab_main -project_pages_metadata: :gitlab_main -project_relation_export_uploads: :gitlab_main -project_relation_exports: :gitlab_main -project_repositories: :gitlab_main -project_repository_states: :gitlab_main -project_repository_storage_moves: :gitlab_main -project_security_settings: :gitlab_main -project_settings: :gitlab_main -projects: :gitlab_main -projects_sync_events: :gitlab_main -project_statistics: :gitlab_main -project_topics: :gitlab_main -project_wiki_repositories: :gitlab_main -project_wiki_repository_states: :gitlab_main -prometheus_alert_events: :gitlab_main -prometheus_alerts: :gitlab_main -prometheus_metrics: :gitlab_main -protected_branches: :gitlab_main -protected_branch_merge_access_levels: :gitlab_main -protected_branch_push_access_levels: :gitlab_main -protected_branch_unprotect_access_levels: :gitlab_main -protected_environment_approval_rules: :gitlab_main -protected_environment_deploy_access_levels: :gitlab_main -protected_environments: :gitlab_main -protected_tag_create_access_levels: :gitlab_main -protected_tags: :gitlab_main -push_event_payloads: :gitlab_main -push_rules: :gitlab_main -raw_usage_data: :gitlab_main -redirect_routes: :gitlab_main -related_epic_links: :gitlab_main -release_links: :gitlab_main -releases: :gitlab_main -remote_mirrors: :gitlab_main -repository_languages: :gitlab_main -required_code_owners_sections: :gitlab_main -requirements: :gitlab_main -requirements_management_test_reports: :gitlab_main -resource_iteration_events: :gitlab_main -resource_label_events: :gitlab_main -resource_milestone_events: :gitlab_main -resource_state_events: :gitlab_main -resource_weight_events: :gitlab_main -reviews: :gitlab_main -routes: :gitlab_main -saml_group_links: :gitlab_main -saml_providers: :gitlab_main -saved_replies: :gitlab_main -sbom_components: :gitlab_main -sbom_occurrences: :gitlab_main -sbom_component_versions: :gitlab_main -sbom_sources: :gitlab_main -sbom_vulnerable_component_versions: :gitlab_main -schema_migrations: :gitlab_internal -scim_identities: :gitlab_main -scim_oauth_access_tokens: :gitlab_main -security_findings: :gitlab_main -security_orchestration_policy_configurations: :gitlab_main -security_orchestration_policy_rule_schedules: :gitlab_main -security_scans: :gitlab_main -security_training_providers: :gitlab_main -security_trainings: :gitlab_main -self_managed_prometheus_alert_events: :gitlab_main -sent_notifications: :gitlab_main -sentry_issues: :gitlab_main -serverless_domain_cluster: :gitlab_main -service_desk_settings: :gitlab_main -shards: :gitlab_main -slack_integrations: :gitlab_main -smartcard_identities: :gitlab_main -snippet_repositories: :gitlab_main -snippet_repository_storage_moves: :gitlab_main -snippets: :gitlab_main -snippet_statistics: :gitlab_main -snippet_user_mentions: :gitlab_main -software_license_policies: :gitlab_main -software_licenses: :gitlab_main -spam_logs: :gitlab_main -sprints: :gitlab_main -ssh_signatures: :gitlab_main -status_check_responses: :gitlab_main -status_page_published_incidents: :gitlab_main -status_page_settings: :gitlab_main -subscriptions: :gitlab_main -suggestions: :gitlab_main -system_note_metadata: :gitlab_main -taggings: :gitlab_ci -tags: :gitlab_ci -term_agreements: :gitlab_main -terraform_states: :gitlab_main -terraform_state_versions: :gitlab_main -timelogs: :gitlab_main -timelog_categories: :gitlab_main -todos: :gitlab_main -token_with_ivs: :gitlab_main -topics: :gitlab_main -trending_projects: :gitlab_main -u2f_registrations: :gitlab_main -upcoming_reconciliations: :gitlab_main -uploads: :gitlab_main -upload_states: :gitlab_main -user_agent_details: :gitlab_main -user_callouts: :gitlab_main -user_canonical_emails: :gitlab_main -user_credit_card_validations: :gitlab_main -user_custom_attributes: :gitlab_main -user_details: :gitlab_main -user_follow_users: :gitlab_main -user_group_callouts: :gitlab_main -user_project_callouts: :gitlab_main -user_highest_roles: :gitlab_main -user_interacted_projects: :gitlab_main -user_phone_number_validations: :gitlab_main -user_permission_export_uploads: :gitlab_main -user_preferences: :gitlab_main -users: :gitlab_main -users_ops_dashboard_projects: :gitlab_main -users_security_dashboard_projects: :gitlab_main -users_star_projects: :gitlab_main -users_statistics: :gitlab_main -user_statuses: :gitlab_main -user_synced_attributes_metadata: :gitlab_main -verification_codes: :gitlab_main -vulnerabilities: :gitlab_main -vulnerability_advisories: :gitlab_main -vulnerability_exports: :gitlab_main -vulnerability_external_issue_links: :gitlab_main -vulnerability_feedback: :gitlab_main -vulnerability_finding_evidences: :gitlab_main -vulnerability_finding_links: :gitlab_main -vulnerability_finding_signatures: :gitlab_main -vulnerability_findings_remediations: :gitlab_main -vulnerability_flags: :gitlab_main -vulnerability_historical_statistics: :gitlab_main -vulnerability_identifiers: :gitlab_main -vulnerability_issue_links: :gitlab_main -vulnerability_merge_request_links: :gitlab_main -vulnerability_occurrence_identifiers: :gitlab_main -vulnerability_occurrence_pipelines: :gitlab_main -vulnerability_occurrences: :gitlab_main -vulnerability_reads: :gitlab_main -vulnerability_remediations: :gitlab_main -vulnerability_scanners: :gitlab_main -vulnerability_state_transitions: :gitlab_main -vulnerability_statistics: :gitlab_main -vulnerability_user_mentions: :gitlab_main -webauthn_registrations: :gitlab_main -web_hook_logs: :gitlab_main -web_hooks: :gitlab_main -wiki_page_meta: :gitlab_main -wiki_page_slugs: :gitlab_main -work_item_parent_links: :gitlab_main -work_item_types: :gitlab_main -x509_certificates: :gitlab_main -x509_commit_signatures: :gitlab_main -x509_issuers: :gitlab_main -zentao_tracker_data: :gitlab_main -# dingtalk_tracker_data JiHu-specific, see https://jihulab.com/gitlab-cn/gitlab/-/merge_requests/417 -dingtalk_tracker_data: :gitlab_main -zoom_meetings: :gitlab_main -batched_background_migration_job_transition_logs: :gitlab_shared -user_namespace_callouts: :gitlab_main diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 8799f8d8af8..f0343f9d8b5 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -95,7 +95,7 @@ module Gitlab # name - The name of the method to call on a connection object. def read_using_load_balancer(...) if current_session.use_primary? && - !current_session.use_replicas_for_read_queries? + !current_session.use_replicas_for_read_queries? @load_balancer.read_write do |connection| connection.send(...) end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 52a9e8798d4..3295301a2d7 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -125,13 +125,6 @@ module Gitlab old_host_list_length: current.length ) replace_hosts(from_dns) - else - ::Gitlab::Database::LoadBalancing::Logger.info( - event: :host_list_unchanged, - message: "Unchanged host list for service discovery", - host_list_length: from_dns.length, - old_host_list_length: current.length - ) end interval diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 13afbd8fd37..619f11ae890 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -57,7 +57,7 @@ module Gitlab if uses_primary? load_balancer.primary_write_location else - load_balancer.host.database_replica_location + load_balancer.host&.database_replica_location || load_balancer.primary_write_location end end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 737852d5ccb..f7b8d2514ba 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -97,14 +97,8 @@ module Gitlab end def databases_in_sync?(wal_locations) - locations = if Feature.enabled?(:indifferent_wal_location_keys) - wal_locations.with_indifferent_access - else - wal_locations - end - ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| - if (location = locations[lb.name]) + if (location = wal_locations.with_indifferent_access[lb.name]) lb.select_up_to_date_host(location) else # If there's no entry for a load balancer it means the Sidekiq diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 2594ee04b35..e3ae2892668 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -10,18 +10,34 @@ module Gitlab # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us EXPECTED_TRIGGER_RECORD_COUNT = 3 + def self.tables_to_lock(connection) + Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + yield table_name, schema_name + end + + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + yield detached_partition.fully_qualified_table_name, detached_partition.table_schema + end + end + end + def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) @table_name = table_name @connection = connection @database_name = database_name @logger = logger @dry_run = dry_run + + @table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table_name) + .identifier end def table_locked_for_writes?(table_name) query = <<~SQL SELECT COUNT(*) from information_schema.triggers - WHERE event_object_table = '#{table_name}' + WHERE event_object_table = '#{table_name_without_schema}' AND trigger_name = '#{write_trigger_name(table_name)}' SQL @@ -56,7 +72,7 @@ module Gitlab private - attr_reader :table_name, :connection, :database_name, :logger, :dry_run + attr_reader :table_name, :connection, :database_name, :logger, :dry_run, :table_name_without_schema def execute_sql_statement(sql) if dry_run @@ -99,7 +115,7 @@ module Gitlab end def write_trigger_name(table_name) - "gitlab_schema_write_trigger_for_#{table_name}" + "gitlab_schema_write_trigger_for_#{table_name_without_schema}" end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index ab8b6988c3d..4d38920f571 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -51,6 +51,10 @@ module Gitlab include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema end + class V2_1 < V2_0 # rubocop:disable Naming/ClassAndModuleCamelCase + include Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables + end + def self.[](version) version = version.to_s name = "V#{version.tr('.', '_')}" @@ -61,7 +65,7 @@ module Gitlab # The current version to be used in new migrations def self.current_version - 2.0 + 2.1 end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 16416dd2507..4858a96c173 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -10,6 +10,7 @@ module Gitlab include Migrations::TimeoutHelpers include Migrations::ConstraintsHelpers include Migrations::ExtensionHelpers + include Migrations::SidekiqHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers @@ -497,17 +498,6 @@ module Gitlab end end - # Adds a column with a default value without locking an entire table. - # - # @deprecated With PostgreSQL 11, adding columns with a default does not lead to a table rewrite anymore. - # As such, this method is not needed anymore and the default `add_column` helper should be used. - # This helper is subject to be removed in a >13.0 release. - def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false) - raise 'Deprecated: add_column_with_default does not support being passed blocks anymore' if block_given? - - add_column(table, column, type, default: default, limit: limit, null: allow_null) - end - # Renames a column without requiring downtime. # # Concurrent renames work by using database triggers to ensure both the @@ -1027,38 +1017,6 @@ module Gitlab rescue ArgumentError end - # Remove any instances of deprecated job classes lingering in queues. - # - # rubocop:disable Cop/SidekiqApiUsage - def sidekiq_remove_jobs(job_klass:) - Sidekiq::Queue.new(job_klass.queue).each do |job| - job.delete if job.klass == job_klass.to_s - end - - Sidekiq::RetrySet.new.each do |retri| - retri.delete if retri.klass == job_klass.to_s - end - - Sidekiq::ScheduledSet.new.each do |scheduled| - scheduled.delete if scheduled.klass == job_klass.to_s - end - end - # rubocop:enable Cop/SidekiqApiUsage - - def sidekiq_queue_migrate(queue_from, to:) - while sidekiq_queue_length(queue_from) > 0 - Sidekiq.redis do |conn| - conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" - end - end - end - - def sidekiq_queue_length(queue_name) - Sidekiq.redis do |conn| - conn.llen("queue:#{queue_name}") - end - end - def check_trigger_permissions!(table) unless Grant.create_and_execute_trigger?(table) dbname = ApplicationRecord.database.database_name diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb new file mode 100644 index 00000000000..0aa4b0d01c4 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module AutomaticLockWritesOnTables + extend ActiveSupport::Concern + + included do + class_attribute :skip_automatic_lock_on_writes + end + + def exec_migration(connection, direction) + return super if %w[main ci].exclude?(Gitlab::Database.db_config_name(connection)) + return super if automatic_lock_on_writes_disabled? + + # This compares the tables only on the `public` schema. Partitions are not affected + tables = connection.tables + super + new_tables = connection.tables - tables + + new_tables.each do |table_name| + lock_writes_on_table(connection, table_name) if should_lock_writes_on_table?(table_name) + end + end + + private + + def automatic_lock_on_writes_disabled? + # Feature flags are set on the main database, see tables features/feature_gates. + # That is why we switch the ActiveRecord::Base.connection temporarily here back to the 'main' database + # for the cases when the migration is targeting another database, like the 'ci' database. + with_restored_connection_stack do |_| + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + skip_automatic_lock_on_writes || + Gitlab::Utils.to_boolean(ENV['SKIP_AUTOMATIC_LOCK_ON_WRITES']) || + Feature.disabled?(:automatic_lock_writes_on_table, type: :ops) + end + end + end + + def should_lock_writes_on_table?(table_name) + # currently gitlab_schema represents only present existing tables, this is workaround for deleted tables + # that should be skipped as they will be removed in a future migration. + return false if Gitlab::Database::GitlabSchema::DELETED_TABLES[table_name] + + table_schema = Gitlab::Database::GitlabSchema.table_schema(table_name.to_s, undefined: false) + + if table_schema.nil? + error_message = <<~ERROR + No gitlab_schema is defined for the table #{table_name}. Please consider + adding it to the database dictionary. + More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html + ERROR + raise error_message + end + + return false unless %i[gitlab_main gitlab_ci].include?(table_schema) + + Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) + end + + def lock_writes_on_table(connection, table_name) + database_name = Gitlab::Database.db_config_name(connection) + LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: database_name, + logger: Logger.new($stdout) + ).lock_writes + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/batched_migration_last_id.rb b/lib/gitlab/database/migrations/batched_migration_last_id.rb new file mode 100644 index 00000000000..c77a2e9a375 --- /dev/null +++ b/lib/gitlab/database/migrations/batched_migration_last_id.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class BatchedMigrationLastId + FILE_NAME = 'last-batched-background-migration-id.txt' + + def initialize(connection, base_dir) + @connection = connection + @base_dir = base_dir + end + + def store + File.open(file_path, 'wb') { |file| file.write(last_background_migration_id) } + end + + # Reads the last id from the file + # + # @info casts the file content into an +Integer+. + # Casts any unexpected content to +nil+ + # + # @example + # Integer('4', exception: false) # => 4 + # Integer('', exception: false) # => nil + # + # @return [Integer, nil] + def read + return unless File.exist?(file_path) + + Integer(File.read(file_path).presence, exception: false) + end + + private + + attr_reader :connection, :base_dir + + def file_path + @file_path ||= base_dir.join(FILE_NAME) + end + + def last_background_migration_id + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration.maximum(:id) + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 27b161419b2..ed55081c9ab 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -29,16 +29,14 @@ module Gitlab def batched_background_migrations(for_database:, legacy_mode: false) runner = nil - result_dir = if legacy_mode - BASE_RESULT_DIR.join('background_migrations') - else - BASE_RESULT_DIR.join(for_database.to_s, 'background_migrations') - end + result_dir = background_migrations_dir(for_database, legacy_mode) # Only one loop iteration since we pass `only:` here Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + from_id = batched_migrations_last_id(for_database).read + runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner - .new(result_dir: result_dir, connection: connection) + .new(result_dir: result_dir, connection: connection, from_id: from_id) end runner @@ -66,6 +64,18 @@ module Gitlab end # rubocop:enable Database/MultipleDatabases + def batched_migrations_last_id(for_database) + runner = nil + base_dir = background_migrations_dir(for_database, false) + + Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + runner = Gitlab::Database::Migrations::BatchedMigrationLastId + .new(connection, base_dir) + end + + runner + end + private def migrations_for_up(database) @@ -90,6 +100,12 @@ module Gitlab existing_versions.include?(migration.version) && versions_this_branch.include?(migration.version) end end + + def background_migrations_dir(db, legacy_mode) + return BASE_RESULT_DIR.join('background_migrations') if legacy_mode + + BASE_RESULT_DIR.join(db.to_s, 'background_migrations') + end end attr_reader :direction, :result_dir, :migrations diff --git a/lib/gitlab/database/migrations/sidekiq_helpers.rb b/lib/gitlab/database/migrations/sidekiq_helpers.rb new file mode 100644 index 00000000000..c536b33bbdf --- /dev/null +++ b/lib/gitlab/database/migrations/sidekiq_helpers.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + # rubocop:disable Cop/SidekiqApiUsage + # rubocop:disable Cop/SidekiqRedisCall + module SidekiqHelpers + # Constants for default sidekiq_remove_jobs values + DEFAULT_MAX_ATTEMPTS = 5 + DEFAULT_TIMES_IN_A_ROW = 2 + + # Probabilistically removes job_klasses from their specific queues, the + # retry set and the scheduled set. + # + # If jobs are still being processed at the same time, then there is a + # small chance it will not remove all instances of job_klass. To + # minimize this risk, it repeatedly removes matching jobs from each + # until nothing is removed twice in a row. + # + # Before calling this method, you should make sure that job_klass is no + # longer being scheduled within the running application. + def sidekiq_remove_jobs( + job_klasses:, + times_in_a_row: DEFAULT_TIMES_IN_A_ROW, + max_attempts: DEFAULT_MAX_ATTEMPTS + ) + + kwargs = { times_in_a_row: times_in_a_row, max_attempts: max_attempts } + + job_klasses_queues = job_klasses + .select { |job_klass| job_klass.to_s.safe_constantize.present? } + .map { |job_klass| job_klass.safe_constantize.queue } + .uniq + + job_klasses_queues.each do |queue| + delete_jobs_for( + set: Sidekiq::Queue.new(queue), + job_klasses: job_klasses, + kwargs: kwargs + ) + end + + delete_jobs_for( + set: Sidekiq::RetrySet.new, + kwargs: kwargs, + job_klasses: job_klasses + ) + + delete_jobs_for( + set: Sidekiq::ScheduledSet.new, + kwargs: kwargs, + job_klasses: job_klasses + ) + end + + def sidekiq_queue_migrate(queue_from, to:) + while sidekiq_queue_length(queue_from) > 0 + Sidekiq.redis do |conn| + conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" + end + end + end + + def sidekiq_queue_length(queue_name) + Sidekiq.redis do |conn| + conn.llen("queue:#{queue_name}") + end + end + + private + + # Handle the "jobs deleted" tracking that is needed in order to track + # whether a job was deleted or not. + def delete_jobs_for(set:, kwargs:, job_klasses:) + until_equal_to(0, **kwargs) do + set.count do |job| + job_klasses.include?(job.klass) && job.delete + end + end + end + + # Control how many times in a row you want to see a job deleted 0 + # times. The idea is that if you see 0 jobs deleted x number of times + # in a row you've *likely* covered the case in which the queue was + # mutating while this was running. + def until_equal_to(target, times_in_a_row:, max_attempts:) + streak = 0 + + result = { attempts: 0, success: false } + + 1.upto(max_attempts) do |current_attempt| + # yield's return value is a count of "jobs_deleted" + if yield == target + streak += 1 + elsif streak > 0 + streak = 0 + end + + result[:attempts] = current_attempt + result[:success] = streak == times_in_a_row + + break if result[:success] + end + result + end + end + # rubocop:enable Cop/SidekiqApiUsage + # rubocop:enable Cop/SidekiqRedisCall + end + end +end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 46855ca1921..a16103f452c 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -6,16 +6,17 @@ module Gitlab class TestBatchedBackgroundRunner < BaseBackgroundRunner include Gitlab::Database::DynamicModelHelpers - def initialize(result_dir:, connection:) + def initialize(result_dir:, connection:, from_id:) super(result_dir: result_dir, connection: connection) @connection = connection + @from_id = from_id end def jobs_by_migration_name Gitlab::Database::SharedModel.using_connection(connection) do Gitlab::Database::BackgroundMigration::BatchedMigration .executable - .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing + .where('id > ?', from_id) .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) @@ -102,6 +103,10 @@ module Gitlab end end end + + private + + attr_reader :from_id end end end diff --git a/lib/gitlab/database/obsolete_ignored_columns.rb b/lib/gitlab/database/obsolete_ignored_columns.rb index ad5473f1b74..2b88ab12380 100644 --- a/lib/gitlab/database/obsolete_ignored_columns.rb +++ b/lib/gitlab/database/obsolete_ignored_columns.rb @@ -23,8 +23,8 @@ module Gitlab private def ignored_columns_safe_to_remove_for(klass) - ignores = ignored_and_not_present(klass).each_with_object({}) do |col, h| - h[col] = klass.ignored_columns_details[col.to_sym] + ignores = ignored_and_not_present(klass).index_with do |col| + klass.ignored_columns_details[col.to_sym] end ignores.select { |_, i| i&.safe_to_remove? } diff --git a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb index 4e38eea963b..fd99062974c 100644 --- a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb +++ b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb @@ -19,7 +19,7 @@ module Gitlab attr_reader :table, :value - def initialize(table, value, partition_name: nil ) + def initialize(table, value, partition_name: nil) @table = table @value = value @partition_name = partition_name diff --git a/lib/gitlab/database/postgres_hll/buckets.rb b/lib/gitlab/database/postgres_hll/buckets.rb index cbc9544d905..3f64eee030e 100644 --- a/lib/gitlab/database/postgres_hll/buckets.rb +++ b/lib/gitlab/database/postgres_hll/buckets.rb @@ -61,7 +61,7 @@ module Gitlab num_uniques = ( ((TOTAL_BUCKETS**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS))) / - (num_zero_buckets + buckets.values.sum { |bucket_hash| 2**(-1 * bucket_hash) } ) + (num_zero_buckets + buckets.values.sum { |bucket_hash| 2**(-1 * bucket_hash) }) ).to_i if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index 3b1751c863d..dd10e0d7992 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -165,8 +165,8 @@ module Gitlab def self.in_factory_bot_create? Rails.env.test? && caller_locations.any? do |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' || - l.path.end_with?('lib/factory_bot/strategy/create.rb') || - l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' + l.path.end_with?('lib/factory_bot/strategy/create.rb') || + l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' end end end diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb index 88fe829c3d2..b54f3442512 100644 --- a/lib/gitlab/database/query_analyzers/query_recorder.rb +++ b/lib/gitlab/database/query_analyzers/query_recorder.rb @@ -4,7 +4,7 @@ module Gitlab module Database module QueryAnalyzers class QueryRecorder < Base - LOG_FILE = 'rspec/query_recorder.ndjson' + LOG_PATH = 'query_recorder/' class << self def raw? @@ -12,8 +12,9 @@ module Gitlab end def enabled? - # Only enable QueryRecorder in CI - ENV['CI'].present? + # Only enable QueryRecorder in CI on database MRs or default branch + ENV['CI_MERGE_REQUEST_LABELS']&.include?('database') || + (ENV['CI_COMMIT_REF_NAME'].present? && ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH']) end def analyze(sql) @@ -24,11 +25,14 @@ module Gitlab log_query(payload) end + def log_file + Rails.root.join(LOG_PATH, "#{ENV.fetch('CI_JOB_NAME_SLUG', 'rspec')}.ndjson") + end + private def log_query(payload) - log_path = Rails.root.join(LOG_FILE) - log_dir = File.dirname(log_path) + log_dir = Rails.root.join(LOG_PATH) # Create log directory if it does not exist since it is only created # ahead of time by certain CI jobs @@ -36,7 +40,7 @@ module Gitlab log_line = "#{Gitlab::Json.dump(payload)}\n" - File.write(log_path, log_line, mode: 'a') + File.write(log_file, log_line, mode: 'a') end end end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb index 74900dc0d26..6da76803f7c 100644 --- a/lib/gitlab/database/schema_cache_with_renamed_table.rb +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -40,10 +40,8 @@ module Gitlab end def renamed_tables_cache - @renamed_tables ||= begin - Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| - connection.view_exists?(old_name) - end + @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| + connection.view_exists?(old_name) end end diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb index c3cdcf1450d..2c8d0a4eb6d 100644 --- a/lib/gitlab/database/schema_cleaner.rb +++ b/lib/gitlab/database/schema_cleaner.rb @@ -25,7 +25,23 @@ module Gitlab # The intention here is to not introduce an assumption about the standard schema, # unless we have a good reason to do so. structure.gsub!(/public\.(\w+)/, '\1') - structure.gsub!(/CREATE EXTENSION IF NOT EXISTS (\w+) WITH SCHEMA public;/, 'CREATE EXTENSION IF NOT EXISTS \1;') + structure.gsub!( + /CREATE EXTENSION IF NOT EXISTS (\w+) WITH SCHEMA public;/, + 'CREATE EXTENSION IF NOT EXISTS \1;' + ) + + # Table lock-writes triggers should not be added to the schema + # These triggers are added by the rake task gitlab:db:lock_writes for a decomposed database. + structure.gsub!( + %r{ + ^CREATE.TRIGGER.gitlab_schema_write_trigger_\w+ + \s + BEFORE.INSERT.OR.DELETE.OR.UPDATE.OR.TRUNCATE.ON.\w+ + \s + FOR.EACH.STATEMENT.EXECUTE.FUNCTION.gitlab_schema_prevent_write\(\);$ + }x, + '' + ) structure.gsub!(/\n{3,}/, "\n\n") diff --git a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb index 9f096904d31..b2a7f5442e9 100644 --- a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb +++ b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb @@ -26,15 +26,32 @@ module Gitlab # it maps the tables to the tables that depend on it def tables_dependencies - @tables.to_h do |table_name| - [table_name, all_foreign_keys[table_name]&.map(&:from_table).to_a] + @tables.index_with do |table_name| + all_foreign_keys[table_name] end end def all_foreign_keys - @all_foreign_keys ||= @tables.flat_map do |table_name| - @connection.foreign_keys(table_name) - end.group_by(&:to_table) + @all_foreign_keys ||= @tables.each_with_object(Hash.new { |h, k| h[k] = [] }) do |table, hash| + foreign_keys_for(table).each do |fk| + hash[fk.to_table] << table + end + end + end + + def foreign_keys_for(table) + # Detached partitions like gitlab_partitions_dynamic._test_gitlab_partition_20220101 + # store their foreign keys in the public schema. + # + # See spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb + # for an example + name = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table) + + if name.schema == ::Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA.to_s + @connection.foreign_keys(name.identifier) + else + @connection.foreign_keys(table) + end end end end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index 8380bf23899..807ecdb862a 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -19,24 +19,32 @@ module Gitlab logger&.info "DRY RUN:" if dry_run - connection = Gitlab::Database.database_base_models[database_name].connection - schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) tables_to_truncate = Gitlab::Database::GitlabSchema.tables_to_schema.reject do |_, schema_name| - (GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection)).include?(schema_name) + GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(schema_name) end.keys + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + next if GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(detached_partition.table_schema) + + tables_to_truncate << detached_partition.fully_qualified_table_name + end + end + tables_sorted = Gitlab::Database::TablesSortedByForeignKeys.new(connection, tables_to_truncate).execute # Checking if all the tables have the write-lock triggers # to make sure we are deleting the right tables on the right database. tables_sorted.flatten.each do |table_name| - query = <<~SQL - SELECT COUNT(*) from information_schema.triggers - WHERE event_object_table = '#{table_name}' - AND trigger_name = 'gitlab_schema_write_trigger_for_#{table_name}' - SQL - - if connection.select_value(query) == 0 + lock_writes_manager = Gitlab::Database::LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: database_name, + logger: logger, + dry_run: dry_run + ) + + unless lock_writes_manager.table_locked_for_writes?(table_name) raise "Table '#{table_name}' is not locked for writes. Run the rake task gitlab:db:lock_writes first" end end @@ -51,18 +59,26 @@ module Gitlab # min_batch_size is the minimum number of new tables to truncate at each stage. # But in each stage we have also have to truncate the already truncated tables in the previous stages logger&.info "Truncating legacy tables for the database #{database_name}" - truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + truncate_tables_in_batches(tables_sorted) end private attr_accessor :database_name, :min_batch_size, :logger, :dry_run, :until_table - def truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + def connection + @connection ||= Gitlab::Database.database_base_models[database_name].connection + end + + def truncate_tables_in_batches(tables_sorted) truncated_tables = [] tables_sorted.flatten.each do |table| - sql_statement = "SELECT set_config('lock_writes.#{table}', 'false', false)" + table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table) + .identifier + + sql_statement = "SELECT set_config('lock_writes.#{table_name_without_schema}', 'false', false)" logger&.info(sql_statement) connection.execute(sql_statement) unless dry_run end diff --git a/lib/gitlab/database/type/indifferent_jsonb.rb b/lib/gitlab/database/type/indifferent_jsonb.rb new file mode 100644 index 00000000000..69bbcb383ba --- /dev/null +++ b/lib/gitlab/database/type/indifferent_jsonb.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' Jsonb data type to deserialize it into indifferent access Hash. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, :ind_jsonb + # end + class IndifferentJsonb < ::ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Jsonb + def type + :ind_jsonb + end + + def deserialize(value) + data = super + return unless data + + ::Gitlab::Utils.deep_indifferent_access(data) + end + end + end + end +end diff --git a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb new file mode 100644 index 00000000000..1181c259a5c --- /dev/null +++ b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module WorkItems + module HierarchyRestrictionsImporter + def self.upsert_restrictions + objective = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:objective]) + key_result = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:key_result]) + issue = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:issue]) + task = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:task]) + incident = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:incident]) + + restrictions = [ + { parent_type_id: objective.id, child_type_id: objective.id, maximum_depth: 9 }, + { parent_type_id: objective.id, child_type_id: key_result.id, maximum_depth: 1 }, + { parent_type_id: issue.id, child_type_id: task.id, maximum_depth: 1 }, + { parent_type_id: incident.id, child_type_id: task.id, maximum_depth: 1 } + ] + + ::WorkItems::HierarchyRestriction.upsert_all( + restrictions, + unique_by: :index_work_item_hierarchy_restrictions_on_parent_and_child + ) + end + + def self.find_or_create_type(name) + type = ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) + return type if type + + Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types + ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index 6d8395d048d..df0d71f3db2 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -4,7 +4,15 @@ module Gitlab module Diff module FileCollection class Compare < Base + delegate :limit_value, :current_page, :next_page, :prev_page, :total_count, :total_pages, to: :@pagination + def initialize(compare, project:, diff_options:, diff_refs: nil) + @pagination = Gitlab::PaginationDelegate.new( + page: diff_options&.delete(:page), + per_page: diff_options&.delete(:per_page), + count: diff_options&.delete(:count) + ) + super(compare, project: project, diff_options: diff_options, diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index 0a601bde612..56027d6a4de 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -10,6 +10,8 @@ module Gitlab # separate file keys (https://gitlab.com/gitlab-org/gitlab/issues/30550). # class MergeRequestDiffBatch < MergeRequestDiffBase + include PaginatedDiffs + DEFAULT_BATCH_PAGE = 1 DEFAULT_BATCH_SIZE = 30 @@ -25,41 +27,8 @@ module Gitlab } end - override :diffs - def diffs - strong_memoize(:diffs) do - @merge_request_diff.opening_external_diff do - # Avoiding any extra queries. - collection = @paginated_collection.to_a - - # The offset collection and calculation is required so that we - # know how much has been loaded in previous batches, collapsing - # the current paginated set accordingly (collection limit calculation). - # See: https://docs.gitlab.com/ee/development/diffs.html#diff-collection-limits - # - offset_index = collection.first&.index - options = diff_options.dup - - collection = - if offset_index && offset_index > 0 - offset_collection = relation.limit(offset_index) # rubocop:disable CodeReuse/ActiveRecord - options[:offset_index] = offset_index - offset_collection + collection - else - collection - end - - Gitlab::Git::DiffCollection.new(collection.map(&:to_hash), options) - end - end - end - private - def relation - @merge_request_diff.merge_request_diff_files - end - # rubocop: disable CodeReuse/ActiveRecord def load_paginated_collection(batch_page, batch_size, diff_options) batch_page ||= DEFAULT_BATCH_PAGE diff --git a/lib/gitlab/diff/file_collection/paginated_diffs.rb b/lib/gitlab/diff/file_collection/paginated_diffs.rb new file mode 100644 index 00000000000..63c186affe9 --- /dev/null +++ b/lib/gitlab/diff/file_collection/paginated_diffs.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + module FileCollection + module PaginatedDiffs + include Gitlab::Utils::StrongMemoize + extend ::Gitlab::Utils::Override + + override :diffs + def diffs + merge_request_diff.opening_external_diff do + # Avoiding any extra queries. + collection = paginated_collection.to_a + + # The offset collection and calculation is required so that we + # know how much has been loaded in previous batches, collapsing + # the current paginated set accordingly (collection limit calculation). + # See: https://docs.gitlab.com/ee/development/diffs.html#diff-collection-limits + # + offset_index = collection.first&.index + options = diff_options.dup + + collection = + if offset_index && offset_index > 0 + offset_collection = relation.limit(offset_index) # rubocop:disable CodeReuse/ActiveRecord + options[:offset_index] = offset_index + offset_collection + collection + else + collection + end + + Gitlab::Git::DiffCollection.new(collection.map(&:to_hash), options) + end + end + strong_memoize_attr :diffs + + private + + attr_reader :merge_request_diff, :paginated_collection + + def relation + merge_request_diff.merge_request_diff_files + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb b/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb new file mode 100644 index 00000000000..37abad81305 --- /dev/null +++ b/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + module FileCollection + # Builds a traditional paginated diff file collection using Kaminari + # `per` and `per_page` which is different from how `MergeRequestDiffBatch` + # works (e.g. supports gradual loading). + class PaginatedMergeRequestDiff < MergeRequestDiffBase + include PaginatedDiffs + + DEFAULT_PAGE = 1 + DEFAULT_PER_PAGE = 30 + + delegate :limit_value, :current_page, :next_page, :prev_page, :total_count, + :total_pages, to: :paginated_collection + + def initialize(merge_request_diff, page, per_page) + super(merge_request_diff, diff_options: nil) + + @paginated_collection = load_paginated_collection(page, per_page) + end + + private + + def load_paginated_collection(page, per_page) + page ||= DEFAULT_PAGE + per_page ||= DEFAULT_PER_PAGE + + relation.page(page).per([per_page.to_i, DEFAULT_PER_PAGE].min) + end + end + end + end +end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 924c28e3db5..b29c75ed467 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -73,7 +73,7 @@ module Gitlab private def filename?(line) - line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b', + line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b', '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit '--- /tmp/diffy', '+++ /tmp/diffy') end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 1e03f5d17ee..32794a6c99d 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -192,7 +192,7 @@ module Gitlab auto_submitted = mail.header['Auto-Submitted']&.value # Mail::Field#value would strip leading and trailing whitespace - # See also https://tools.ietf.org/html/rfc3834 + # See also https://www.rfc-editor.org/rfc/rfc3834 auto_submitted && auto_submitted != 'no' end diff --git a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb index cc822e4c10b..e168fa10630 100644 --- a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb +++ b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb @@ -228,7 +228,7 @@ module Gitlab def configured_api_url url = Gitlab::CurrentSettings.current_application_settings.error_tracking_api_url || - 'http://localhost:8080' + 'http://localhost:8080' Gitlab::UrlBlocker.validate!(url, schemes: %w[http https], allow_localhost: true) diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 721518c6fcc..8e48b482462 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -34,11 +34,9 @@ module Gitlab end def available_status_names - @available_status_names ||= begin - Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png')) + @available_status_names ||= Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png')) .map { |file| File.basename(file, '.png') } .sort - end end private diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 58b46a85aae..05d680c139c 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -54,8 +54,6 @@ module Gitlab file = find_file(match[:secret], match[:file]) # No file will be returned for a path traversal - return '' if file.nil? - return markdown unless file.try(:exists?) klass = @target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 4b877bf44da..8e1b51fcec5 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -16,6 +16,7 @@ module Gitlab CommitError = Class.new(BaseError) OSError = Class.new(BaseError) UnknownRef = Class.new(BaseError) + AmbiguousRef = Class.new(BaseError) CommandTimedOut = Class.new(CommandError) InvalidPageToken = Class.new(BaseError) InvalidRefFormatError = Class.new(BaseError) diff --git a/lib/gitlab/git/base_error.rb b/lib/gitlab/git/base_error.rb index a7eaa82b347..0b0fdef54cc 100644 --- a/lib/gitlab/git/base_error.rb +++ b/lib/gitlab/git/base_error.rb @@ -1,20 +1,50 @@ # frozen_string_literal: true +require 'grpc' module Gitlab module Git class BaseError < StandardError DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m.freeze + GRPC_CODES = { + '0' => 'ok', + '1' => 'cancelled', + '2' => 'unknown', + '3' => 'invalid_argument', + '4' => 'deadline_exceeded', + '5' => 'not_found', + '6' => 'already_exists', + '7' => 'permission_denied', + '8' => 'resource_exhausted', + '9' => 'failed_precondition', + '10' => 'aborted', + '11' => 'out_of_range', + '12' => 'unimplemented', + '13' => 'internal', + '14' => 'unavailable', + '15' => 'data_loss', + '16' => 'unauthenticated' + }.freeze + + attr_reader :status, :code, :service def initialize(msg = nil) - if msg - raw_message = msg.to_s - match = DEBUG_ERROR_STRING_REGEX.match(raw_message) - raw_message = match[1] if match + super && return if msg.nil? + + set_grpc_error_code(msg) if msg.is_a?(::GRPC::BadStatus) + + super(build_raw_message(msg)) + end + + def build_raw_message(message) + raw_message = message.to_s + match = DEBUG_ERROR_STRING_REGEX.match(raw_message) + match ? match[1] : raw_message + end - super(raw_message) - else - super - end + def set_grpc_error_code(grpc_error) + @status = grpc_error.code + @code = GRPC_CODES[@status.to_s] + @service = 'git' end end end diff --git a/lib/gitlab/git/cross_repo_comparer.rb b/lib/gitlab/git/cross_repo.rb index d42b2a3bd98..d44657e7db1 100644 --- a/lib/gitlab/git/cross_repo_comparer.rb +++ b/lib/gitlab/git/cross_repo.rb @@ -2,7 +2,7 @@ module Gitlab module Git - class CrossRepoComparer + class CrossRepo attr_reader :source_repo, :target_repo def initialize(source_repo, target_repo) @@ -10,15 +10,8 @@ module Gitlab @target_repo = target_repo end - def compare(source_ref, target_ref, straight:) - ensuring_ref_in_source(target_ref) do |target_commit_id| - Gitlab::Git::Compare.new( - source_repo, - target_commit_id, - source_ref, - straight: straight - ) - end + def execute(target_ref, &blk) + ensuring_ref_in_source(target_ref, &blk) end private diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3b5151ef4f2..2f9cfe3e764 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -116,9 +116,9 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - wrapped_gitaly_errors do - gitaly_ref_client.branch_names - end + refs = list_refs([Gitlab::Git::BRANCH_REF_PREFIX]) + + refs.map { |ref| Gitlab::Git.branch_name(ref.name) } end # Returns an Array of Branches @@ -134,6 +134,10 @@ module Gitlab wrapped_gitaly_errors do gitaly_ref_client.find_branch(name) end + rescue Gitlab::Git::AmbiguousRef + # Gitaly returns "reference is ambiguous" error in case when users request + # branch "my-branch", when another branch "my-branch/branch" exists. + # We handle this error here and return nil for this case. end def find_tag(name) @@ -158,9 +162,7 @@ module Gitlab # Returns the number of valid branches def branch_count - wrapped_gitaly_errors do - gitaly_ref_client.count_branch_names - end + branch_names.count end def rename(new_relative_path) @@ -202,16 +204,14 @@ module Gitlab # Returns the number of valid tags def tag_count - wrapped_gitaly_errors do - gitaly_ref_client.count_tag_names - end + tag_names.count end # Returns an Array of tag names def tag_names - wrapped_gitaly_errors do - gitaly_ref_client.tag_names - end + refs = list_refs([Gitlab::Git::TAG_REF_PREFIX]) + + refs.map { |ref| Gitlab::Git.tag_name(ref.name) } end # Returns an Array of Tags @@ -385,6 +385,12 @@ module Gitlab end end + def check_objects_exist(refs) + wrapped_gitaly_errors do + gitaly_commit_client.object_existence_map(Array.wrap(refs)) + end + end + def new_blobs(newrevs, dynamic_timeout: nil) newrevs = Array.wrap(newrevs).reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA } return [] if newrevs.empty? @@ -823,9 +829,14 @@ module Gitlab end def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) - CrossRepoComparer - .new(source_repository, self) - .compare(source_branch_name, target_branch_name, straight: straight) + CrossRepo.new(source_repository, self).execute(target_branch_name) do |target_commit_id| + Gitlab::Git::Compare.new( + source_repository, + target_commit_id, + source_branch_name, + straight: straight + ) + end end def write_ref(ref_path, ref, old_ref: nil) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index da2a81983ec..344dd27589c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -157,10 +157,10 @@ module Gitlab # for deploy tokens and builds def can_download? deploy_key_can_download_code? || - deploy_token_can_download? || - build_can_download? || - user_can_download? || - guest_can_download? + deploy_token_can_download? || + build_can_download? || + user_can_download? || + guest_can_download? end def check_container! @@ -339,7 +339,7 @@ module Gitlab def check_change_access! if changes == ANY can_push = deploy_key? || - user_can_push? || + user_can_push? || project&.any_branch_allows_collaboration?(user_access.user) unless can_push diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 6bcf4802fbe..de66ca7305f 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -571,7 +571,7 @@ module Gitlab end def encode_repeated(array) - Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) } ) + Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) }) end def call_find_commit(revision) diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb index dbcebec3aa2..05aee2fa55d 100644 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -40,6 +40,13 @@ module Gitlab gitaly_client_call(:rename_namespace, request, timeout: GitalyClient.fast_timeout) end + def exists?(name) + request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) + + response = gitaly_client_call(:namespace_exists, request, timeout: GitalyClient.fast_timeout) + response.exists + end + private def gitaly_client_call(type, request, timeout: nil) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 2312def5efc..66f70ed9dc6 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -452,6 +452,14 @@ module Gitlab when :index_update raise Gitlab::Git::Index::IndexError, index_error_message(detailed_error.index_update) else + # Some invalid path errors are caught by Gitaly directly and returned + # as an :index_update error, while others are found by libgit2 and + # come as generic errors. We need to convert the latter as IndexErrors + # as well. + if e.to_status.details.start_with?('invalid path') + raise Gitlab::Git::Index::IndexError, e.to_status.details + end + raise e end end @@ -600,17 +608,17 @@ module Gitlab case index_error.error_type when :ERROR_TYPE_EMPTY_PATH - "Received empty path" + "You must provide a file path" when :ERROR_TYPE_INVALID_PATH - "Invalid path: #{encoded_path}" + "invalid path: '#{encoded_path}'" when :ERROR_TYPE_DIRECTORY_EXISTS - "Directory already exists: #{encoded_path}" + "A directory with this name already exists" when :ERROR_TYPE_DIRECTORY_TRAVERSAL - "Directory traversal in path escapes repository: #{encoded_path}" + "Path cannot include directory traversal" when :ERROR_TYPE_FILE_EXISTS - "File already exists: #{encoded_path}" + "A file with this name already exists" when :ERROR_TYPE_FILE_NOT_FOUND - "File not found: #{encoded_path}" + "A file with this name doesn't exist" else "Unknown error performing git operation" end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index de76ade76cb..da579276101 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -17,6 +17,8 @@ module Gitlab 'desc' => Gitaly::SortDirection::DESCENDING }.freeze + AMBIGUOUS_REFERENCE = 'reference is ambiguous' + # 'repository' is a Gitlab::Git::Repository def initialize(repository) @repository = repository @@ -54,26 +56,6 @@ module Gitlab Gitlab::Git.branch_name(response.name) end - def branch_names - request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = gitaly_client_call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout) - consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } - end - - def tag_names - request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = gitaly_client_call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout) - consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } - end - - def count_tag_names - tag_names.count - end - - def count_branch_names - branch_names.count - end - def local_branches(sort_by: nil, pagination_params: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo, pagination_params: pagination_params) request.sort_by = sort_local_branches_by_param(sort_by) if sort_by @@ -109,6 +91,10 @@ module Gitlab target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name.dup, branch.target_commit.id, target_commit) + rescue GRPC::BadStatus => e + raise e unless e.message.include?(AMBIGUOUS_REFERENCE) + + raise Gitlab::Git::AmbiguousRef, "branch is ambiguous: #{branch_name}" end def find_tag(tag_name) diff --git a/lib/gitlab/gitaly_client/with_feature_flag_actors.rb b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb index 92fc524b724..3d81292da16 100644 --- a/lib/gitlab/gitaly_client/with_feature_flag_actors.rb +++ b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb @@ -16,8 +16,6 @@ module Gitlab # gitaly_client_call performs Gitaly calls including collected feature flag actors. The actors are retrieved # from repository actor and memoized. The service must set `self.repository_actor = a_repository` beforehand. def gitaly_client_call(*args, **kargs) - return GitalyClient.call(*args, **kargs) unless actors_aware_gitaly_calls? - unless repository_actor Gitlab::ErrorTracking.track_and_raise_for_dev_exception( Feature::InvalidFeatureFlagError.new("gitaly_client_call called without setting repository_actor") @@ -34,11 +32,8 @@ module Gitlab end end - # gitaly_feature_flag_actors returns a hash of actors implied from input repository. If actors_aware_gitaly_calls - # flag is not on, this method returns an empty hash. + # gitaly_feature_flag_actors returns a hash of actors implied from input repository. def gitaly_feature_flag_actors(repository) - return {} unless actors_aware_gitaly_calls? - container = find_repository_container(repository) { repository: repository, @@ -92,10 +87,6 @@ module Gitlab repository.container end end - - def actors_aware_gitaly_calls? - Feature.enabled?(:actors_aware_gitaly_calls) - end end end end diff --git a/lib/gitlab/github_gists_import/importer/gist_importer.rb b/lib/gitlab/github_gists_import/importer/gist_importer.rb new file mode 100644 index 00000000000..a5e87d3cf7d --- /dev/null +++ b/lib/gitlab/github_gists_import/importer/gist_importer.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module GithubGistsImport + module Importer + class GistImporter + attr_reader :gist, :user + + FileCountLimitError = Class.new(StandardError) + + # gist - An instance of `Gitlab::GithubGistsImport::Representation::Gist`. + def initialize(gist, user_id) + @gist = gist + @user = User.find(user_id) + end + + def execute + snippet = build_snippet + import_repository(snippet) if snippet.save! + + return ServiceResponse.success unless max_snippet_files_count_exceeded?(snippet) + + fail_and_track(snippet) + end + + private + + def build_snippet + attrs = { + title: gist.truncated_title, + visibility_level: gist.visibility_level, + content: gist.first_file[:file_content], + file_name: gist.first_file[:file_name], + author: user, + created_at: gist.created_at, + updated_at: gist.updated_at + } + + PersonalSnippet.new(attrs) + end + + def import_repository(snippet) + resolved_address = get_resolved_address + + snippet.create_repository + snippet.repository.fetch_as_mirror(gist.git_pull_url, forced: true, resolved_address: resolved_address) + rescue StandardError + remove_snippet_and_repository(snippet) + + raise + end + + def get_resolved_address + validated_pull_url, host = Gitlab::UrlBlocker.validate!(gist.git_pull_url, + schemes: Project::VALID_IMPORT_PROTOCOLS, + ports: Project::VALID_IMPORT_PORTS, + allow_localhost: allow_local_requests?, + allow_local_network: allow_local_requests?) + + host.present? ? validated_pull_url.host.to_s : '' + end + + def max_snippet_files_count_exceeded?(snippet) + snippet.all_files.size > Snippet.max_file_limit + end + + def remove_snippet_and_repository(snippet) + snippet.repository.remove if snippet.repository_exists? + snippet.destroy + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def fail_and_track(snippet) + remove_snippet_and_repository(snippet) + + ServiceResponse.error(message: 'Snippet max file count exceeded').track_exception(as: FileCountLimitError) + end + end + end + end +end diff --git a/lib/gitlab/github_gists_import/importer/gists_importer.rb b/lib/gitlab/github_gists_import/importer/gists_importer.rb new file mode 100644 index 00000000000..08744dbaf5f --- /dev/null +++ b/lib/gitlab/github_gists_import/importer/gists_importer.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module GithubGistsImport + module Importer + class GistsImporter + attr_reader :user, :client, :already_imported_cache_key + + ALREADY_IMPORTED_CACHE_KEY = 'github-gists-importer/already-imported/%{user}' + RESULT_CONTEXT = Struct.new(:success?, :error, :waiter, :next_attempt_in, keyword_init: true) + + def initialize(user, token) + @user = user + @client = Gitlab::GithubImport::Client.new(token, parallel: true) + @already_imported_cache_key = format(ALREADY_IMPORTED_CACHE_KEY, user: user.id) + end + + def execute + waiter = spread_parallel_import + + expire_already_imported_cache! + + RESULT_CONTEXT.new(success?: true, waiter: waiter) + rescue Gitlab::GithubImport::RateLimitError => e + RESULT_CONTEXT.new(success?: false, error: e, next_attempt_in: client.rate_limit_resets_in) + rescue StandardError => e + RESULT_CONTEXT.new(success?: false, error: e) + end + + private + + def spread_parallel_import + waiter = JobWaiter.new + worker_arguments = fetch_gists_to_import.map { |gist_hash| [user.id, gist_hash, waiter.key] } + waiter.jobs_remaining = worker_arguments.size + + schedule_bulk_perform(worker_arguments) + waiter + end + + def fetch_gists_to_import + page_counter = Gitlab::GithubImport::PageCounter.new(user, :gists, 'github-gists-importer') + collection = [] + + client.each_page(:gists, nil, page: page_counter.current) do |page| + next unless page_counter.set(page.number) + + collection += gists_from(page) + end + + page_counter.expire! + + collection + end + + def gists_from(page) + page.objects.each.with_object([]) do |gist, page_collection| + gist = gist.to_h + next if already_imported?(gist) + + page_collection << ::Gitlab::GithubGistsImport::Representation::Gist.from_api_response(gist).to_hash + + mark_as_imported(gist) + end + end + + def schedule_bulk_perform(worker_arguments) + # rubocop:disable Scalability/BulkPerformWithContext + Gitlab::ApplicationContext.with_context(user: user) do + Gitlab::GithubGistsImport::ImportGistWorker.bulk_perform_in( + 1.second, + worker_arguments, + batch_size: 1000, + batch_delay: 1.minute + ) + end + # rubocop:enable Scalability/BulkPerformWithContext + end + + def already_imported?(gist) + Gitlab::Cache::Import::Caching.set_includes?(already_imported_cache_key, gist[:id]) + end + + def mark_as_imported(gist) + Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, gist[:id]) + end + + def expire_already_imported_cache! + Gitlab::Cache::Import::Caching + .expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) + end + end + end + end +end diff --git a/lib/gitlab/github_gists_import/representation/gist.rb b/lib/gitlab/github_gists_import/representation/gist.rb new file mode 100644 index 00000000000..0d309a98f38 --- /dev/null +++ b/lib/gitlab/github_gists_import/representation/gist.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module GithubGistsImport + module Representation + class Gist + include Gitlab::GithubImport::Representation::ToHash + include Gitlab::GithubImport::Representation::ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :description, :is_public, :created_at, :updated_at, :files, :git_pull_url + + # Builds a gist from a GitHub API response. + # + # gist - An instance of `Hash` containing the gist + # details. + def self.from_api_response(gist, additional_data = {}) + hash = { + id: gist[:id], + description: gist[:description], + is_public: gist[:public], + files: gist[:files], + git_pull_url: gist[:git_pull_url], + created_at: gist[:created_at], + updated_at: gist[:updated_at] + } + + new(hash) + end + + # Builds a new gist using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Gitlab::GithubImport::Representation.symbolize_hash(raw_hash)) + end + + # attributes - A hash containing the raw gist details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + # Gist description can be an empty string, so we returning nil to use first file + # name as a title in such case on snippet creation + # Gist description has a limit of 256, while the snippet's title can be up to 255 + def truncated_title + title = description.presence || first_file[:file_name] + + title.truncate(255) + end + + def visibility_level + is_public ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE + end + + def first_file + _key, value = files.first + + { + file_name: value[:filename], + file_content: Gitlab::HTTP.try_get(value[:raw_url])&.body + } + end + + def github_identifiers + { id: id } + end + end + end + end +end diff --git a/lib/gitlab/github_gists_import/status.rb b/lib/gitlab/github_gists_import/status.rb new file mode 100644 index 00000000000..e997eb0bf88 --- /dev/null +++ b/lib/gitlab/github_gists_import/status.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module GithubGistsImport + class Status + IMPORT_STATUS_KEY = 'gitlab:github-gists-import:%{user_id}' + EXPIRATION_TIME = 24.hours + + def initialize(user_id) + @user_id = user_id + end + + def start! + change_status('started') + end + + def fail! + change_status('failed') + end + + def finish! + change_status('finished') + end + + def started? + Gitlab::Redis::SharedState.with { |redis| redis.get(import_status_key) == 'started' } + end + + private + + def change_status(status_name) + Gitlab::Redis::SharedState.with do |redis| + redis.set(import_status_key, status_name) + redis.expire(import_status_key, EXPIRATION_TIME) unless status_name == 'started' + end + end + + def import_status_key + format(IMPORT_STATUS_KEY, user_id: @user_id) + end + end + end +end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index 28a39128ec9..0c91eff1d10 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -10,25 +10,38 @@ module Gitlab def initialize(project, client) @project = project @client = client + @validation_errors = [] end # Builds and returns an Array of objects to bulk insert into the - # database. + # database and array of validation errors if object is invalid. # # enum - An Enumerable that returns the objects to turn into database # rows. def build_database_rows(enum) + errors = [] rows = enum.each_with_object([]) do |(object, _), result| - result << build(object) unless already_imported?(object) + next if already_imported?(object) + + attrs = build_attributes(object) + build_record = model.new(attrs) + + if build_record.invalid? + log_error(object[:id], build_record.errors.full_messages) + errors << build_record.errors + next + end + + result << attrs end log_and_increment_counter(rows.size, :fetched) - rows + [rows, errors] end # Bulk inserts the given rows into the database. - def bulk_insert(model, rows, batch_size: 100) + def bulk_insert(rows, batch_size: 100) rows.each_slice(batch_size) do |slice| ApplicationRecord.legacy_bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert @@ -40,6 +53,23 @@ module Gitlab raise NotImplementedError end + def bulk_insert_failures(validation_errors) + rows = validation_errors.map do |error| + correlation_id_value = Labkit::Correlation::CorrelationId.current_or_new_id + + { + source: self.class.name, + exception_class: 'ActiveRecord::RecordInvalid', + exception_message: error.full_messages.first.truncate(255), + correlation_id_value: correlation_id_value, + retry_count: nil, + created_at: Time.zone.now + } + end + + project.import_failures.insert_all(rows) + end + private def log_and_increment_counter(value, operation) @@ -57,6 +87,16 @@ module Gitlab value: value ) end + + def log_error(object_id, messages) + Gitlab::Import::Logger.error( + import_type: :github, + project_id: project.id, + importer: self.class.name, + message: messages, + github_identifier: object_id + ) + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index d6060141bce..065410693e5 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -15,6 +15,7 @@ module Gitlab # end class Client include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::GithubImport::Clients::SearchRepos attr_reader :octokit @@ -182,19 +183,6 @@ module Gitlab end end - def search_repos_by_name(name, options = {}) - with_retry { octokit.search_repositories(search_query(str: name, type: :name), options).to_h } - end - - def search_query(str:, type:, include_collaborations: true, include_orgs: true) - query = "#{str} in:#{type} is:public,private user:#{octokit.user.to_h[:login]}" - - query = [query, collaborations_subquery].join(' ') if include_collaborations - query = [query, organizations_subquery].join(' ') if include_orgs - - query - end - # Returns `true` if we're still allowed to perform API calls. # Search API has rate limit of 30, use lowered threshold when search is used. def requests_remaining? diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb new file mode 100644 index 00000000000..f6d1c8ed23c --- /dev/null +++ b/lib/gitlab/github_import/clients/proxy.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Clients + class Proxy + attr_reader :client + + def initialize(access_token, client_options) + @client = pick_client(access_token, client_options) + end + + def repos(search_text, pagination_options) + return { repos: filtered(client.repos, search_text) } if use_legacy? + + if use_graphql? + fetch_repos_via_graphql(search_text, pagination_options) + else + fetch_repos_via_rest(search_text, pagination_options) + end + end + + private + + def fetch_repos_via_rest(search_text, pagination_options) + { repos: client.search_repos_by_name(search_text, pagination_options)[:items] } + end + + def fetch_repos_via_graphql(search_text, pagination_options) + response = client.search_repos_by_name_graphql(search_text, pagination_options) + { + repos: response.dig(:data, :search, :nodes), + page_info: response.dig(:data, :search, :pageInfo) + } + end + + def pick_client(access_token, client_options) + return Gitlab::GithubImport::Client.new(access_token) unless use_legacy? + + Gitlab::LegacyGithubImport::Client.new(access_token, **client_options) + end + + def filtered(collection, search_text) + return collection if search_text.blank? + + collection.select { |item| item[:name].to_s.downcase.include?(search_text) } + end + + def use_legacy? + Feature.disabled?(:remove_legacy_github_client) + end + + def use_graphql? + Feature.enabled?(:github_client_fetch_repos_via_graphql) + end + end + end + end +end diff --git a/lib/gitlab/github_import/clients/search_repos.rb b/lib/gitlab/github_import/clients/search_repos.rb new file mode 100644 index 00000000000..bcd226087e7 --- /dev/null +++ b/lib/gitlab/github_import/clients/search_repos.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Clients + module SearchRepos + def search_repos_by_name_graphql(name, options = {}) + with_retry do + octokit.post( + '/graphql', + { query: graphql_search_repos_body(name, options) }.to_json + ).to_h + end + end + + def search_repos_by_name(name, options = {}) + with_retry do + octokit.search_repositories( + search_repos_query(str: name, type: :name), + options + ).to_h + end + end + + private + + def graphql_search_repos_body(name, options) + query = search_repos_query(str: name, type: :name) + query = "query: \"#{query}\"" + first = options[:first].present? ? ", first: #{options[:first]}" : '' + after = options[:after].present? ? ", after: \"#{options[:after]}\"" : '' + <<-TEXT + { + search(type: REPOSITORY, #{query}#{first}#{after}) { + nodes { + __typename + ... on Repository { + id: databaseId + name + full_name: nameWithOwner + owner { login } + } + } + pageInfo { + startCursor + endCursor + hasNextPage + hasPreviousPage + } + } + } + TEXT + end + + def search_repos_query(str:, type:, include_collaborations: true, include_orgs: true) + query = "#{str} in:#{type} is:public,private user:#{octokit.user.to_h[:login]}" + + query = [query, collaborations_subquery].join(' ') if include_collaborations + query = [query, organizations_subquery].join(' ') if include_orgs + + query + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index a9f8483d8c3..44ffcd7a1e4 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -18,7 +18,6 @@ module Gitlab def execute return if merge_request_id.blank? - note.project = project note.merge_request = merge_request build_author_attributes @@ -65,7 +64,7 @@ module Gitlab # To work around this we're using bulk_insert with a single row. This # allows us to efficiently insert data (even if it's just 1 row) # without having to use all sorts of hacks to disable callbacks. - ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [{ + attributes = { noteable_type: note.noteable_type, system: false, type: 'LegacyDiffNote', @@ -79,7 +78,12 @@ module Gitlab created_at: note.created_at, updated_at: note.updated_at, st_diff: note.diff_hash.to_yaml - }]) + } + + diff_note = LegacyDiffNote.new(attributes.merge(importing: true)) + diff_note.validate! + + ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [attributes]) end # rubocop:enabled Gitlab/BulkInsert diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index d964bae3dd2..b477468d327 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -60,6 +60,9 @@ module Gitlab work_item_type_id: issue.work_item_type_id } + issue = project.issues.new(attributes.merge(importing: true)) + issue.validate! + insert_and_return_id(attributes, project.issues) rescue ActiveRecord::InvalidForeignKey # It's possible the project has been deleted since scheduling this diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index 5e248c7cfc5..52c87dda347 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -22,7 +22,7 @@ module Gitlab def create_labels time = Time.zone.now - rows = [] + items = [] target_id = find_target_id issue.label_names.each do |label_name| @@ -31,16 +31,16 @@ module Gitlab # the project's labels. next unless (label_id = label_finder.id_for(label_name)) - rows << { + items << LabelLink.new( label_id: label_id, target_id: target_id, target_type: issue.issuable_type, created_at: time, updated_at: time - } + ) end - ApplicationRecord.legacy_bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert + LabelLink.bulk_insert!(items) end def find_target_id diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index 9a011f17a18..d5d1cd28b7c 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -13,7 +13,10 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def execute - bulk_insert(Label, build_labels) + rows, validation_errors = build_labels + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? build_labels_cache end @@ -29,7 +32,7 @@ module Gitlab LabelFinder.new(project).build_cache end - def build(label) + def build_attributes(label) time = Time.zone.now { @@ -49,6 +52,10 @@ module Gitlab def object_type :label end + + def model + Label + end end end end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 775afd5f53a..d064278e4a0 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -27,9 +27,9 @@ module Gitlab end def each_object_to_import - lfs_objects = Projects::LfsPointers::LfsObjectDownloadListService.new(project).execute + download_service = Projects::LfsPointers::LfsObjectDownloadListService.new(project) - lfs_objects.each do |object| + download_service.each_list_item do |object| Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) yield object diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 1a3a54d0053..560fbdc66e3 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -13,7 +13,10 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def execute - bulk_insert(Milestone, build_milestones) + rows, validation_errors = build_milestones + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? build_milestones_cache end @@ -29,7 +32,7 @@ module Gitlab MilestoneFinder.new(project).build_cache end - def build(milestone) + def build_attributes(milestone) { iid: milestone[:number], title: milestone[:title], @@ -53,6 +56,10 @@ module Gitlab def object_type :milestone end + + def model + Milestone + end end end end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 69b7b2c2a38..04da015a33f 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -33,6 +33,9 @@ module Gitlab updated_at: note.updated_at } + note = Note.new(attributes.merge(importing: true)) + note.validate! + # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 3c17ea1195e..5690a2cc997 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -61,6 +61,9 @@ module Gitlab updated_at: pull_request.updated_at } + mr = project.merge_requests.new(attributes.merge(importing: true)) + mr.validate! + create_merge_request_without_hooks(project, attributes, pull_request.iid) end @@ -93,7 +96,7 @@ module Gitlab return if project.repository.branch_exists?(source_branch) project.repository.add_branch(project.creator, source_branch, pull_request.source_branch_sha) - rescue Gitlab::Git::CommandError => e + rescue Gitlab::Git::PreReceiveError, Gitlab::Git::CommandError => e Gitlab::ErrorTracking.track_exception(e, source_branch: source_branch, project_id: merge_request.project.id, diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb index 640914acf4d..f05aa26a449 100644 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -4,42 +4,62 @@ module Gitlab module GithubImport module Importer class PullRequestMergedByImporter + # pull_request - An instance of + # `Gitlab::GithubImport::Representation::PullRequest` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` def initialize(pull_request, project, client) - @project = project @pull_request = pull_request + @project = project @client = client end def execute - merge_request = project.merge_requests.find_by_iid(pull_request.iid) - timestamp = Time.new.utc - merged_at = pull_request.merged_at user_finder = GithubImport::UserFinder.new(project, client) - gitlab_user_id = user_finder.user_id_for(pull_request.merged_by) + gitlab_user_id = begin + user_finder.user_id_for(pull_request.merged_by) + rescue ::Octokit::NotFound + nil + end + + metrics_upsert(gitlab_user_id) + + add_note! + end + + private + + attr_reader :project, :pull_request, :client + + def metrics_upsert(gitlab_user_id) MergeRequest::Metrics.upsert({ target_project_id: project.id, merge_request_id: merge_request.id, merged_by_id: gitlab_user_id, - merged_at: merged_at, + merged_at: pull_request.merged_at, created_at: timestamp, updated_at: timestamp }, unique_by: :merge_request_id) + end - unless gitlab_user_id - merge_request.notes.create!( - importing: true, - note: missing_author_note, - author_id: project.creator_id, - project: project, - created_at: merged_at - ) - end + def add_note! + merge_request.notes.create!( + importing: true, + note: missing_author_note, + author_id: project.creator_id, + project: project, + created_at: pull_request.merged_at + ) end - private + def merge_request + @merge_request ||= project.merge_requests.find_by_iid(pull_request.iid) + end - attr_reader :project, :pull_request, :client + def timestamp + @timestamp ||= Time.new.utc + end def missing_author_note s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index b11af90aa6f..de66f310edf 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -4,6 +4,9 @@ module Gitlab module GithubImport module Importer class PullRequestReviewImporter + # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` def initialize(review, project, client) @review = review @project = project @@ -13,7 +16,12 @@ module Gitlab def execute user_finder = GithubImport::UserFinder.new(project, client) - gitlab_user_id = user_finder.user_id_for(review.author) + + gitlab_user_id = begin + user_finder.user_id_for(review.author) + rescue ::Octokit::NotFound + nil + end if gitlab_user_id add_review_note!(gitlab_user_id) diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index fe6da30bbf8..62d579fda08 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -16,7 +16,10 @@ module Gitlab # to generate HTML version - you also need to regenerate it in # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. def execute - bulk_insert(Release, build_releases) + rows, validation_errors = build_releases + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? end def build_releases @@ -27,7 +30,7 @@ module Gitlab existing_tags.include?(release[:tag_name]) || release[:tag_name].nil? end - def build(release) + def build_attributes(release) existing_tags.add(release[:tag_name]) { @@ -66,6 +69,10 @@ module Gitlab def user_finder @user_finder ||= GithubImport::UserFinder.new(project, client) end + + def model + Release + end end end end diff --git a/lib/gitlab/github_import/markdown/attachment.rb b/lib/gitlab/github_import/markdown/attachment.rb index a5cf5ffa60e..1c814e34a39 100644 --- a/lib/gitlab/github_import/markdown/attachment.rb +++ b/lib/gitlab/github_import/markdown/attachment.rb @@ -28,6 +28,7 @@ module Gitlab def from_markdown_image(markdown_node) url = markdown_node.url + return unless url return unless github_url?(url, media: true) return unless whitelisted_type?(url, media: true) @@ -37,6 +38,7 @@ module Gitlab def from_markdown_link(markdown_node) url = markdown_node.url + return unless url return unless github_url?(url, docs: true) return unless whitelisted_type?(url, docs: true) @@ -46,7 +48,7 @@ module Gitlab def from_inline_html(markdown_node) img = Nokogiri::HTML.parse(markdown_node.string_content).xpath('//img')[0] - return unless img + return if img.nil? || img[:src].blank? return unless github_url?(img[:src], media: true) return unless whitelisted_type?(img[:src], media: true) diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb index 3face4c794b..c238ccb8932 100644 --- a/lib/gitlab/github_import/page_counter.rb +++ b/lib/gitlab/github_import/page_counter.rb @@ -9,10 +9,10 @@ module Gitlab attr_reader :cache_key # The base cache key to use for storing the last page number. - CACHE_KEY = 'github-importer/page-counter/%{project}/%{collection}' + CACHE_KEY = '%{import_type}/page-counter/%{object}/%{collection}' - def initialize(project, collection) - @cache_key = CACHE_KEY % { project: project.id, collection: collection } + def initialize(object, collection, import_type = 'github-importer') + @cache_key = CACHE_KEY % { import_type: import_type, object: object.id, collection: collection } end # Sets the page number to the given value. diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index f3be90834c7..9259d0295d5 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -4,18 +4,15 @@ module Gitlab module GithubImport module Representation class DiffNote - include Gitlab::Utils::StrongMemoize include ToHash include ExposeAttribute - NOTEABLE_TYPE = 'MergeRequest' NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze - DISCUSSION_CACHE_KEY = 'github-importer/discussion-id-map/%{project_id}/%{noteable_id}/%{original_note_id}' expose_attribute :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :created_at, :updated_at, :original_commit_id, :note_id, :end_line, :start_line, - :side, :in_reply_to_id + :side, :in_reply_to_id, :discussion_id # Builds a diff note from a GitHub API response. # @@ -45,7 +42,8 @@ module Gitlab end_line: note[:line], start_line: note[:start_line], side: note[:side], - in_reply_to_id: note[:in_reply_to_id] + in_reply_to_id: note[:in_reply_to_id], + discussion_id: DiffNotes::DiscussionId.new(note).find_or_generate } new(hash) @@ -59,7 +57,7 @@ module Gitlab new(hash) end - attr_accessor :merge_request, :project + attr_accessor :merge_request # attributes - A Hash containing the raw note details. The keys of this # Hash must be Symbols. @@ -74,7 +72,7 @@ module Gitlab end def noteable_type - NOTEABLE_TYPE + DiffNotes::DiscussionId::NOTEABLE_TYPE end def contains_suggestion? @@ -127,12 +125,6 @@ module Gitlab } end - def discussion_id - strong_memoize(:discussion_id) do - (in_reply_to_id.present? && current_discussion_id) || generate_discussion_id - end - end - private # Required by ExposeAttribute @@ -149,32 +141,6 @@ module Gitlab def addition? side == 'RIGHT' end - - def generate_discussion_id - Discussion.discussion_id( - Struct - .new(:noteable_id, :noteable_type) - .new(merge_request.id, NOTEABLE_TYPE) - ).tap do |discussion_id| - cache_discussion_id(discussion_id) - end - end - - def cache_discussion_id(discussion_id) - Gitlab::Cache::Import::Caching.write(discussion_id_cache_key(note_id), discussion_id) - end - - def current_discussion_id - Gitlab::Cache::Import::Caching.read(discussion_id_cache_key(in_reply_to_id)) - end - - def discussion_id_cache_key(id) - DISCUSSION_CACHE_KEY % { - project_id: project.id, - noteable_id: merge_request.id, - original_note_id: id - } - end end end end diff --git a/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb new file mode 100644 index 00000000000..38b560f21c0 --- /dev/null +++ b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module DiffNotes + class DiscussionId + NOTEABLE_TYPE = 'MergeRequest' + DISCUSSION_CACHE_REGEX = %r{/(?<repo>[^/]*)/pull/(?<iid>\d+)}i.freeze + DISCUSSION_CACHE_KEY = 'github-importer/discussion-id-map/%{project}/%{noteable_id}/%{original_note_id}' + + def initialize(note) + @note = note + @matches = note[:html_url].match(DISCUSSION_CACHE_REGEX) + end + + def find_or_generate + (note[:in_reply_to_id].present? && current_discussion_id) || generate_discussion_id + end + + private + + attr_reader :note, :matches + + def generate_discussion_id + discussion_id = Discussion.discussion_id( + Struct + .new(:noteable_id, :noteable_type) + .new(matches[:iid].to_i, NOTEABLE_TYPE) + ) + cache_discussion_id(discussion_id) + end + + def cache_discussion_id(discussion_id) + Gitlab::Cache::Import::Caching.write( + discussion_id_cache_key(note[:id]), discussion_id + ) + end + + def current_discussion_id + Gitlab::Cache::Import::Caching.read( + discussion_id_cache_key(note[:in_reply_to_id]) + ) + end + + def discussion_id_cache_key(id) + format(DISCUSSION_CACHE_KEY, + project: matches[:repo], + noteable_id: matches[:iid].to_i, + original_note_id: id + ) + end + end + end + end + end +end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 05278b2dd35..7792ef55b28 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -66,10 +66,10 @@ module Gitlab def valid?(repository_path) repository_path.end_with?(path_suffix) && - ( - !snippet? || - repository_path.match?(Gitlab::PathRegex.full_snippets_repository_path_regex) - ) + ( + !snippet? || + repository_path.match?(Gitlab::PathRegex.full_snippets_repository_path_regex) + ) end private diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ecb57bfc1a2..12cdcf445f7 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -65,8 +65,10 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) + push_frontend_feature_flag(:vscode_web_ide, current_user) push_frontend_feature_flag(:integration_slack_app_notifications) push_frontend_feature_flag(:vue_group_select) + push_frontend_feature_flag(:new_fonts, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/expose_permissions.rb b/lib/gitlab/graphql/expose_permissions.rb index ab9ed354673..070d3c188a4 100644 --- a/lib/gitlab/graphql/expose_permissions.rb +++ b/lib/gitlab/graphql/expose_permissions.rb @@ -5,11 +5,15 @@ module Gitlab module ExposePermissions extend ActiveSupport::Concern prepended do - def self.expose_permissions(permission_type, description: 'Permissions for the current user on the resource') + def self.expose_permissions( + permission_type, + description: 'Permissions for the current user on the resource', + &block) field :user_permissions, permission_type, description: description, null: false, - method: :itself + method: :itself, + &block end end end diff --git a/lib/gitlab/graphql/extensions/forward_only_externally_paginated_array_extension.rb b/lib/gitlab/graphql/extensions/forward_only_externally_paginated_array_extension.rb new file mode 100644 index 00000000000..651b4266756 --- /dev/null +++ b/lib/gitlab/graphql/extensions/forward_only_externally_paginated_array_extension.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +module Gitlab + module Graphql + module Extensions + # This extension is meant for resolvers that only support forward looking pagination. So in order to limit + # confusion for allowed GraphQL pagination arguments on the field, we limit this to just `first` and `after`. + class ForwardOnlyExternallyPaginatedArrayExtension < ExternallyPaginatedArrayExtension + def apply + field.argument :after, GraphQL::Types::String, + description: "Returns the elements in the list that come after the specified cursor.", + required: false + field.argument :first, GraphQL::Types::Int, + description: "Returns the first _n_ elements from the list.", + required: false + end + end + end + end +end diff --git a/lib/gitlab/graphql/limit/field_call_count.rb b/lib/gitlab/graphql/limit/field_call_count.rb index 4165970a2a6..3a02e8abbb5 100644 --- a/lib/gitlab/graphql/limit/field_call_count.rb +++ b/lib/gitlab/graphql/limit/field_call_count.rb @@ -14,9 +14,18 @@ module Gitlab private def increment_call_count(context) + query_id = fetch_query_id(context) + context[:call_count] ||= {} - context[:call_count][field] ||= 0 - context[:call_count][field] += 1 + context[:call_count][query_id] ||= {} + context[:call_count][query_id][field] ||= 0 + context[:call_count][query_id][field] += 1 + end + + def fetch_query_id(context) + context.query.operation_fingerprint + rescue TypeError + '' end def limit diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index eca4d42fb9a..208ca5f2d24 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -59,16 +59,7 @@ module Gitlab if before true elsif first - if Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) - limited_nodes.size > limit_value - else - case sliced_nodes - when Array - sliced_nodes.size > limit_value - else - sliced_nodes.limit(1).offset(limit_value).exists? # rubocop: disable CodeReuse/ActiveRecord - end - end + limited_nodes.size > limit_value else false end @@ -126,15 +117,9 @@ module Gitlab @has_previous_page = paginated_nodes.count > limit_value @has_previous_page ? paginated_nodes.last(limit_value) : paginated_nodes elsif loaded?(sliced_nodes) - if Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) - sliced_nodes.take(limit_value + 1) # rubocop: disable CodeReuse/ActiveRecord - else - sliced_nodes.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord - end - elsif Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) - sliced_nodes.limit(limit_value + 1).to_a + sliced_nodes.take(limit_value + 1) # rubocop: disable CodeReuse/ActiveRecord else - sliced_nodes.limit(limit_value) + sliced_nodes.limit(limit_value + 1).to_a end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 4eea96f8344..b112740c4ad 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -12,23 +12,11 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def users - # get all groups the current user has access to - # ignore order inherited from GroupsFinder to improve performance - current_user_groups = GroupsFinder.new(current_user).execute.unscope(:order) + groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user) + members = GroupMember.where(group: groups).non_invite - # the hierarchy of the current group - group_groups = @group.self_and_hierarchy.unscope(:order) - - # the groups where the above hierarchies intersect - intersect_groups = group_groups.where(id: current_user_groups) - - # members of @group hierarchy where the user has access to the groups - members = GroupMember.where(group: intersect_groups).non_invite - - # get all users the current user has access to (-> `SearchResults#users`), which also applies the query users = super - # filter users that belong to the previously selected groups users.where(id: members.select(:user_id)) end # rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 567c4dc899f..b05767c7ed4 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -59,7 +59,7 @@ module Gitlab raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" end - block.call fragment if block + yield fragment if block end rescue HTTParty::RedirectionTooDeep raise RedirectionTooDeep diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 7b1657d3854..3ef60be67a9 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -44,7 +44,8 @@ module Gitlab Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?, allow_object_storage: allow_object_storage?, - dns_rebind_protection: dns_rebind_protection?) + dns_rebind_protection: dns_rebind_protection?, + schemes: %w[http https]) rescue Gitlab::UrlBlocker::BlockedUrlError => e raise Gitlab::HTTP::BlockedUrlError, "URL '#{url}' is blocked: #{e.message}" end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index a42cac61a55..7a42ffca779 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -50,30 +50,30 @@ module Gitlab 'eo' => 0, 'es' => 35, 'fil_PH' => 0, - 'fr' => 85, + 'fr' => 94, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, 'ja' => 30, - 'ko' => 21, - 'nb_NO' => 25, + 'ko' => 20, + 'nb_NO' => 24, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 58, - 'ro_RO' => 98, - 'ru' => 25, + 'pt_BR' => 57, + 'ro_RO' => 96, + 'ru' => 26, 'si_LK' => 11, 'tr_TR' => 11, 'uk' => 52, - 'zh_CN' => 98, + 'zh_CN' => 97, 'zh_HK' => 1, - 'zh_TW' => 100 + 'zh_TW' => 99 }.freeze private_constant :TRANSLATION_LEVELS - def selectable_locales + def selectable_locales(minimum_translation_level = MINIMUM_TRANSLATION_LEVEL) AVAILABLE_LANGUAGES.reject do |code, _name| - percentage_translated_for(code) < MINIMUM_TRANSLATION_LEVEL + percentage_translated_for(code) < minimum_translation_level end end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index b05d9cb2489..d1fd45882d3 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -211,23 +211,21 @@ module Gitlab def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. - @existing_or_new_object ||= begin - if existing_object? - attribute_hash = attribute_hash_for(['events']) - - existing_object.assign_attributes(attribute_hash) if attribute_hash.any? - - existing_object - else - # Because of single-type inheritance, we need to be careful to use the `type` field - # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 - inheritance_column = relation_class.try(:inheritance_column) - inheritance_attributes = parsed_relation_hash.slice(inheritance_column) - object = relation_class.new(inheritance_attributes) - object.assign_attributes(parsed_relation_hash) - object - end - end + @existing_or_new_object ||= if existing_object? + attribute_hash = attribute_hash_for(['events']) + + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? + + existing_object + else + # Because of single-type inheritance, we need to be careful to use the `type` field + # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 + inheritance_column = relation_class.try(:inheritance_column) + inheritance_attributes = parsed_relation_hash.slice(inheritance_column) + object = relation_class.new(inheritance_attributes) + object.assign_attributes(parsed_relation_hash) + object + end end def attribute_hash_for(attributes) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index aa66fe8a5ae..564008e7a73 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -35,7 +35,7 @@ module Gitlab Timeout.timeout(TIMEOUT_LIMIT) do stderr_r, stderr_w = IO.pipe - stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w ) + stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w) # When validation is performed on a small archive (e.g. 100 bytes) # `wait_thr` finishes before we can get process group id. Do not diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index a08efdf400b..7f3254be3e8 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -132,3 +132,23 @@ ee: - :push_event_payload - iterations_cadences: - :iterations + +# When associated resources are from outside the group, you might need to +# validate that a user who is exporting the group can access these +# associations. `include_if_exportable` accepts an array of associations for a +# resource. During export, the `exportable_association?` method on the +# resource is called with the association's name and user to validate if +# associated resource can be included in the export. +# +# This definition will call epic's `exportable_association?(:parent, +# current_user: current_user)` method and include epic's parent association +# for each epic only if the method returns true: +# +# include_if_exportable: +# group: +# epics: +# - :parent +include_if_exportable: + group: + epics: + - :parent diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index dc80c92f507..ee360020556 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -27,7 +27,7 @@ module Gitlab end def read_hash - Gitlab::Json.parse(IO.read(@path)) + Gitlab::Json.parse(::File.read(@path)) rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) raise Gitlab::ImportExport::Error, 'Incorrect JSON format' diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 9931b09e9ca..83aab6d031e 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -71,7 +71,7 @@ module Gitlab @lfs_json ||= begin - json = IO.read(lfs_json_path) + json = File.read(lfs_json_path) Gitlab::Json.parse(json) rescue StandardError raise Gitlab::ImportExport::Error, 'Incorrect JSON format' diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index c94549a2b3f..0ad19f82e71 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -15,15 +15,13 @@ module Gitlab def map @map ||= - begin - @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| - if member['user'] - old_user_id = member['user']['id'] - existing_user_id = existing_users_email_map[get_email(member)] - hash[old_user_id] = existing_user_id if existing_user_id && add_team_member(member, existing_user_id) - else - add_team_member(member) - end + @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| + if member['user'] + old_user_id = member['user']['id'] + existing_user_id = existing_users_email_map[get_email(member)] + hash[old_user_id] = existing_user_id if existing_user_id && add_team_member(member, existing_user_id) + else + add_team_member(member) end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 2d9c8d1108e..cc69ed55744 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -763,6 +763,19 @@ excluded_attributes: - :import_type - :import_source - :integrations + - :push_hooks_integrations + - :tag_push_hooks_integrations + - :issue_hooks_integrations + - :confidential_issue_hooks_integrations + - :merge_request_hooks_integrations + - :note_hooks_integrations + - :confidential_note_hooks_integrations + - :job_hooks_integrations + - :archive_trace_hooks_integrations + - :pipeline_hooks_integrations + - :wiki_page_hooks_integrations + - :deployment_hooks_integrations + - :alert_hooks_integrations - :mirror - :runners_token - :runners_token_encrypted @@ -1209,7 +1222,9 @@ ee: - :description iterations_cadence: - :title - + excluded_attributes: + project: + - :vulnerability_hooks_integrations preloads: issues: epic: diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index bd34cd3ff6e..05b96f7e8ce 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -81,15 +81,13 @@ module Gitlab end def json_writer - @json_writer ||= begin - if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) - full_path = File.join(@shared.export_path, 'tree') - Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) - else - full_path = File.join(@shared.export_path, ImportExport.project_filename) - Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') - end - end + @json_writer ||= if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) + full_path = File.join(@shared.export_path, 'tree') + Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) + else + full_path = File.join(@shared.export_path, ImportExport.project_filename) + Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') + end end end end diff --git a/lib/gitlab/import_export/remote_stream_upload.rb b/lib/gitlab/import_export/remote_stream_upload.rb index f3bd241c0bd..1fb3faf0767 100644 --- a/lib/gitlab/import_export/remote_stream_upload.rb +++ b/lib/gitlab/import_export/remote_stream_upload.rb @@ -25,6 +25,7 @@ module Gitlab end end end + class StreamError < StandardError attr_reader :response_body @@ -33,6 +34,7 @@ module Gitlab @response_body = response_body end end + class ChunkStream DEFAULT_BUFFER_SIZE = 128.kilobytes diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 1c6629cf942..cc214d730fe 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -46,6 +46,11 @@ module Gitlab ) Repositories::DestroyService.new(repository).execute + + # Because Gitlab::Git::Repository#remove happens inside a run_after_commit + # callback in the Repositories::DestroyService#execute we need to trigger + # the callback. + repository.project.touch end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 5d3a6b0c6e1..a94ea6f595b 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -15,7 +15,6 @@ module Gitlab ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), - ImportSource.new('google_code', 'Google Code', nil), ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repository by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), diff --git a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb index 6aeeb1d31aa..cbc4f126293 100644 --- a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb +++ b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb @@ -19,7 +19,7 @@ module Gitlab "**Incident key:** #{incident_payload['incident_key']}", "**Created at:** #{markdown_incident_created_at}", "**Assignees:** #{markdown_assignees.join(', ')}", - "**Impacted services:** #{markdown_impacted_services.join(', ')}" + "**Impacted service:** #{markdown_impacted_service}" ].join(markdown_line_break) end @@ -47,10 +47,9 @@ module Gitlab end end - def markdown_impacted_services - Array(incident_payload['impacted_services']).map do |is| - markdown_link(is['summary'], is['url']) - end + def markdown_impacted_service + service = incident_payload['impacted_service'] + markdown_link(service['summary'], service['url']) unless service.nil? end def markdown_link(label, url) diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index a371930621d..a664656c467 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -39,7 +39,8 @@ module Gitlab end end - %i[get_request_count query_time read_bytes write_bytes].each do |method| + %i[get_request_count get_cross_slot_request_count get_allowed_cross_slot_request_count query_time read_bytes + write_bytes].each do |method| define_method method do STORAGES.sum(&method) end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 268c6cdf459..de24132a28e 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -45,6 +45,16 @@ module Gitlab ::RequestStore[write_bytes_key] += num_bytes end + def increment_cross_slot_request_count(amount = 1) + ::RequestStore[cross_slots_key] ||= 0 + ::RequestStore[cross_slots_key] += amount + end + + def increment_allowed_cross_slot_request_count(amount = 1) + ::RequestStore[allowed_cross_slots_key] ||= 0 + ::RequestStore[allowed_cross_slots_key] += amount + end + def get_request_count ::RequestStore[request_count_key] || 0 end @@ -61,13 +71,32 @@ module Gitlab ::RequestStore[call_details_key] ||= [] end + def get_cross_slot_request_count + ::RequestStore[cross_slots_key] || 0 + end + + def get_allowed_cross_slot_request_count + ::RequestStore[allowed_cross_slots_key] || 0 + end + def query_time query_time = ::RequestStore[call_duration_key] || 0 query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def redis_cluster_validate!(commands) - ::Gitlab::Instrumentation::RedisClusterValidator.validate!(commands) if @redis_cluster_validation + return true unless @redis_cluster_validation + + result = ::Gitlab::Instrumentation::RedisClusterValidator.validate(commands) + return true if result.nil? + + if !result[:valid] && !result[:allowed] && (Rails.env.development? || Rails.env.test?) + raise RedisClusterValidator::CrossSlotError, "Redis command #{result[:command_name]} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" + end + + increment_allowed_cross_slot_request_count if result[:allowed] + + result[:valid] end def enable_redis_cluster_validation @@ -122,6 +151,14 @@ module Gitlab strong_memoize(:call_details_key) { build_key(:redis_call_details) } end + def cross_slots_key + strong_memoize(:cross_slots_key) { build_key(:redis_cross_slot_request_count) } + end + + def allowed_cross_slots_key + strong_memoize(:allowed_cross_slots_key) { build_key(:redis_allowed_cross_slot_request_count) } + end + def build_key(namespace) "#{storage_key}_#{namespace}" end diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 36d3e088956..1567e54d8da 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -183,19 +183,22 @@ module Gitlab CrossSlotError = Class.new(StandardError) class << self - def validate!(commands) - return unless Rails.env.development? || Rails.env.test? - return if allow_cross_slot_commands? + def validate(commands) return if commands.empty? # early exit for single-command (non-pipelined) if it is a single-key-command command_name = commands.size > 1 ? "PIPELINE/MULTI" : commands.first.first.to_s.upcase return if commands.size == 1 && REDIS_COMMANDS.dig(command_name, :single_key) - key_slots = commands.map { |command| key_slots(command) }.flatten - if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord - raise CrossSlotError, "Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" - end + keys = commands.map { |command| extract_keys(command) }.flatten + + { + # calculate key-slots only if not allowed + valid: allow_cross_slot_commands? || !has_cross_slot_keys?(keys), + command_name: command_name, + key_count: keys.size, + allowed: allow_cross_slot_commands? + } end # Keep track of the call stack to allow nested calls to work. @@ -210,15 +213,17 @@ module Gitlab private - def key_slots(command) + def extract_keys(command) argument_positions = REDIS_COMMANDS[command.first.to_s.upcase] return [] unless argument_positions arguments = command.flatten[argument_positions[:first]..argument_positions[:last]] - arguments.each_slice(argument_positions[:step]).map do |args| - key_slot(args.first) - end + arguments.each_slice(argument_positions[:step]).map(&:first) + end + + def has_cross_slot_keys?(keys) + keys.map { |key| key_slot(key) }.uniq.many? # rubocop: disable CodeReuse/ActiveRecord end def allow_cross_slot_commands? diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index f19279df2fe..35dd7cbfeb8 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -33,7 +33,10 @@ module Gitlab def instrument_call(commands) start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request(commands.size) - instrumentation_class.redis_cluster_validate!(commands) + + if !instrumentation_class.redis_cluster_validate!(commands) && ::RequestStore.active? + instrumentation_class.increment_cross_slot_request_count + end yield rescue ::Redis::BaseError => ex @@ -62,13 +65,11 @@ module Gitlab # This count is an approximation that omits the Redis protocol overhead # of type prefixes, length prefixes and line endings. command.each do |x| - size += begin - if x.is_a? Array - x.inject(0) { |sum, y| sum + y.to_s.bytesize } - else - x.to_s.bytesize - end - end + size += if x.is_a? Array + x.inject(0) { |sum, y| sum + y.to_s.bytesize } + else + x.to_s.bytesize + end end instrumentation_class.increment_write_bytes(size) diff --git a/lib/gitlab/instrumentation/redis_payload.rb b/lib/gitlab/instrumentation/redis_payload.rb index 86a6525c8d0..62a4d1a846f 100644 --- a/lib/gitlab/instrumentation/redis_payload.rb +++ b/lib/gitlab/instrumentation/redis_payload.rb @@ -20,6 +20,8 @@ module Gitlab { "#{key_prefix}_calls": -> { get_request_count }, + "#{key_prefix}_cross_slot_calls": -> { get_cross_slot_request_count }, + "#{key_prefix}_allowed_cross_slot_calls": -> { get_allowed_cross_slot_request_count }, "#{key_prefix}_duration_s": -> { query_time }, "#{key_prefix}_read_bytes": -> { read_bytes }, "#{key_prefix}_write_bytes": -> { write_bytes } diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index b8a2567b775..15a760fada0 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -35,6 +35,7 @@ module Gitlab instrument_uploads(payload) instrument_rate_limiting_gates(payload) instrument_global_search_api(payload) + instrument_ldap(payload) end def instrument_gitaly(payload) @@ -136,6 +137,14 @@ module Gitlab payload.merge!(::Gitlab::Instrumentation::GlobalSearchApi.payload) end + def instrument_ldap(payload) + ldap_count = Gitlab::Metrics::Subscribers::Ldap.count + + return if ldap_count == 0 + + payload.merge! Gitlab::Metrics::Subscribers::Ldap.payload + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index d0702fb5c7d..12ec6447251 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -27,8 +27,8 @@ module Gitlab def data return {} if issuable_ids.empty? - issuable_ids.each_with_object({}) do |id, issuable_meta| - issuable_meta[id] = metadata_for_issuable(id) + issuable_ids.index_with do |id| + metadata_for_issuable(id) end end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index 02b0c902a70..7abfe8e38e8 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -35,12 +35,6 @@ module Gitlab request_params[:base_uri] = uri.to_s request_params.merge!(auth_params) - if Feature.enabled?(:jira_raise_timeouts, type: :ops) - request_params[:open_timeout] = 2.minutes - request_params[:read_timeout] = 2.minutes - request_params[:write_timeout] = 2.minutes - end - result = Gitlab::HTTP.public_send(http_method, path, **request_params) # rubocop:disable GitlabSecurity/PublicSend @authenticated = result.response.is_a?(Net::HTTPOK) store_cookies(result) if options[:use_cookies] diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 5057317ae01..7b031c26b72 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -16,7 +16,7 @@ module Gitlab @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id) @imported_items_cache_key = JiraImport.already_imported_cache_key(:issues, project.id) @job_waiter = JobWaiter.new - @issue_type_id = WorkItems::Type.default_issue_type.id + @issue_type_id = ::WorkItems::Type.default_issue_type.id end def execute diff --git a/lib/gitlab/jwt_authenticatable.rb b/lib/gitlab/jwt_authenticatable.rb index 08d9f69497e..7c36bbf3426 100644 --- a/lib/gitlab/jwt_authenticatable.rb +++ b/lib/gitlab/jwt_authenticatable.rb @@ -3,7 +3,7 @@ module Gitlab module JwtAuthenticatable # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 - # bytes https://tools.ietf.org/html/rfc4868#section-2.6 + # bytes https://www.rfc-editor.org/rfc/rfc4868#section-2.6 SECRET_LENGTH = 32 def self.included(base) diff --git a/lib/gitlab/jwt_token.rb b/lib/gitlab/jwt_token.rb index 11bc5479b6e..83aa7fa4a15 100644 --- a/lib/gitlab/jwt_token.rb +++ b/lib/gitlab/jwt_token.rb @@ -42,7 +42,7 @@ module Gitlab def ==(other) self.id == other.id && - self.payload == other.payload + self.payload == other.payload end def issued_at=(value) diff --git a/lib/gitlab/kubernetes/helm/v2/install_command.rb b/lib/gitlab/kubernetes/helm/v2/install_command.rb index 10e16723e45..c50db6bf177 100644 --- a/lib/gitlab/kubernetes/helm/v2/install_command.rb +++ b/lib/gitlab/kubernetes/helm/v2/install_command.rb @@ -36,13 +36,13 @@ module Gitlab # installation and uprade of applications def install_command command = ['helm', 'upgrade', name, chart] + - install_flag + - rollback_support_flag + - reset_values_flag + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag + install_flag + + rollback_support_flag + + reset_values_flag + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag command.shelljoin end diff --git a/lib/gitlab/kubernetes/helm/v2/patch_command.rb b/lib/gitlab/kubernetes/helm/v2/patch_command.rb index 2855e6444b1..40e56771e47 100644 --- a/lib/gitlab/kubernetes/helm/v2/patch_command.rb +++ b/lib/gitlab/kubernetes/helm/v2/patch_command.rb @@ -37,10 +37,10 @@ module Gitlab def upgrade_command command = ['helm', 'upgrade', name, chart] + - reuse_values_flag + - version_flag + - namespace_flag + - value_flag + reuse_values_flag + + version_flag + + namespace_flag + + value_flag command.shelljoin end diff --git a/lib/gitlab/kubernetes/helm/v3/install_command.rb b/lib/gitlab/kubernetes/helm/v3/install_command.rb index 20d17f49115..8d521f0dcd4 100644 --- a/lib/gitlab/kubernetes/helm/v3/install_command.rb +++ b/lib/gitlab/kubernetes/helm/v3/install_command.rb @@ -33,13 +33,13 @@ module Gitlab # installation and uprade of applications def install_command command = ['helm', 'upgrade', name, chart] + - install_flag + - rollback_support_flag + - reset_values_flag + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag + install_flag + + rollback_support_flag + + reset_values_flag + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag command.shelljoin end diff --git a/lib/gitlab/kubernetes/helm/v3/patch_command.rb b/lib/gitlab/kubernetes/helm/v3/patch_command.rb index 00f340591e7..1278e524bd2 100644 --- a/lib/gitlab/kubernetes/helm/v3/patch_command.rb +++ b/lib/gitlab/kubernetes/helm/v3/patch_command.rb @@ -34,10 +34,10 @@ module Gitlab def upgrade_command command = ['helm', 'upgrade', name, chart] + - reuse_values_flag + - version_flag + - namespace_flag + - value_flag + reuse_values_flag + + version_flag + + namespace_flag + + value_flag command.shelljoin end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 92ffa65fe74..44e53e9ec70 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -161,7 +161,7 @@ module Gitlab def validate_url! return if Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? - Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false) + Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false, schemes: %w[http https]) end def service_account_exists?(resource) diff --git a/lib/gitlab/memory/jemalloc.rb b/lib/gitlab/memory/jemalloc.rb index e20e186cab9..6025e6ab6f2 100644 --- a/lib/gitlab/memory/jemalloc.rb +++ b/lib/gitlab/memory/jemalloc.rb @@ -14,41 +14,22 @@ module Gitlab STATS_DEFAULT_FORMAT = :json - FILENAME_PREFIX = 'jemalloc_stats' - # Return jemalloc stats as a string. def stats(format: STATS_DEFAULT_FORMAT) - verify_format!(format) - - with_malloc_stats_print do |stats_print| - StringIO.new.tap { |io| write_stats(stats_print, io, STATS_FORMATS[format]) }.string - end + dump_stats(StringIO.new, format: format).string end - # Write jemalloc stats to the given directory - # @param [String] path Directory path the dump will be put into - # @param [String] tmp_dir Directory path the dump will be streaming to. It is moved to `path` when finished. - # @param [String] format `json` or `txt` - # @param [String] filename_label Optional custom string that will be injected into the file name, e.g. `worker_0` - # @return [String] Full path to the resulting dump file - def dump_stats(path:, tmp_dir: Dir.tmpdir, format: STATS_DEFAULT_FORMAT, filename_label: nil) + # Streams jemalloc stats to the given IO object. + def dump_stats(io, format: STATS_DEFAULT_FORMAT) verify_format!(format) format_settings = STATS_FORMATS[format] - tmp_file_path = File.join(tmp_dir, file_name(format_settings[:extension], filename_label)) - file_path = File.join(path, file_name(format_settings[:extension], filename_label)) with_malloc_stats_print do |stats_print| - File.open(tmp_file_path, 'wb') do |io| - write_stats(stats_print, io, format_settings) - end + write_stats(stats_print, io, format_settings) end - # On OSX, `with_malloc_stats_print` is no-op, and, as result, no file will be written - return unless File.exist?(tmp_file_path) - - FileUtils.mv(tmp_file_path, file_path) - file_path + io end private @@ -95,10 +76,6 @@ module Gitlab stats_print.call(callback, nil, format[:options]) end - - def file_name(extension, filename_label) - [FILENAME_PREFIX, $$, filename_label, Time.current.to_i, extension].reject(&:blank?).join('.') - end end end end diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb new file mode 100644 index 00000000000..710c89c6216 --- /dev/null +++ b/lib/gitlab/memory/reporter.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Reporter + attr_reader :reports_path + + def initialize(reports_path: nil, logger: Gitlab::AppLogger) + @reports_path = reports_path || ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"] || Dir.mktmpdir + @logger = logger + + @worker_id = ::Prometheus::PidProvider.worker_id + @worker_uuid = SecureRandom.uuid + + init_prometheus_metrics + end + + def run_report(report) + return false unless report.active? + + @logger.info( + log_labels( + message: 'started', + perf_report: report.name + )) + + start_monotonic_time = Gitlab::Metrics::System.monotonic_time + start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + + report_file = store_report(report) + + cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) + duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + + @logger.info( + log_labels( + message: 'finished', + perf_report: report.name, + cpu_s: cpu_s.round(2), + duration_s: duration_s.round(2), + perf_report_file: report_file, + perf_report_size_bytes: file_size(report_file) + )) + + @report_duration_counter.increment({ report: report.name }, duration_s) + + true + rescue StandardError => e + @logger.error( + log_labels( + message: 'failed', + perf_report: report.name, + error: e.inspect + )) + + false + end + + private + + def store_report(report) + # Store report in tmp subdir while it is still streaming. + # This will clearly separate finished reports from the files we are still writing to. + tmp_dir = File.join(@reports_path, 'tmp') + FileUtils.mkdir_p(tmp_dir) + + report_file = file_name(report) + tmp_file_path = File.join(tmp_dir, report_file) + + io_r, io_w = IO.pipe + pid = nil + File.open(tmp_file_path, 'wb') do |file| + extras = { + in: io_r, + out: file, + err: $stderr + } + pid = Process.spawn('gzip', '--fast', **extras) + io_r.close + + report.run(io_w) + io_w.close + + Process.waitpid(pid) + end + + File.join(@reports_path, report_file).tap do |report_file_path| + FileUtils.mv(tmp_file_path, report_file_path) + end + ensure + [io_r, io_w].each(&:close) + + # Make sure we don't leave any running processes behind. + Gitlab::ProcessManagement.signal(pid, :KILL) if pid + end + + def log_labels(**extra_labels) + { + pid: $$, + worker_id: @worker_id, + perf_report_worker_uuid: @worker_uuid + }.merge(extra_labels) + end + + def file_name(report) + timestamp = Time.current.strftime('%Y-%m-%d.%H:%M:%S:%L') + + report_id = [@worker_id, @worker_uuid].join(".") + + [report.name, timestamp, report_id, 'gz'].compact_blank.join('.') + end + + def file_size(file_path) + File.size(file_path.to_s) + rescue Errno::ENOENT + 0 + end + + def init_prometheus_metrics + default_labels = { pid: @worker_id } + + @report_duration_counter = Gitlab::Metrics.counter( + :gitlab_diag_report_duration_seconds_total, + 'Total time elapsed for running diagnostic report', + default_labels + ) + end + end + end +end diff --git a/lib/gitlab/memory/reports/heap_dump.rb b/lib/gitlab/memory/reports/heap_dump.rb new file mode 100644 index 00000000000..95779407f12 --- /dev/null +++ b/lib/gitlab/memory/reports/heap_dump.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + module Reports + class HeapDump + class << self + def enqueue! + @write_heap_dump = true + end + + def enqueued? + !!@write_heap_dump + end + end + + def name + 'heap_dump' + end + + def active? + Feature.enabled?(:report_heap_dumps, type: :ops) + end + + def run(writer) + return false unless self.class.enqueued? + + ObjectSpace.dump_all(output: writer) + + true + end + end + end + end +end diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb index 05f0717d7c3..cfda409594f 100644 --- a/lib/gitlab/memory/reports/jemalloc_stats.rb +++ b/lib/gitlab/memory/reports/jemalloc_stats.rb @@ -4,70 +4,19 @@ module Gitlab module Memory module Reports class JemallocStats - # On prod, Jemalloc reports sizes were ~2.5 MB: - # https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/15993#note_1014767214 - # We configured 1GB emptyDir per pod: - # https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/merge_requests/1949 - # The pod will be evicted when the size limit is exceeded. We never want this to happen, for availability. - # - # With the default, we have a headroom (250*2.5MB=625<1000 MB) to fit into configured emptyDir. - # It would allow us to keep 3+ days worth of reports for 6 workers running every 2 hours: 3*6*12=216<250 - # - # The cleanup logic will be redundant after we'll implement the uploads, which would perform the cleanup. - DEFAULT_MAX_REPORTS_STORED = 250 - - def initialize(reports_path:) - @reports_path = reports_path - - # Store report in tmp subdir while it is still streaming. - # This will clearly separate finished reports from the files we are still writing to. - @tmp_dir = File.join(@reports_path, 'tmp') - FileUtils.mkdir_p(@tmp_dir) + def name + 'jemalloc_stats' end - def run + def run(writer) return unless active? - Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, filename_label: worker_id).tap do - cleanup - end + Gitlab::Memory::Jemalloc.dump_stats(writer) end def active? Feature.enabled?(:report_jemalloc_stats, type: :ops) end - - private - - attr_reader :reports_path - - def cleanup - reports_files_modified_order[0...-max_reports_stored].each do |f| - File.unlink(f) if File.exist?(f) - rescue Errno::ENOENT - # Path does not exist: Ignore. We already check `File.exist?` - # Rescue to be extra safe, because each worker could perform a cleanup - end - end - - def reports_files_modified_order - pattern = File.join(reports_path, "#{Gitlab::Memory::Jemalloc::FILENAME_PREFIX}*") - - Dir.glob(pattern).sort_by do |f| - test('M', f) - rescue Errno::ENOENT - # Path does not exist: Return any timestamp to proceed with the sort - Time.current - end - end - - def worker_id - ::Prometheus::PidProvider.worker_id - end - - def max_reports_stored - ENV["GITLAB_DIAGNOSTIC_REPORTS_JEMALLOC_MAX_REPORTS_STORED"] || DEFAULT_MAX_REPORTS_STORED - end end end end diff --git a/lib/gitlab/memory/reports_daemon.rb b/lib/gitlab/memory/reports_daemon.rb index 0dfc31235e7..9bbfe81116d 100644 --- a/lib/gitlab/memory/reports_daemon.rb +++ b/lib/gitlab/memory/reports_daemon.rb @@ -7,9 +7,7 @@ module Gitlab DEFAULT_SLEEP_MAX_DELTA_S = 600 # 0..10 minutes DEFAULT_SLEEP_BETWEEN_REPORTS_S = 120 # 2 minutes - DEFAULT_REPORTS_PATH = Dir.tmpdir - - def initialize(**options) + def initialize(reporter: nil, reports: nil, **options) super @alive = true @@ -21,31 +19,20 @@ module Gitlab @sleep_between_reports_s = ENV['GITLAB_DIAGNOSTIC_REPORTS_SLEEP_BETWEEN_REPORTS_S']&.to_i || DEFAULT_SLEEP_BETWEEN_REPORTS_S - @reports_path = - ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"] || DEFAULT_REPORTS_PATH - - @reports = [Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path)] - - init_prometheus_metrics + @reporter = reporter || Reporter.new + @reports = reports || [ + Gitlab::Memory::Reports::JemallocStats.new + ] end - attr_reader :sleep_s, :sleep_max_delta_s, :sleep_between_reports_s, :reports_path + attr_reader :sleep_s, :sleep_max_delta_s, :sleep_between_reports_s def run_thread while alive sleep interval_with_jitter reports.select(&:active?).each do |report| - start_monotonic_time = Gitlab::Metrics::System.monotonic_time - start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time - - file_path = report.run - - cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) - duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time - - log_report(label: report_label(report), cpu_s: cpu_s, duration_s: duration_s, size: file_size(file_path)) - @report_duration_counter.increment({ report: report_label(report) }, duration_s) + @reporter.run_report(report) sleep sleep_between_reports_s end @@ -62,45 +49,9 @@ module Gitlab sleep_s + rand(sleep_max_delta_s) end - def log_report(label:, duration_s:, cpu_s:, size:) - Gitlab::AppLogger.info( - message: 'finished', - pid: $$, - worker_id: worker_id, - perf_report: label, - duration_s: duration_s.round(2), - cpu_s: cpu_s.round(2), - perf_report_size_bytes: size - ) - end - - def worker_id - ::Prometheus::PidProvider.worker_id - end - - def report_label(report) - report.class.to_s.demodulize.underscore - end - def stop_working @alive = false end - - def init_prometheus_metrics - default_labels = { pid: worker_id } - - @report_duration_counter = Gitlab::Metrics.counter( - :gitlab_diag_report_duration_seconds_total, - 'Total time elapsed for running diagnostic report', - default_labels - ) - end - - def file_size(file_path) - File.size(file_path.to_s) - rescue Errno::ENOENT - 0 - end end end end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 19dfc640b5d..25af5bd781a 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -50,27 +50,17 @@ module Gitlab def initialize @configuration = Configuration.new @alive = true - - init_prometheus_metrics end ## - # Configuration for Watchdog, use like: - # - # watchdog.configure do |config| - # config.handler = Gitlab::Memory::Watchdog::TermProcessHandler - # config.sleep_time_seconds = 60 - # config.logger = Gitlab::AppLogger - # config.monitors do |stack| - # stack.push MyMonitorClass, args*, max_strikes:, kwargs**, &block - # end - # end + # Configuration for Watchdog, see Gitlab::Memory::Watchdog::Configurator + # for examples. def configure - yield @configuration + yield configuration end def call - logger.info(log_labels.merge(message: 'started')) + event_reporter.started(log_labels) while @alive sleep(sleep_time_seconds) @@ -78,35 +68,45 @@ module Gitlab monitor if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) end - logger.info(log_labels.merge(message: 'stopped')) + event_reporter.stopped(log_labels(memwd_reason: @reason).compact) end - def stop + def stop(reason: nil) + @reason = reason @alive = false end private + attr_reader :configuration + + delegate :event_reporter, :monitors, :sleep_time_seconds, to: :configuration + def monitor - @configuration.monitors.call_each do |result| + if monitors.empty? + stop(reason: 'monitors are not configured') + return + end + + monitors.call_each do |result| break unless @alive next unless result.threshold_violated? - @counter_violations.increment(reason: result.monitor_name) + event_reporter.threshold_violated(result.monitor_name) next unless result.strikes_exceeded? - @alive = !memory_limit_exceeded_callback(result.monitor_name, result.payload) + strike_exceeded_callback(result.monitor_name, result.payload) end end - def memory_limit_exceeded_callback(monitor_name, monitor_payload) - all_labels = log_labels.merge(monitor_payload) - logger.warn(all_labels) - @counter_violations_handled.increment(reason: monitor_name) + def strike_exceeded_callback(monitor_name, monitor_payload) + event_reporter.strikes_exceeded(monitor_name, log_labels(monitor_payload)) + + Gitlab::Memory::Reports::HeapDump.enqueue! - handler.call + stop(reason: 'successfully handled') if handler.call end def handler @@ -114,46 +114,13 @@ module Gitlab # all that happens is we collect logs and Prometheus events for fragmentation violations. return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) - @configuration.handler - end - - def logger - @configuration.logger + configuration.handler end - def sleep_time_seconds - @configuration.sleep_time_seconds - end - - def log_labels - { - pid: $$, - worker_id: worker_id, + def log_labels(extra = {}) + extra.merge( memwd_handler_class: handler.class.name, - memwd_sleep_time_s: sleep_time_seconds, - memwd_rss_bytes: process_rss_bytes - } - end - - def process_rss_bytes - Gitlab::Metrics::System.memory_usage_rss[:total] - end - - def worker_id - ::Prometheus::PidProvider.worker_id - end - - def init_prometheus_metrics - default_labels = { pid: worker_id } - @counter_violations = Gitlab::Metrics.counter( - :gitlab_memwd_violations_total, - 'Total number of times a Ruby process violated a memory threshold', - default_labels - ) - @counter_violations_handled = Gitlab::Metrics.counter( - :gitlab_memwd_violations_handled_total, - 'Total number of times Ruby process memory violations were handled', - default_labels + memwd_sleep_time_s: sleep_time_seconds ) end end diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb index 793f75adf59..5c459220be8 100644 --- a/lib/gitlab/memory/watchdog/configuration.rb +++ b/lib/gitlab/memory/watchdog/configuration.rb @@ -10,7 +10,6 @@ module Gitlab end def push(monitor_class, *args, **kwargs, &block) - remove(monitor_class) @monitors.push(build_monitor_state(monitor_class, *args, **kwargs, &block)) end @@ -20,16 +19,17 @@ module Gitlab end end - private - - def remove(monitor_class) - @monitors.delete_if { |monitor| monitor.monitor_class == monitor_class } + def empty? + @monitors.empty? end - def build_monitor_state(monitor_class, *args, max_strikes:, **kwargs, &block) + private + + def build_monitor_state(monitor_class, *args, max_strikes:, monitor_name: nil, **kwargs, &block) monitor = build_monitor(monitor_class, *args, **kwargs, &block) + monitor_name ||= monitor_class.name.demodulize.underscore - Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes) + Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes, monitor_name: monitor_name) end def build_monitor(monitor_class, *args, **kwargs, &block) @@ -39,7 +39,7 @@ module Gitlab DEFAULT_SLEEP_TIME_SECONDS = 60 - attr_writer :logger, :handler, :sleep_time_seconds + attr_writer :event_reporter, :handler, :sleep_time_seconds def monitors @monitor_stack ||= MonitorStack.new @@ -51,8 +51,8 @@ module Gitlab @handler ||= NullHandler.instance end - def logger - @logger ||= Gitlab::Logger.new($stdout) + def event_reporter + @event_reporter ||= EventReporter.new end # Used to control the frequency with which the watchdog will wake up and poll the GC. diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb index 82b1b02b63f..04c04cbde02 100644 --- a/lib/gitlab/memory/watchdog/configurator.rb +++ b/lib/gitlab/memory/watchdog/configurator.rb @@ -4,36 +4,43 @@ module Gitlab module Memory class Watchdog class Configurator + DEFAULT_PUMA_WORKER_RSS_LIMIT_MB = 1200 + DEFAULT_SLEEP_INTERVAL_S = 60 + DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S = 3 + MIN_SIDEKIQ_SLEEP_INTERVAL_S = 2 + DEFAULT_MAX_STRIKES = 5 + DEFAULT_MAX_HEAP_FRAG = 0.5 + DEFAULT_MAX_MEM_GROWTH = 3.0 + # grace_time / sleep_interval = max_strikes allowed for Sidekiq process to violate defined limits. + DEFAULT_SIDEKIQ_GRACE_TIME_S = 300 + class << self def configure_for_puma - lambda do |config| - sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', 60).to_i - config.logger = Gitlab::AppLogger + ->(config) do config.handler = Gitlab::Memory::Watchdog::PumaHandler.new - config.sleep_time_seconds = sleep_time_seconds + config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', DEFAULT_SLEEP_INTERVAL_S).to_i config.monitors(&configure_monitors_for_puma) end end def configure_for_sidekiq - lambda do |config| - sleep_time_seconds = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max - config.logger = Sidekiq.logger + ->(config) do config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new - config.sleep_time_seconds = sleep_time_seconds + config.sleep_time_seconds = sidekiq_sleep_time config.monitors(&configure_monitors_for_sidekiq) + config.event_reporter = SidekiqEventReporter.new end end private def configure_monitors_for_puma - lambda do |stack| - max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', 5).to_i + ->(stack) do + max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', DEFAULT_MAX_STRIKES).to_i if Gitlab::Utils.to_boolean(ENV['DISABLE_PUMA_WORKER_KILLER']) - max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', 0.5).to_f - max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', 3.0).to_f + max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', DEFAULT_MAX_HEAP_FRAG).to_f + max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', DEFAULT_MAX_MEM_GROWTH).to_f # stack.push MonitorClass, args*, max_strikes:, kwargs**, &block stack.push Gitlab::Memory::Watchdog::Monitor::HeapFragmentation, @@ -44,17 +51,44 @@ module Gitlab max_mem_growth: max_mem_growth, max_strikes: max_strikes else - memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', 1200).to_i + memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', DEFAULT_PUMA_WORKER_RSS_LIMIT_MB).to_i stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, - memory_limit: memory_limit.megabytes, + memory_limit_bytes: memory_limit.megabytes, max_strikes: max_strikes end end end + def sidekiq_sleep_time + [ + ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S).to_i, + MIN_SIDEKIQ_SLEEP_INTERVAL_S + ].max + end + def configure_monitors_for_sidekiq - # NOP - At the moment we don't run watchdog for Sidekiq + ->(stack) do + if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero? + soft_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.kilobytes + grace_time = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', DEFAULT_SIDEKIQ_GRACE_TIME_S).to_i + max_strikes = grace_time / sidekiq_sleep_time + + stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, + memory_limit_bytes: soft_limit_bytes, + max_strikes: max_strikes.to_i, + monitor_name: :rss_memory_soft_limit + end + + if ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.nonzero? + hard_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.kilobytes + + stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, + memory_limit_bytes: hard_limit_bytes, + max_strikes: 0, + monitor_name: :rss_memory_hard_limit + end + end end end end diff --git a/lib/gitlab/memory/watchdog/event_reporter.rb b/lib/gitlab/memory/watchdog/event_reporter.rb new file mode 100644 index 00000000000..c37426cb660 --- /dev/null +++ b/lib/gitlab/memory/watchdog/event_reporter.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class EventReporter + include ::Gitlab::Utils::StrongMemoize + + attr_reader :logger + + def initialize(logger: Gitlab::AppLogger) + @logger = logger + init_prometheus_metrics + end + + def started(labels = {}) + logger.info(message: 'started', **log_labels(labels)) + end + + def stopped(labels = {}) + logger.info(message: 'stopped', **log_labels(labels)) + end + + def threshold_violated(monitor_name) + @counter_violations.increment(reason: monitor_name) + end + + def strikes_exceeded(monitor_name, labels = {}) + logger.warn(log_labels(labels)) + + @counter_violations_handled.increment(reason: monitor_name) + end + + private + + def log_labels(extra = {}) + extra.merge( + pid: $$, + worker_id: worker_id, + memwd_rss_bytes: process_rss_bytes + ) + end + + def process_rss_bytes + Gitlab::Metrics::System.memory_usage_rss[:total] + end + + def worker_id + ::Prometheus::PidProvider.worker_id + end + + def init_prometheus_metrics + default_labels = { pid: worker_id } + @counter_violations = Gitlab::Metrics.counter( + :gitlab_memwd_violations_total, + 'Total number of times a Ruby process violated a memory threshold', + default_labels + ) + @counter_violations_handled = Gitlab::Metrics.counter( + :gitlab_memwd_violations_handled_total, + 'Total number of times Ruby process memory violations were handled', + default_labels + ) + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb index 8f230980eac..ce99b68464e 100644 --- a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb +++ b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb @@ -4,10 +4,7 @@ module Gitlab module Memory class Watchdog module Monitor - # A monitor that observes Ruby heap fragmentation and calls - # memory_violation_callback when the Ruby heap has been fragmented for an extended - # period of time. - # + # A monitor that observes Ruby heap fragmentation. # See Gitlab::Metrics::Memory for how heap fragmentation is defined. class HeapFragmentation attr_reader :max_heap_fragmentation diff --git a/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb index 3e7de024630..ac71592294c 100644 --- a/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb +++ b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb @@ -5,29 +5,38 @@ module Gitlab class Watchdog module Monitor class RssMemoryLimit - attr_reader :memory_limit + attr_reader :memory_limit_bytes - def initialize(memory_limit:) - @memory_limit = memory_limit + def initialize(memory_limit_bytes:) + @memory_limit_bytes = memory_limit_bytes + init_memory_limit_metrics end def call - worker_rss = Gitlab::Metrics::System.memory_usage_rss[:total] + worker_rss_bytes = Gitlab::Metrics::System.memory_usage_rss[:total] - return { threshold_violated: false, payload: {} } if worker_rss <= memory_limit + return { threshold_violated: false, payload: {} } if worker_rss_bytes <= memory_limit_bytes - { threshold_violated: true, payload: payload(worker_rss, memory_limit) } + { threshold_violated: true, payload: payload(worker_rss_bytes, memory_limit_bytes) } end private - def payload(worker_rss, memory_limit) + def payload(worker_rss_bytes, memory_limit_bytes) { message: 'rss memory limit exceeded', - memwd_rss_bytes: worker_rss, - memwd_max_rss_bytes: memory_limit + memwd_rss_bytes: worker_rss_bytes, + memwd_max_rss_bytes: memory_limit_bytes } end + + def init_memory_limit_metrics + rss_memory_limit = Gitlab::Metrics.gauge( + :gitlab_memwd_max_memory_limit, + 'The configured fixed limit for rss memory' + ) + rss_memory_limit.set({}, memory_limit_bytes) + end end end end diff --git a/lib/gitlab/memory/watchdog/monitor_state.rb b/lib/gitlab/memory/watchdog/monitor_state.rb index 73be5de3e45..bb083fedf2c 100644 --- a/lib/gitlab/memory/watchdog/monitor_state.rb +++ b/lib/gitlab/memory/watchdog/monitor_state.rb @@ -5,12 +5,12 @@ module Gitlab class Watchdog class MonitorState class Result - attr_reader :payload + attr_reader :payload, :monitor_name - def initialize(strikes_exceeded:, threshold_violated:, monitor_class:, payload: ) + def initialize(strikes_exceeded:, threshold_violated:, monitor_name:, payload:) @strikes_exceeded = strikes_exceeded @threshold_violated = threshold_violated - @monitor_class = monitor_class + @monitor_name = monitor_name.to_s.to_sym @payload = payload end @@ -21,15 +21,12 @@ module Gitlab def threshold_violated? @threshold_violated end - - def monitor_name - @monitor_class.name.demodulize.underscore.to_sym - end end - def initialize(monitor, max_strikes:) + def initialize(monitor, max_strikes:, monitor_name:) @monitor = monitor @max_strikes = max_strikes + @monitor_name = monitor_name @strikes = 0 end @@ -47,16 +44,12 @@ module Gitlab build_result(monitor_result) end - def monitor_class - @monitor.class - end - private def build_result(monitor_result) Result.new( strikes_exceeded: strikes_exceeded?, - monitor_class: monitor_class, + monitor_name: @monitor_name, threshold_violated: monitor_result[:threshold_violated], payload: payload.merge(monitor_result[:payload])) end diff --git a/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb b/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb new file mode 100644 index 00000000000..473ed1b8094 --- /dev/null +++ b/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class SidekiqEventReporter + include ::Gitlab::Utils::StrongMemoize + + delegate :threshold_violated, :started, :stopped, :logger, to: :event_reporter + + def initialize(logger: ::Sidekiq.logger) + @event_reporter = EventReporter.new(logger: logger) + @sidekiq_daemon_monitor = Gitlab::SidekiqDaemon::Monitor.instance + init_prometheus_metrics + end + + def strikes_exceeded(monitor_name, labels = {}) + running_jobs = fetch_running_jobs + labels[:running_jobs] = running_jobs + increment_worker_counters(running_jobs) + + event_reporter.strikes_exceeded(monitor_name, labels) + end + + private + + attr_reader :event_reporter + + def fetch_running_jobs + @sidekiq_daemon_monitor.jobs.map do |jid, job| + { + jid: jid, + worker_class: job[:worker_class].name + } + end + end + + def increment_worker_counters(running_jobs) + running_jobs.each do |job| + @sidekiq_watchdog_running_jobs_counter.increment({ worker_class: job[:worker_class] }) + end + end + + def init_prometheus_metrics + @sidekiq_watchdog_running_jobs_counter = ::Gitlab::Metrics.counter( + :sidekiq_watchdog_running_jobs_total, + 'Current running jobs when limit was reached' + ) + end + end + end + end +end diff --git a/lib/gitlab/merge_requests/commit_message_generator.rb b/lib/gitlab/merge_requests/message_generator.rb index ef5c63925c2..5113fbdcd7b 100644 --- a/lib/gitlab/merge_requests/commit_message_generator.rb +++ b/lib/gitlab/merge_requests/message_generator.rb @@ -1,28 +1,41 @@ # frozen_string_literal: true module Gitlab module MergeRequests - class CommitMessageGenerator + class MessageGenerator def initialize(merge_request:, current_user:) @merge_request = merge_request @current_user = @merge_request.metrics&.merged_by || @merge_request.merge_user || current_user end - def merge_message + def merge_commit_message return unless @merge_request.target_project.merge_commit_template.present? - replace_placeholders(@merge_request.target_project.merge_commit_template) + replace_placeholders(@merge_request.target_project.merge_commit_template, allowed_placeholders: PLACEHOLDERS) end - def squash_message + def squash_commit_message return unless @merge_request.target_project.squash_commit_template.present? - replace_placeholders(@merge_request.target_project.squash_commit_template, squash: true) + replace_placeholders( + @merge_request.target_project.squash_commit_template, + allowed_placeholders: PLACEHOLDERS, + squash: true + ) + end + + def new_mr_description + return unless @merge_request.description.present? + + replace_placeholders( + @merge_request.description, + allowed_placeholders: ALLOWED_NEW_MR_PLACEHOLDERS, + keep_carriage_return: true + ) end private - attr_reader :merge_request - attr_reader :current_user + attr_reader :merge_request, :current_user PLACEHOLDERS = { 'source_branch' => ->(merge_request, _, _) { merge_request.source_branch.to_s }, @@ -38,10 +51,25 @@ module Gitlab end, 'description' => ->(merge_request, _, _) { merge_request.description }, 'reference' => ->(merge_request, _, _) { merge_request.to_reference(full: true) }, - 'first_commit' => -> (merge_request, _, _) { merge_request.first_commit&.safe_message&.strip }, - 'first_multiline_commit' => -> (merge_request, _, _) { merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title }, + 'first_commit' => -> (merge_request, _, _) { + return unless merge_request.persisted? || merge_request.compare_commits.present? + + merge_request.first_commit&.safe_message&.strip + }, + 'first_multiline_commit' => -> (merge_request, _, _) { + merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title + }, 'url' => ->(merge_request, _, _) { Gitlab::UrlBuilder.build(merge_request) }, - 'approved_by' => ->(merge_request, _, _) { merge_request.approved_by_users.map { |user| "Approved-by: #{user.name} <#{user.commit_email_or_default}>" }.join("\n") }, + 'reviewed_by' => ->(merge_request, _, _) { + merge_request.reviewed_by_users + .map { |user| "Reviewed-by: #{user.name} <#{user.commit_email_or_default}>" } + .join("\n") + }, + 'approved_by' => ->(merge_request, _, _) { + merge_request.approved_by_users + .map { |user| "Approved-by: #{user.name} <#{user.commit_email_or_default}>" } + .join("\n") + }, 'merged_by' => ->(_, user, _) { "#{user&.name} <#{user&.commit_email_or_default}>" }, 'co_authored_by' => ->(merge_request, merged_by, squash) do commit_author = squash ? merge_request.author : merged_by @@ -66,15 +94,34 @@ module Gitlab end }.freeze + # A new merge request that is in the process of being created and hasn't + # been persisted to the database. + # + # Limit the placeholders to a subset of the available ones where the + # placeholders wouldn't make sense in context. Disallowed placeholders + # will be replaced with an empty string. + ALLOWED_NEW_MR_PLACEHOLDERS = %w[ + source_branch + target_branch + first_commit + first_multiline_commit + co_authored_by + all_commits + ].freeze + PLACEHOLDERS_COMBINED_REGEX = /%{(#{Regexp.union(PLACEHOLDERS.keys)})}/.freeze - def replace_placeholders(message, squash: false) + def replace_placeholders(message, allowed_placeholders: [], squash: false, keep_carriage_return: false) # Convert CRLF to LF. - message = message.delete("\r") + message = message.delete("\r") unless keep_carriage_return used_variables = message.scan(PLACEHOLDERS_COMBINED_REGEX).map { |value| value[0] }.uniq values = used_variables.to_h do |variable_name| - ["%{#{variable_name}}", PLACEHOLDERS[variable_name].call(merge_request, current_user, squash)] + replacement = if allowed_placeholders.include?(variable_name) + PLACEHOLDERS[variable_name].call(merge_request, current_user, squash) + end + + ["%{#{variable_name}}", replacement] end names_of_empty_variables = values.filter_map { |name, value| name if value.blank? } diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 6d7ecb53ec3..e99761a0459 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -20,6 +20,10 @@ module Gitlab status.to_i.between?(200, 499) end + def self.server_error?(status) + status.to_i >= 500 + end + # Tracks an event. # # See `Gitlab::Metrics::Transaction#add_event` for more details. diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb index e2b43798b22..531e4079632 100644 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -63,13 +63,11 @@ module Gitlab end def prometheus_metrics_attributes - @prometheus_metrics_attributes ||= begin - Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( - dashboard_hash, + @prometheus_metrics_attributes ||= Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( + dashboard_hash, project: project, dashboard_path: dashboard_path - ).execute - end + ).execute end end end diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb index 1e8dc059968..57b4b5c068d 100644 --- a/lib/gitlab/metrics/dashboard/validator.rb +++ b/lib/gitlab/metrics/dashboard/validator.rb @@ -16,6 +16,8 @@ module Gitlab errors.empty? || raise(errors.first) end + private + def errors(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) Validator::Client .new(content, schema_path, dashboard_path: dashboard_path, project: project) diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index 200c6eb4043..fc2e805047a 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -14,9 +14,6 @@ module Gitlab def initialize_slis! Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) - - return unless Feature.enabled?(:global_search_error_rate_sli) - Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:global_search, possible_labels) end @@ -28,8 +25,6 @@ module Gitlab end def record_error_rate(error:, search_type:, search_level:, search_scope:) - return unless Feature.enabled?(:global_search_error_rate_sli) - Gitlab::Metrics::Sli::ErrorRate[:global_search].increment( labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), error: error diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb index 71da0085c8c..9fd4eec479e 100644 --- a/lib/gitlab/metrics/rails_slis.rb +++ b/lib/gitlab/metrics/rails_slis.rb @@ -6,6 +6,7 @@ module Gitlab class << self def initialize_request_slis! Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, possible_request_labels) + initialize_rails_request_error_rate Gitlab::Metrics::Sli::Apdex.initialize_sli(:graphql_query, possible_graphql_query_labels) end @@ -13,6 +14,10 @@ module Gitlab Gitlab::Metrics::Sli::Apdex[:rails_request] end + def request_error_rate + Gitlab::Metrics::Sli::ErrorRate[:rails_request] + end + def graphql_query_apdex Gitlab::Metrics::Sli::Apdex[:graphql_query] end @@ -58,6 +63,12 @@ module Gitlab } end end + + def initialize_rails_request_error_rate + return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development) + + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:rails_request, possible_request_labels) + end end end end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index d7fe983c553..0172de8731d 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -75,15 +75,17 @@ module Gitlab begin status, headers, body = @app.call(env) + return [status, headers, body] if health_endpoint - elapsed = ::Gitlab::Metrics::System.monotonic_time - started - - if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status) + urgency = urgency_for_env(env) + if ::Gitlab::Metrics.record_duration_for_status?(status) + elapsed = ::Gitlab::Metrics::System.monotonic_time - started self.class.http_request_duration_seconds.observe({ method: method }, elapsed) - - record_apdex(env, elapsed) + record_apdex(urgency, elapsed) end + record_error(urgency, status) + [status, headers, body] rescue StandardError self.class.rack_uncaught_errors_count.increment @@ -115,15 +117,22 @@ module Gitlab ::Gitlab::ApplicationContext.current_context_attribute(:caller_id) end - def record_apdex(env, elapsed) - urgency = urgency_for_env(env) - + def record_apdex(urgency, elapsed) Gitlab::Metrics::RailsSlis.request_apdex.increment( labels: labels_from_context.merge(request_urgency: urgency.name), success: elapsed < urgency.duration ) end + def record_error(urgency, status) + return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development) + + Gitlab::Metrics::RailsSlis.request_error_rate.increment( + labels: labels_from_context.merge(request_urgency: urgency.name), + error: ::Gitlab::Metrics.server_error?(status) + ) + end + def labels_from_context { feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT, diff --git a/lib/gitlab/metrics/subscribers/ldap.rb b/lib/gitlab/metrics/subscribers/ldap.rb new file mode 100644 index 00000000000..9cac5f41090 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/ldap.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + class Ldap < ActiveSupport::Subscriber + # This namespace is configured in the Net::LDAP library, and appears + # at the end of the event key, e.g. `open.net_ldap` + attach_to :net_ldap + + COUNTER = :net_ldap_count + DURATION = :net_ldap_duration_s + + # Assembled from methods that are instrumented inside Net::LDAP + OBSERVABLE_EVENTS = %i[ + open + bind + add + modify + modify_password + rename + delete + search + ].freeze + + class << self + # @return [Integer] the total number of LDAP requests + def count + Gitlab::SafeRequestStore[COUNTER].to_i + end + + # @return [Float] the total duration spent on LDAP requests + def duration + Gitlab::SafeRequestStore[DURATION].to_f + end + + # Used in Gitlab::InstrumentationHelper to merge the LDAP stats + # into the log output + # + # @return [Hash<Integer, Float>] a hash of the stored statistics + def payload + { + net_ldap_count: count, + net_ldap_duration_s: duration + } + end + end + + # Called when an event is triggered in ActiveSupport::Notifications + # + # This method is aliased to the various events triggered by the + # Net::LDAP library, as the method will be called by those names + # when triggered. + # + # It stores statistics in the request for output to logs, and also + # resubmits the event data into Prometheus for monitoring purposes. + def observe_event(event) + add_to_request_store(event) + expose_metrics(event) + end + + OBSERVABLE_EVENTS.each do |event| + alias_method event, :observe_event + end + + private + + def current_transaction + ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current + end + + # Track these events as statistics for the current requests, for logging purposes + def add_to_request_store(event) + return unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 + Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + event.duration.to_f + end + + # Converts the observed events into Prometheus metrics + def expose_metrics(event) + return unless current_transaction + + # event.name will be, for example, `search.net_ldap` + # and so we only want the first part, which is the + # true name of the event + labels = { name: event.name.split(".").first } + + current_transaction.increment(:gitlab_net_ldap_total, 1, labels) do + docstring 'Net::LDAP calls' + label_keys labels.keys + end + + current_transaction.observe(:gitlab_net_ldap_duration_seconds, event.duration, labels) do + docstring 'Net::LDAP time' + buckets [0.001, 0.01, 0.1, 1.0, 2.0, 5.0] + label_keys labels.keys + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index b5e087d107b..b12db9df66d 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -8,6 +8,17 @@ module Gitlab class RailsCache < ActiveSupport::Subscriber attach_to :active_support + def cache_read_multi(event) + observe(:read_multi, event.duration) + + return unless current_transaction + + current_transaction.observe(:gitlab_cache_read_multikey_count, event.payload[:key].size) do + buckets [10, 50, 100, 1000] + docstring 'Number of keys for mget in read_multi/fetch_multi' + end + end + def cache_read(event) observe(:read, event.duration) diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index f66dfe44054..80916eab5ac 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -4,7 +4,18 @@ module Gitlab module Middleware class CompressedJson COLLECTOR_PATH = '/api/v4/error_tracking/collector' + PACKAGES_PATH = %r{ + \A/api/v4/ (?# prefix) + (?:projects/ + (?<project_id> + .+ (?# at least one character) + )/ + )? (?# projects segment) + packages/npm/-/npm/v1/security/ + (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) + }xi.freeze MAXIMUM_BODY_SIZE = 200.kilobytes.to_i + UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi.freeze def initialize(app) @app = app @@ -60,7 +71,21 @@ module Gitlab end def match_path?(env) - env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) + env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) || + match_packages_path?(env) + end + + def match_packages_path?(env) + match_data = env['PATH_INFO'].delete_prefix(relative_url).match(PACKAGES_PATH) + return false unless match_data + + return true unless match_data[:project_id] # instance level endpoint was matched + + url_encoded?(match_data[:project_id]) + end + + def url_encoded?(project_id) + project_id !~ UNSAFE_CHARACTERS end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index dcbb4557377..13f7ab36823 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -72,8 +72,8 @@ module Gitlab "#{project_url}.git" end - meta_import_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{repository_url}" - meta_source_tag = tag :meta, name: 'go-source', content: "#{import_prefix} #{project_url} #{project_url}/-/tree/#{branch}{/dir} #{project_url}/-/blob/#{branch}{/dir}/{file}#L{line}" + meta_import_tag = tag.meta(name: 'go-import', content: "#{import_prefix} git #{repository_url}") + meta_source_tag = tag.meta(name: 'go-source', content: "#{import_prefix} #{project_url} #{project_url}/-/tree/#{branch}{/dir} #{project_url}/-/blob/#{branch}{/dir}/{file}#L{line}") head_tag = content_tag :head, meta_import_tag + meta_source_tag html_tag = content_tag :html, head_tag + body_tag [html_tag, 200] diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 0dd6b8a809c..2368ea3ad28 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -3,18 +3,34 @@ module Gitlab # Parser/renderer for markups without other special support code. module OtherMarkup + RENDER_TIMEOUT = 10.seconds + # Public: Converts the provided markup into HTML. # # input - the source text in a markup format # def self.render(file_name, input, context) - html = GitHub::Markup.render(file_name, input) - .force_encoding(input.encoding) + html = render_markup(file_name, input, context).force_encoding(input.encoding) + context[:pipeline] ||= :markup html = Banzai.render(html, context) - html.html_safe end + + def self.render_markup(file_name, input, context) + Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { GitHub::Markup.render(file_name, input) } + rescue Timeout::Error => e + class_name = name.demodulize + timeout_counter.increment(source: class_name) + Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name, + file_name: file_name) + + ActionController::Base.helpers.simple_format(input) + end + + def self.timeout_counter + Gitlab::Metrics.counter(:banzai_filter_timeouts_total, 'Count of the Banzai filters that time out') + end end end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb index 187d5f907e4..be39e52b342 100644 --- a/lib/gitlab/pages/cache_control.rb +++ b/lib/gitlab/pages/cache_control.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'set' + module Gitlab module Pages class CacheControl @@ -9,7 +11,9 @@ module Gitlab # To avoid delivering expired deployment URL in the cached payload, # use a longer expiration time in the deployment URL DEPLOYMENT_EXPIRATION = (EXPIRE + 12.hours) - CACHE_KEY_FORMAT = 'pages_domain_for_%{type}_%{id}_%{settings}' + + SETTINGS_CACHE_KEY = 'pages_domain_for_%{type}_%{id}' + PAYLOAD_CACHE_KEY = '%{settings_cache_key}_%{settings_hash}' class << self def for_project(project_id) @@ -29,29 +33,63 @@ module Gitlab end def cache_key - strong_memoize(:cache_key) do - CACHE_KEY_FORMAT % { - type: @type, - id: @id, - settings: settings - } + strong_memoize(:payload_cache_key) do + cache_settings_hash! + + payload_cache_key_for(settings_hash) end end + # Invalidates the cache. + # + # Since rails nodes and sidekiq nodes have different application settings, + # and the invalidation happens in a sidekiq node, we have to use the + # cached settings hash to build the payload cache key to be invalidated. def clear_cache - Rails.cache.delete(cache_key) + keys = cached_settings_hashes + .map { |hash| payload_cache_key_for(hash) } + .push(settings_cache_key) + + Rails.cache.delete_multi(keys) end private - def settings - values = ::Gitlab.config.pages.dup + # Since rails nodes and sidekiq nodes have different application settings, + # we cache the application settings hash when creating the payload cache + # so we can use these values to invalidate the cache in a sidekiq node later. + def cache_settings_hash! + cached = cached_settings_hashes.to_set + Rails.cache.write(settings_cache_key, cached.add(settings_hash)) + end + + def cached_settings_hashes + Rails.cache.read(settings_cache_key) || [] + end + + def payload_cache_key_for(settings_hash) + PAYLOAD_CACHE_KEY % { + settings_cache_key: settings_cache_key, + settings_hash: settings_hash + } + end - values['app_settings'] = ::Gitlab::CurrentSettings.attributes.slice( - 'force_pages_access_control' - ) + def settings_cache_key + strong_memoize(:settings_cache_key) do + SETTINGS_CACHE_KEY % { type: @type, id: @id } + end + end - ::Digest::SHA256.hexdigest(values.inspect) + def settings_hash + strong_memoize(:settings_hash) do + values = ::Gitlab.config.pages.dup + + values['app_settings'] = ::Gitlab::CurrentSettings.attributes.slice( + 'force_pages_access_control' + ) + + ::Digest::SHA256.hexdigest(values.inspect) + end end end end diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 2d9fb0a50fc..199ec16d4df 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -22,7 +22,7 @@ module Gitlab def self.available?(cursor_based_request_context, relation) available_for_type?(relation) && - order_satisfied?(relation, cursor_based_request_context) + order_satisfied?(relation, cursor_based_request_context) end def self.enforced_for_type?(relation) diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 00304f48dc5..a98199dae2e 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -11,15 +11,17 @@ module Gitlab @request_context = request_context end - def paginate(relation, exclude_total_headers: false, skip_default_order: false) - paginate_with_limit_optimization(add_default_order(relation, skip_default_order: skip_default_order)).tap do |data| + def paginate(relation, exclude_total_headers: false, skip_default_order: false, without_count: false) + ordered_relation = add_default_order(relation, skip_default_order: skip_default_order) + + paginate_with_limit_optimization(ordered_relation, without_count: without_count).tap do |data| add_pagination_headers(data, exclude_total_headers) end end private - def paginate_with_limit_optimization(relation) + def paginate_with_limit_optimization(relation, without_count:) pagination_data = if needs_pagination?(relation) relation.page(params[:page]).per(params[:per_page]) else @@ -28,8 +30,7 @@ module Gitlab return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - limited_total_count = pagination_data.total_count_with_limit - if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + if without_count || exceeeds_count?(pagination_data) # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` # We need to call `reset` because `without_count` relies on `@arel` being unmemoized pagination_data.reset.without_count @@ -78,6 +79,12 @@ module Gitlab # Ensure there is in total at least 1 page [paginated_data.total_pages, 1].max end + + def exceeeds_count?(paginated_data) + limited_total_count = paginated_data.total_count_with_limit + + limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + end end end end diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index 1ed341e1c26..b974c0b2c7f 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -26,7 +26,7 @@ module Gitlab # https://github.com/rails/rails/pull/42067 # # Let's keep our own implementation, until the issue is fixed - Module.instance_method(:prepend_features).bind(self).call(base) + Module.instance_method(:prepend_features).bind_call(self, base) if const_defined?(:ClassMethods) klass_methods = const_get(:ClassMethods, false) diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb index c842798ca74..4de9eaa9500 100644 --- a/lib/gitlab/phabricator_import/project_creator.rb +++ b/lib/gitlab/phabricator_import/project_creator.rb @@ -55,13 +55,11 @@ module Gitlab end def project_feature_attributes + # everything disabled except for issues @project_features_attributes ||= - begin - # everything disabled except for issues - ProjectFeature::FEATURES.to_h do |feature| - [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] - end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) - end + ProjectFeature::FEATURES.to_h do |feature| + [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] + end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) end def import_data diff --git a/lib/gitlab/process_management.rb b/lib/gitlab/process_management.rb index f8a1a3a97de..89ffd71c2d8 100644 --- a/lib/gitlab/process_management.rb +++ b/lib/gitlab/process_management.rb @@ -40,15 +40,6 @@ module Gitlab pids.each { |pid| signal(pid, signal) } end - # Waits for the given process to complete using a separate thread. - def self.wait_async(pid) - Thread.new do - Process.wait(pid) - rescue StandardError - nil # There is no reason to return `Errno::ECHILD` if it catches a `TypeError` - end - end - # Returns true if all the processes are alive. def self.all_alive?(pids) pids.each do |pid| diff --git a/lib/gitlab/process_supervisor.rb b/lib/gitlab/process_supervisor.rb index 714034f043d..09e923d1449 100644 --- a/lib/gitlab/process_supervisor.rb +++ b/lib/gitlab/process_supervisor.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative './daemon' + module Gitlab # Given a set of process IDs, the supervisor can monitor processes # for being alive and invoke a callback if some or all should go away. diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index f8a85f693bc..5af06e82c55 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -123,18 +123,18 @@ module Gitlab def self.with_custom_logger(logger) original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging - original_activerecord_logger = ApplicationRecord.logger + original_activerecord_logger = ActiveRecord::Base.logger original_actioncontroller_logger = ActionController::Base.logger if logger ActiveSupport::LogSubscriber.colorize_logging = false - ApplicationRecord.logger = logger + ActiveRecord::Base.logger = logger ActionController::Base.logger = logger end yield.tap do ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging - ApplicationRecord.logger = original_activerecord_logger + ActiveRecord::Base.logger = original_activerecord_logger ActionController::Base.logger = original_actioncontroller_logger end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 8cc96970ebd..13718e63b25 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -125,17 +125,15 @@ module Gitlab def wiki_blobs(limit: count_limit) return [] unless Ability.allowed?(@current_user, :read_wiki, @project) - @wiki_blobs ||= begin - if project.wiki_enabled? && query.present? - if project.wiki.empty? - [] - else - Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query, content_match_cutoff: limit) - end - else - [] - end - end + @wiki_blobs ||= if project.wiki_enabled? && query.present? + if project.wiki.empty? + [] + else + Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query, content_match_cutoff: limit) + end + else + [] + end end def notes @@ -195,3 +193,5 @@ module Gitlab end end end + +Gitlab::ProjectSearchResults.prepend_mod_with('Gitlab::ProjectSearchResults') diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 51a5bedc44b..9bc0001be81 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -60,6 +60,7 @@ module Gitlab ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/third-party-logos/dotnet.svg'), ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'), + ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/gitlab-org/project-templates/bridgetown'), ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby'), 'https://gitlab.com/pages/gatsby', 'illustrations/third-party-logos/gatsby.svg'), ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'), @@ -79,7 +80,8 @@ module Gitlab ProjectTemplate.new('tencent_serverless_framework', 'Tencent Serverless Framework/NextjsSSR', _('A project boilerplate for Tencent Serverless Framework that uses Next.js SSR'), 'https://gitlab.com/gitlab-org/project-templates/nextjsssr_demo', 'illustrations/logos/tencent_serverless_framework.svg'), ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), - ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux') + ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'), + ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/ochorocho/typo3-distribution', 'illustrations/logos/typo3.svg') ].freeze end # rubocop:enable Metrics/AbcSize diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index dda28ffdf90..6a5613ddd98 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -150,7 +150,7 @@ module Gitlab end def get(path, args) - Gitlab::HTTP.get(path, { query: args }.merge(http_options) ) + Gitlab::HTTP.get(path, { query: args }.merge(http_options)) rescue *Gitlab::HTTP::HTTP_ERRORS => e raise PrometheusClient::ConnectionError, e.message end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 0b37c80dc5f..a12457d89c9 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -84,7 +84,7 @@ module Gitlab current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) && find_labels.any? end - command :label do |labels_param| + command :label, :labels do |labels_param| run_label_command(labels: find_labels(labels_param), command: :label, updates_key: :add_label_ids) end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index e74c58e45b1..14e9e66e037 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -80,7 +80,7 @@ module Gitlab desc { _('Mark this issue as a duplicate of another issue') } explanation do |duplicate_reference| - _("Marks this issue as a duplicate of %{duplicate_reference}.") % { duplicate_reference: duplicate_reference } + _("Closes this issue. Marks as related to, and a duplicate of, %{duplicate_reference}.") % { duplicate_reference: duplicate_reference } end params '#issue' types Issue @@ -94,7 +94,7 @@ module Gitlab if canonical_issue.present? @updates[:canonical_issue_id] = canonical_issue.id - message = _("Marked this issue as a duplicate of %{duplicate_param}.") % { duplicate_param: duplicate_param } + message = _("Closed this issue. Marked as related to, and a duplicate of, %{duplicate_param}.") % { duplicate_param: duplicate_param } else message = _('Failed to mark this issue as a duplicate because referenced issue was not found.') end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 8b1ff5d298a..e549ee2e43a 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -92,7 +92,7 @@ module Gitlab types Issue, MergeRequest condition do quick_action_target.supports_milestone? && - current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) && + current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) && find_milestones(project, state: 'active').any? end parse_params do |milestone_param| @@ -156,7 +156,7 @@ module Gitlab types Issue, MergeRequest condition do quick_action_target.supports_time_tracking? && - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |raw_duration| Gitlab::TimeTrackingFormatter.parse(raw_duration) @@ -179,7 +179,7 @@ module Gitlab types Issue, MergeRequest condition do quick_action_target.supports_time_tracking? && - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end parse_params do |raw_time_date| Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index f5fb6b5af3d..bedbe9c0bff 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -49,7 +49,7 @@ module Gitlab # # - Retry-After: the remaining duration in seconds until the quota is # reset. This is a standardized HTTP header: - # https://tools.ietf.org/html/rfc7231#page-69 + # https://www.rfc-editor.org/rfc/rfc7231#page-69 # # - RateLimit-Reset: the point of time that the request quota is reset, in Unix time # diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index 08a5ddb6ad1..d7abacb5b67 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -79,96 +79,96 @@ module Gitlab def throttle_unauthenticated_api? api_request? && - !should_be_skipped? && - !frontend_request? && - !throttle_unauthenticated_packages_api? && - !throttle_unauthenticated_files_api? && - !throttle_unauthenticated_deprecated_api? && - Gitlab::Throttle.settings.throttle_unauthenticated_api_enabled && - unauthenticated? + !should_be_skipped? && + !frontend_request? && + !throttle_unauthenticated_packages_api? && + !throttle_unauthenticated_files_api? && + !throttle_unauthenticated_deprecated_api? && + Gitlab::Throttle.settings.throttle_unauthenticated_api_enabled && + unauthenticated? end def throttle_unauthenticated_web? (web_request? || frontend_request?) && - !should_be_skipped? && - # TODO: Column will be renamed in https://gitlab.com/gitlab-org/gitlab/-/issues/340031 - Gitlab::Throttle.settings.throttle_unauthenticated_enabled && - unauthenticated? + !should_be_skipped? && + # TODO: Column will be renamed in https://gitlab.com/gitlab-org/gitlab/-/issues/340031 + Gitlab::Throttle.settings.throttle_unauthenticated_enabled && + unauthenticated? end def throttle_authenticated_api? api_request? && - !frontend_request? && - !throttle_authenticated_packages_api? && - !throttle_authenticated_files_api? && - !throttle_authenticated_deprecated_api? && - Gitlab::Throttle.settings.throttle_authenticated_api_enabled + !frontend_request? && + !throttle_authenticated_packages_api? && + !throttle_authenticated_files_api? && + !throttle_authenticated_deprecated_api? && + Gitlab::Throttle.settings.throttle_authenticated_api_enabled end def throttle_authenticated_web? (web_request? || frontend_request?) && - !throttle_authenticated_git_lfs? && - Gitlab::Throttle.settings.throttle_authenticated_web_enabled + !throttle_authenticated_git_lfs? && + Gitlab::Throttle.settings.throttle_authenticated_web_enabled end def throttle_unauthenticated_protected_paths? post? && - !should_be_skipped? && - protected_path? && - Gitlab::Throttle.protected_paths_enabled? && - unauthenticated? + !should_be_skipped? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? && + unauthenticated? end def throttle_authenticated_protected_paths_api? post? && - api_request? && - protected_path? && - Gitlab::Throttle.protected_paths_enabled? + api_request? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? end def throttle_authenticated_protected_paths_web? post? && - web_request? && - protected_path? && - Gitlab::Throttle.protected_paths_enabled? + web_request? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? end def throttle_unauthenticated_packages_api? packages_api_path? && - Gitlab::Throttle.settings.throttle_unauthenticated_packages_api_enabled && - unauthenticated? + Gitlab::Throttle.settings.throttle_unauthenticated_packages_api_enabled && + unauthenticated? end def throttle_authenticated_packages_api? packages_api_path? && - Gitlab::Throttle.settings.throttle_authenticated_packages_api_enabled + Gitlab::Throttle.settings.throttle_authenticated_packages_api_enabled end def throttle_authenticated_git_lfs? git_lfs_path? && - Gitlab::Throttle.settings.throttle_authenticated_git_lfs_enabled + Gitlab::Throttle.settings.throttle_authenticated_git_lfs_enabled end def throttle_unauthenticated_files_api? files_api_path? && - Gitlab::Throttle.settings.throttle_unauthenticated_files_api_enabled && - unauthenticated? + Gitlab::Throttle.settings.throttle_unauthenticated_files_api_enabled && + unauthenticated? end def throttle_authenticated_files_api? files_api_path? && - Gitlab::Throttle.settings.throttle_authenticated_files_api_enabled + Gitlab::Throttle.settings.throttle_authenticated_files_api_enabled end def throttle_unauthenticated_deprecated_api? deprecated_api_request? && - Gitlab::Throttle.settings.throttle_unauthenticated_deprecated_api_enabled && - unauthenticated? + Gitlab::Throttle.settings.throttle_unauthenticated_deprecated_api_enabled && + unauthenticated? end def throttle_authenticated_deprecated_api? deprecated_api_request? && - Gitlab::Throttle.settings.throttle_authenticated_deprecated_api_enabled + Gitlab::Throttle.settings.throttle_authenticated_deprecated_api_enabled end private diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 12cb1fc6153..4f58bee49d0 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -10,6 +10,7 @@ module Gitlab 'Value not found on the redis primary store. Read from the redis secondary store successful.' end end + class PipelinedDiffError < StandardError def initialize(result_primary, result_secondary) @result_primary = result_primary @@ -22,6 +23,7 @@ module Gitlab "Result from the secondary: #{@result_secondary.inspect}." end end + class MethodMissingError < StandardError def message 'Method missing. Falling back to execute method on the redis secondary store.' diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 75dbccb965d..0e5389dc995 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -144,11 +144,20 @@ module Gitlab def redis_store_options config = raw_config_hash + config[:instrumentation_class] ||= self.class.instrumentation_class + + if config[:cluster].present? + config[:db] = 0 # Redis Cluster only supports db 0 + config + else + parse_redis_url(config) + end + end + + def parse_redis_url(config) redis_url = config.delete(:url) redis_uri = URI.parse(redis_url) - config[:instrumentation_class] ||= self.class.instrumentation_class - if redis_uri.scheme == 'unix' # Redis::Store does not handle Unix sockets well, so let's do it for them config[:path] = redis_uri.path @@ -178,7 +187,7 @@ module Gitlab { url: '' } end - if config_hash[:url].blank? + if config_hash[:url].blank? && config_hash[:cluster].blank? config_hash[:url] = legacy_fallback_urls[self.class.store_name] || legacy_fallback_urls[self.class.config_fallback.store_name] end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index c5798bec0d7..540394f04bd 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,7 +4,8 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project - merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability).freeze + merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability + alert).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) @@ -71,7 +72,7 @@ module Gitlab return @pattern if @pattern patterns = REFERABLES.map do |type| - Banzai::ReferenceParser[type].reference_type.to_s.classify.constantize.try(:reference_pattern) + Banzai::ReferenceParser[type].reference_class.try(:reference_pattern) end.uniq @pattern = Regexp.union(patterns.compact) diff --git a/lib/gitlab/repository_size_error_message.rb b/lib/gitlab/repository_size_error_message.rb index 8da840779c9..f5d82e61187 100644 --- a/lib/gitlab/repository_size_error_message.rb +++ b/lib/gitlab/repository_size_error_message.rb @@ -6,7 +6,7 @@ module Gitlab delegate :current_size, :limit, :exceeded_size, :additional_repo_storage_available?, to: :@checker - # @param checher [RepositorySizeChecker] + # @param checker [RepositorySizeChecker] def initialize(checker) @checker = checker end diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb index 664afd1cc21..203d7d10532 100644 --- a/lib/gitlab/safe_request_store.rb +++ b/lib/gitlab/safe_request_store.rb @@ -40,7 +40,7 @@ module Gitlab def self.delete_if(&block) return unless RequestStore.active? - storage.delete_if { |k, v| block.call(k) } + storage.delete_if { |k, v| yield(k) } end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index bc59d4ce943..ba822955133 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -24,9 +24,7 @@ module Gitlab # # @return [String] secret token def secret_token - @secret_token ||= begin - File.read(Gitlab.config.gitlab_shell.secret_file).chomp - end + @secret_token ||= File.read(Gitlab.config.gitlab_shell.secret_file).chomp end # Ensure gitlab shell has a secret token stored in the secret_file diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index d5227e7a007..4bf9fd8470a 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -35,6 +35,7 @@ module Gitlab @enabled = true @metrics = init_metrics + @sidekiq_daemon_monitor = Gitlab::SidekiqDaemon::Monitor.instance end private @@ -78,7 +79,7 @@ module Gitlab rescue StandardError => e log_exception(e, __method__) rescue Exception => e # rubocop:disable Lint/RescueException - log_exception(e, __method__ ) + log_exception(e, __method__) raise e end end @@ -188,22 +189,17 @@ module Gitlab def increment_worker_counters(running_jobs, deadline_exceeded) running_jobs.each do |job| - @metrics[:sidekiq_memory_killer_running_jobs].increment( { worker_class: job[:worker_class], deadline_exceeded: deadline_exceeded } ) + @metrics[:sidekiq_memory_killer_running_jobs].increment({ worker_class: job[:worker_class], deadline_exceeded: deadline_exceeded }) end end def fetch_running_jobs - jobs = [] - Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do - jobs = Gitlab::SidekiqDaemon::Monitor.instance.jobs.map do |jid, job| - { - jid: jid, - worker_class: job[:worker_class].name - } - end + @sidekiq_daemon_monitor.jobs.map do |jid, job| + { + jid: jid, + worker_class: job[:worker_class].name + } end - - jobs end def out_of_range_description(rss, hard_limit, soft_limit, deadline_exceeded) @@ -269,10 +265,8 @@ module Gitlab end def rss_increase_by_jobs - Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do - Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| - rss_increase_by_job(job) - end + @sidekiq_daemon_monitor.jobs.sum do |_, job| + rss_increase_by_job(job) end end @@ -297,7 +291,7 @@ module Gitlab end def any_jobs? - Gitlab::SidekiqDaemon::Monitor.instance.jobs.any? + @sidekiq_daemon_monitor.jobs.any? end end end diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb index 655e95c82d3..125402c1e4b 100644 --- a/lib/gitlab/sidekiq_daemon/monitor.rb +++ b/lib/gitlab/sidekiq_daemon/monitor.rb @@ -15,9 +15,6 @@ module Gitlab # that should not be caught by application CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException - attr_reader :jobs - attr_reader :jobs_mutex - def initialize super @@ -31,8 +28,8 @@ module Gitlab end def within_job(worker_class, jid, queue) - jobs_mutex.synchronize do - jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time } + @jobs_mutex.synchronize do + @jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time } end if cancelled?(jid) @@ -48,8 +45,8 @@ module Gitlab yield ensure - jobs_mutex.synchronize do - jobs.delete(jid) + @jobs_mutex.synchronize do + @jobs.delete(jid) end end @@ -65,6 +62,12 @@ module Gitlab end end + def jobs + @jobs_mutex.synchronize do + @jobs.dup + end + end + private def run_thread @@ -166,14 +169,14 @@ module Gitlab # This is why it passes thread in block, # to ensure that we do process this thread def find_thread_unsafe(jid) - jobs.dig(jid, :thread) + @jobs.dig(jid, :thread) end def find_thread_with_lock(jid) # don't try to lock if we cannot find the thread return unless find_thread_unsafe(jid) - jobs_mutex.synchronize do + @jobs_mutex.synchronize do find_thread_unsafe(jid).tap do |thread| yield(thread) if thread end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 357e9d41187..4f7cd340461 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -253,7 +253,7 @@ module Gitlab def with_redis if Feature.enabled?(:use_primary_and_secondary_stores_for_duplicate_jobs) || - Feature.enabled?(:use_primary_store_as_default_for_duplicate_jobs) + Feature.enabled?(:use_primary_store_as_default_for_duplicate_jobs) # TODO: Swap for Gitlab::Redis::SharedState after store transition # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 Gitlab::Redis::DuplicateJobs.with { |redis| yield redis } diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb index 63e8bee4443..fc6a849da95 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb @@ -7,9 +7,9 @@ module Gitlab UnknownStrategyError = Class.new(StandardError) STRATEGIES = { - until_executing: UntilExecuting, - until_executed: UntilExecuted, - none: None + until_executing: ::Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::UntilExecuting, + until_executed: ::Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::UntilExecuted, + none: ::Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::None }.freeze def self.for(name) diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 17234bdf519..778d278146d 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -120,7 +120,7 @@ module Gitlab def self.with_redis if Feature.enabled?(:use_primary_and_secondary_stores_for_sidekiq_status) || - Feature.enabled?(:use_primary_store_as_default_for_sidekiq_status) + Feature.enabled?(:use_primary_store_as_default_for_sidekiq_status) # TODO: Swap for Gitlab::Redis::SharedState after store transition # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 Gitlab::Redis::SidekiqStatus.with { |redis| yield redis } diff --git a/lib/gitlab/slash_commands/application_help.rb b/lib/gitlab/slash_commands/application_help.rb index bfdb65a816d..94abc8b4508 100644 --- a/lib/gitlab/slash_commands/application_help.rb +++ b/lib/gitlab/slash_commands/application_help.rb @@ -3,6 +3,11 @@ module Gitlab module SlashCommands class ApplicationHelp < BaseCommand + def initialize(project, params) + @project = project + @params = params + end + def execute Gitlab::SlashCommands::Presenters::Help .new(project, commands, params) @@ -16,11 +21,7 @@ module Gitlab end def commands - Gitlab::SlashCommands::Command.new( - project, - chat_name, - params - ).commands + Gitlab::SlashCommands::Command.commands end end end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 265eda46489..f8b55f1a91d 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -3,7 +3,7 @@ module Gitlab module SlashCommands class Command < BaseCommand - def commands + def self.commands commands = [ Gitlab::SlashCommands::IssueShow, Gitlab::SlashCommands::IssueNew, @@ -15,7 +15,7 @@ module Gitlab Gitlab::SlashCommands::Run ] - if Feature.enabled?(:incident_declare_slash_command, current_user) + if Feature.enabled?(:incident_declare_slash_command) commands << Gitlab::SlashCommands::IncidentManagement::IncidentNew end @@ -50,7 +50,7 @@ module Gitlab private def available_commands - commands.keep_if do |klass| + self.class.commands.keep_if do |klass| klass.available?(project) end end diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index 9fcefd99f81..16a4875be91 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -54,7 +54,7 @@ module Gitlab return unless environment actions = environment.actions_for(to).select do |action| - action.starts_environment? + action.deployment_job? end if actions.many? diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index d13ccde8576..cd5587bbaef 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -51,16 +51,14 @@ module Gitlab if words.any? words.map { |word| arel_column.matches(to_pattern(word, use_minimum_char_limit: use_minimum_char_limit)) }.reduce(:and) - else + elsif lower_exact_match # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole - if lower_exact_match - Arel::Nodes::NamedFunction + Arel::Nodes::NamedFunction .new('LOWER', [arel_column]) .eq(query) - else - arel_column.matches(sanitize_sql_like(query)) - end + else + arel_column.matches(sanitize_sql_like(query)) end end diff --git a/lib/gitlab/ssh/signature.rb b/lib/gitlab/ssh/signature.rb index 3b4df9a8d0c..a654d5b2ff1 100644 --- a/lib/gitlab/ssh/signature.rb +++ b/lib/gitlab/ssh/signature.rb @@ -9,6 +9,8 @@ module Gitlab class Signature include Gitlab::Utils::StrongMemoize + GIT_NAMESPACE = 'git' + def initialize(signature_text, signed_text, committer_email) @signature_text = signature_text @signed_text = signed_text @@ -18,11 +20,16 @@ module Gitlab def verification_status strong_memoize(:verification_status) do next :unverified unless all_attributes_present? - next :unverified unless valid_signature_blob? && committer + next :unverified unless valid_signature_blob? next :unknown_key unless signed_by_key + next :other_user unless committer next :other_user unless signed_by_key.user == committer - :verified + if signed_by_user_email_verified? + :verified + else + :unverified + end end end @@ -30,7 +37,7 @@ module Gitlab strong_memoize(:signed_by_key) do next unless key_fingerprint - Key.find_by_fingerprint_sha256(key_fingerprint) + Key.signing.find_by_fingerprint_sha256(key_fingerprint) end end @@ -48,6 +55,7 @@ module Gitlab # still need to check that the key belongs to the committer. def valid_signature_blob? return false unless signature + return false unless signature.namespace == GIT_NAMESPACE signature.verify(@signed_text) end @@ -55,7 +63,11 @@ module Gitlab def committer # Lookup by email because users can push verified commits that were made # by someone else. For example: Doing a rebase. - strong_memoize(:committer) { User.find_by_any_email(@committer_email, confirmed: true) } + strong_memoize(:committer) { User.find_by_any_email(@committer_email) } + end + + def signed_by_user_email_verified? + signed_by_key.user.verified_emails.include?(@committer_email) end def signature diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 54db31ffd6c..9dba8c99b99 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -149,18 +149,6 @@ module Gitlab end end - def all_repos - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp - end - end - end - end - end - def repository_storage_paths_args Gitlab::GitalyClient::StorageSettings.allow_disk_access do Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index ededc3db18e..223e3d40751 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -121,8 +121,8 @@ module Gitlab grouped = items.group_by(&:category) categories = grouped.keys - categories.each_with_object({}) do |category, hash| - hash[category] = grouped[category].map do |item| + categories.index_with do |category| + grouped[category].map do |item| { name: item.name, id: item.key, key: item.key, project_id: item.try(:project_id) } end end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index b72d33113dd..ed0b7b4ed87 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -8,9 +8,9 @@ module Gitlab # negative arity means arguments are optional if block.arity == 1 || block.arity < 0 - block.call(model) + yield(model) else - block.call + yield end ensure diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb index ddcd4693738..fd877bc0137 100644 --- a/lib/gitlab/tracking/destinations/snowplow.rb +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -14,7 +14,15 @@ module Gitlab def event(category, action, label: nil, property: nil, value: nil, context: nil) return unless enabled? - tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + tracker.track_struct_event( + category: category, + action: action, + label: label, + property: property, + value: value, + context: context, + tstamp: (Time.now.to_f * 1000).to_i + ) increment_total_events_counter end @@ -54,19 +62,21 @@ module Gitlab def tracker @tracker ||= SnowplowTracker::Tracker.new( - emitter, - SnowplowTracker::Subject.new, - SNOWPLOW_NAMESPACE, - app_id + emitters: [emitter], + subject: SnowplowTracker::Subject.new, + namespace: SNOWPLOW_NAMESPACE, + app_id: app_id ) end def emitter SnowplowTracker::AsyncEmitter.new( - hostname, - protocol: protocol, - on_success: method(:increment_successful_events_emissions), - on_failure: method(:failure_callback) + endpoint: hostname, + options: { + protocol: protocol, + on_success: method(:increment_successful_events_emissions), + on_failure: method(:failure_callback) + } ) end diff --git a/lib/gitlab/tracking/incident_management.rb b/lib/gitlab/tracking/incident_management.rb index df2a0658b36..a912fdbaeca 100644 --- a/lib/gitlab/tracking/incident_management.rb +++ b/lib/gitlab/tracking/incident_management.rb @@ -17,7 +17,7 @@ module Gitlab details = label ? { label: label, property: v } : {} - ::Gitlab::Tracking.event('IncidentManagement::Settings', "#{prefix}_#{key}", **details ) + ::Gitlab::Tracking.event('IncidentManagement::Settings', "#{prefix}_#{key}", **details) end end diff --git a/lib/gitlab/tracking/service_ping_context.rb b/lib/gitlab/tracking/service_ping_context.rb index 393cd647e7f..d31ca69a10c 100644 --- a/lib/gitlab/tracking/service_ping_context.rb +++ b/lib/gitlab/tracking/service_ping_context.rb @@ -4,21 +4,51 @@ module Gitlab module Tracking class ServicePingContext SCHEMA_URL = 'iglu:com.gitlab/gitlab_service_ping/jsonschema/1-0-0' - ALLOWED_SOURCES = %i[redis_hll].freeze + REDISHLL_SOURCE = :redis_hll + REDIS_SOURCE = :redis - def initialize(data_source:, event:) + ALLOWED_SOURCES = [REDISHLL_SOURCE, REDIS_SOURCE].freeze + + def initialize(data_source:, event: nil, key_path: nil) + check_configuration(data_source, event, key_path) + + @payload = { data_source: data_source } + + payload[:event_name] = event if data_source.eql? REDISHLL_SOURCE + payload[:key_path] = key_path if data_source.eql? REDIS_SOURCE + end + + def to_context + SnowplowTracker::SelfDescribingJson.new(SCHEMA_URL, payload) + end + + def to_h + { + schema: SCHEMA_URL, + data: @payload + } + end + + private + + attr_reader :payload + + def check_configuration(data_source, event, key_path) unless ALLOWED_SOURCES.include?(data_source) - raise ArgumentError, "#{data_source} is not acceptable data source for ServicePingContext" + configuration_error("#{data_source} is not acceptable data source for ServicePingContext") end - @payload = { - data_source: data_source, - event_name: event - } + if REDISHLL_SOURCE.eql?(data_source) && event.nil? + configuration_error("event attribute can not be missing for #{REDISHLL_SOURCE} data source") + end + + return unless REDIS_SOURCE.eql?(data_source) && key_path.nil? + + configuration_error("key_path attribute can not be missing for #{REDIS_SOURCE} data source") end - def to_context - SnowplowTracker::SelfDescribingJson.new(SCHEMA_URL, @payload) + def configuration_error(message) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(ArgumentError.new(message)) end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 1e447923a39..00e609511f2 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -22,8 +22,8 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def validate!( url, + schemes:, ports: [], - schemes: [], allow_localhost: false, allow_local_network: true, allow_object_storage: false, @@ -35,6 +35,8 @@ module Gitlab return [nil, nil] if url.nil? + raise ArgumentError, 'The schemes is a required argument' if schemes.blank? + # Param url can be a string, URI or Addressable::URI uri = parse_url(url) @@ -204,7 +206,7 @@ module Gitlab end def validate_scheme(scheme, schemes) - if scheme.blank? || (schemes.any? && !schemes.include?(scheme)) + if scheme.blank? || (schemes.any? && schemes.exclude?(scheme)) raise BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}" end end diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb index 02d9fa74289..4b38809dde4 100644 --- a/lib/gitlab/usage/metrics/aggregates.rb +++ b/lib/gitlab/usage/metrics/aggregates.rb @@ -11,6 +11,7 @@ module Gitlab UnknownAggregationOperator = Class.new(AggregatedMetricError) UnknownAggregationSource = Class.new(AggregatedMetricError) DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError) + UndefinedEvents = Class.new(AggregatedMetricError) DATABASE_SOURCE = 'database' REDIS_SOURCE = 'redis_hll' diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 78f1ddc8a29..8d816c8d902 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -76,3 +76,5 @@ module Gitlab end end end + +Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_mod diff --git a/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb b/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb index dabf757c8a7..c8c248905f7 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb @@ -18,10 +18,8 @@ module Gitlab subset_powers_data = subsets_intersection_powers(metric_names, start_date, end_date, recorded_at, subset_powers_cache) # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| - power_of_union_of_all_metrics = begin - subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ - calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) - end + power_of_union_of_all_metrics = subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ + calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below diff --git a/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb deleted file mode 100644 index a7f8bca8e08..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class CountMergeRequestAuthorsMetric < DatabaseMetric - operation :distinct_count, column: :author_id - - relation { MergeRequest } - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index f0d5298870c..f731057309e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -117,6 +117,8 @@ module Gitlab case time_frame when '28d' monthly_time_range_db_params(column: self.class.metric_timestamp_column) + when '7d' + weekly_time_range_db_params(column: self.class.metric_timestamp_column) when 'all' {} when 'none' diff --git a/lib/gitlab/usage/service_ping/payload_keys_processor.rb b/lib/gitlab/usage/service_ping/payload_keys_processor.rb index ea2043ffb83..89931d8c012 100644 --- a/lib/gitlab/usage/service_ping/payload_keys_processor.rb +++ b/lib/gitlab/usage/service_ping/payload_keys_processor.rb @@ -28,8 +28,8 @@ module Gitlab payload.map do |key, value| if has_metric_definition?(key, parents) parents.dup.append(key).join('.') - else - payload_keys(value, parents.dup << key) if value.is_a?(Hash) + elsif value.is_a?(Hash) + payload_keys(value, parents.dup << key) end end end diff --git a/lib/gitlab/usage/time_frame.rb b/lib/gitlab/usage/time_frame.rb index 39b0855b917..e2eed969200 100644 --- a/lib/gitlab/usage/time_frame.rb +++ b/lib/gitlab/usage/time_frame.rb @@ -21,6 +21,10 @@ module Gitlab def monthly_time_range_db_params(column: nil) { (column || DEFAULT_TIMESTAMP_COLUMN) => 30.days.ago..2.days.ago } end + + def weekly_time_range_db_params(column: nil) + { (column || DEFAULT_TIMESTAMP_COLUMN) => 9.days.ago..2.days.ago } + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 5021dac453f..24f6cc725f6 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -157,7 +157,6 @@ module Gitlab runners_usage, integrations_usage, user_preferences_usage, - container_expiration_policies_usage, service_desk_counts ).tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) @@ -300,7 +299,7 @@ module Gitlab object_store: { enabled: alt_usage_data { config['enabled'] }, direct_upload: alt_usage_data { config['direct_upload'] }, - background_upload: alt_usage_data { config['background_upload'] }, + background_upload: alt_usage_data { false }, # This setting no longer exists provider: alt_usage_data { config['connection']['provider'] } } } @@ -328,26 +327,6 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def container_expiration_policies_usage - results = {} - start = minimum_id(Project) - finish = maximum_id(Project) - - # rubocop: disable UsageData/LargeTable - base = ::ContainerExpirationPolicy.active - # rubocop: enable UsageData/LargeTable - - # rubocop: disable UsageData/LargeTable - ::ContainerExpirationPolicy.older_than_options.keys.each do |value| - results["projects_with_expiration_policy_enabled_with_older_than_set_to_#{value}".to_sym] = distinct_count(base.where(older_than: value), :project_id, start: start, finish: finish) - end - # rubocop: enable UsageData/LargeTable - - results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish) - - results - end - def integrations_usage # rubocop: disable UsageData/LargeTable: Integration.available_integration_names(include_dev: false).each_with_object({}) do |name, response| @@ -611,10 +590,6 @@ module Gitlab {} end - def redis_hll_counters - { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } - end - def action_monthly_active_users(time_period) counter = Gitlab::UsageDataCounters::EditorUniqueCounter date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last } @@ -665,7 +640,6 @@ module Gitlab .merge(topology_usage_data) .merge(usage_activity_by_stage) .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params)) - .merge(redis_hll_counters) end def metric_time_period(time_period) @@ -794,8 +768,8 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def distinct_count_user_auth_by_provider(time_period) - counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash| - hash[provider] = distinct_count( + counts = auth_providers_except_ldap.index_with do |provider| + distinct_count( ::AuthenticationEvent.success.for_provider(provider).where(time_period), :user_id) end diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 5ede840661a..0b448f68153 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -45,20 +45,23 @@ module Gitlab private - def track_unique_action(action, author, time, project = nil) + def track_unique_action(event_name, author, time, project = nil) return unless author if Feature.enabled?(:route_hll_to_snowplow_phase2) Gitlab::Tracking.event( + name, 'ide_edit', - action.to_s, + property: event_name.to_s, project: project, namespace: project&.namespace, - user: author + user: author, + label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] ) end - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id, time: time) end def count_unique(actions, date_from, date_to) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 24a87ae01f4..992cec2d174 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -25,32 +25,6 @@ module Gitlab pipeline_authoring ].freeze - CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ - analytics - ci_users - deploy_token_packages - code_review - ecosystem - error_tracking - ide_edit - importer - incident_management - incident_management_alerts - issues_edit - kubernetes_agent - manage - pipeline_authoring - quickactions - search - secure - snippets - source_code - terraform - testing - user_packages - work_items - ].freeze - # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # @@ -114,41 +88,12 @@ module Gitlab @categories ||= known_events.map { |event| event[:category] }.uniq end - def categories_collected_from_metrics_definitions - CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS - end - # @param category [String] the category name # @return [Array<String>] list of event names for given category def events_for_category(category) known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] } end - # Recent 7 or 28 days unique events data for events defined in /lib/gitlab/usage_data_counters/known_events/ - # - # - For metrics for which we store a key per day, we have the last 7 days or last 28 days of data. - # - For metrics for which we store a key per week, we have the last complete week or last 4 complete weeks - # daily or weekly information is in the file we have for events definition /lib/gitlab/usage_data_counters/known_events/ - # - Most of the metrics have weekly aggregation. We recommend this as it generates fewer keys in Redis to store. - # - The aggregation used doesn't affect data granulation. - def unique_events_data - categories_pending_migration.each_with_object({}) do |category, category_results| - events_names = events_for_category(category) - - event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) unless event == "i_package_composer_deploy_token" - hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event])) - end - - if eligible_for_totals?(events_names) && CATEGORIES_FOR_TOTALS.include?(category) - event_results["#{category}_total_unique_counts_weekly"] = unique_events(**weekly_time_range.merge(event_names: events_names)) - event_results["#{category}_total_unique_counts_monthly"] = unique_events(**monthly_time_range.merge(event_names: events_names)) - end - - category_results["#{category}"] = event_results - end - end - def known_event?(event_name) event_for(event_name).present? end @@ -166,16 +111,13 @@ module Gitlab private - def categories_pending_migration - (categories - categories_collected_from_metrics_definitions) - end - def track(values, event_name, context: '', time: Time.zone.now) return unless ::ServicePing::ServicePingSettings.enabled? event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? + return if event.blank? return unless feature_enabled?(event) Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index dda72f7fa3b..477fa288874 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -173,7 +173,7 @@ module Gitlab private - def track_snowplow_action(action, author, project) + def track_snowplow_action(event_name, author, project) return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) return unless author @@ -181,17 +181,18 @@ module Gitlab ISSUE_CATEGORY, ISSUE_ACTION, label: ISSUE_LABEL, - property: action, + property: event_name, project: project, namespace: project.namespace, - user: author + user: author, + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] ) end - def track_unique_action(action, author) + def track_unique_action(event_name, author) return unless author - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id) end end end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 5b80f6c6c0d..b9f143a3a56 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -347,6 +347,14 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_jobs_container_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_container_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning_latest category: ci_templates redis_slot: ci_templates @@ -519,6 +527,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_jobs_container_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_dast_default_branch_deploy category: ci_templates redis_slot: ci_templates diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 0bd809f8aa5..3bb6655d762 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -39,6 +39,10 @@ redis_slot: code_review category: code_review aggregation: weekly +- name: i_code_review_create_mr + redis_slot: code_review + category: code_review + aggregation: weekly - name: i_code_review_user_create_mr redis_slot: code_review category: code_review @@ -256,7 +260,6 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_diff_searches - name: i_code_review_total_suggestions_applied redis_slot: code_review category: code_review @@ -427,3 +430,28 @@ redis_slot: code_review category: code_review aggregation: weekly +## Security Reports +- name: i_code_review_merge_request_widget_security_reports_view + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_full_report_clicked + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_expand + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_expand_success + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_expand_warning + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_merge_request_widget_security_reports_expand_failed + redis_slot: code_review + category: code_review + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index c1720b26a22..a64b7c4032b 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -131,7 +131,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_ci_i_testing_coverage_report_uploaded # Project Management group - name: g_project_management_issue_title_changed category: issues_edit diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 93137b762ec..10dae35d0bf 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -6,7 +6,8 @@ module Gitlab MR_DIFFS_ACTION = 'i_code_review_mr_diffs' MR_DIFFS_SINGLE_FILE_ACTION = 'i_code_review_mr_single_file_diffs' MR_DIFFS_USER_SINGLE_FILE_ACTION = 'i_code_review_user_single_file_diffs' - MR_CREATE_ACTION = 'i_code_review_user_create_mr' + MR_CREATE_ACTION = 'i_code_review_create_mr' + MR_USER_CREATE_ACTION = 'i_code_review_user_create_mr' MR_CLOSE_ACTION = 'i_code_review_user_close_mr' MR_REOPEN_ACTION = 'i_code_review_user_reopen_mr' MR_MERGE_ACTION = 'i_code_review_user_merge_mr' @@ -62,8 +63,24 @@ module Gitlab track_unique_action_by_user(MR_DIFFS_USER_SINGLE_FILE_ACTION, user) end - def track_create_mr_action(user:) - track_unique_action_by_user(MR_CREATE_ACTION, user) + def track_create_mr_action(user:, merge_request:) + track_unique_action_by_user(MR_USER_CREATE_ACTION, user) + track_unique_action_by_merge_request(MR_CREATE_ACTION, merge_request) + + project = merge_request.target_project + return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) + + Gitlab::Tracking.event( + name, + :create, + project: project, + namespace: project.namespace, + user: user, + property: MR_CREATE_ACTION, + label: 'redis_hll_counters.code_review.i_code_review_create_mr_monthly', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: MR_CREATE_ACTION).to_context] + ) end def track_close_mr_action(user:) @@ -85,11 +102,15 @@ module Gitlab return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) Gitlab::Tracking.event( - 'merge_requests', - MR_APPROVE_ACTION, + name, + :approve, project: project, namespace: project.namespace, - user: user + user: user, + property: MR_APPROVE_ACTION, + label: 'redis_hll_counters.code_review.i_code_review_user_approve_mr_monthly', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: MR_APPROVE_ACTION).to_context] ) end diff --git a/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb b/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb index f88bbc41c70..639da9bfee0 100644 --- a/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb @@ -5,7 +5,16 @@ module Gitlab class MergeRequestWidgetExtensionCounter < BaseCounter KNOWN_EVENTS = %w[view full_report_clicked expand expand_success expand_warning expand_failed].freeze PREFIX = 'i_code_review_merge_request_widget' - WIDGETS = %w[accessibility code_quality license_compliance status_checks terraform test_summary metrics].freeze + WIDGETS = %w[ + accessibility + code_quality + license_compliance + status_checks + terraform + test_summary + metrics + security_reports + ].freeze class << self private diff --git a/lib/gitlab/utils/delegator_override/validator.rb b/lib/gitlab/utils/delegator_override/validator.rb index 4449fa75877..7fa5cc4deef 100644 --- a/lib/gitlab/utils/delegator_override/validator.rb +++ b/lib/gitlab/utils/delegator_override/validator.rb @@ -67,7 +67,7 @@ module Gitlab (delegator_class.instance_methods - allowlist).each do |method_name| target_classes.each do |target_class| - next unless target_class.instance_methods.include?(method_name) + next unless target_class.method_defined?(method_name) errors << generate_error(method_name, target_class, delegator_class) end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 39670a835a6..f83ebba7c3f 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -67,8 +67,8 @@ module Gitlab private def instance_method_defined?(klass, name) - klass.instance_methods(false).include?(name) || - klass.private_instance_methods(false).include?(name) + klass.method_defined?(name, false) || + klass.private_method_defined?(name, false) end def find_direct_method(klass, name) diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index b0dfa087fcf..9c34302f75e 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -30,7 +30,7 @@ module Gitlab # Remove all invalid scheme characters before checking against the # list of unsafe protocols. # - # See https://tools.ietf.org/html/rfc3986#section-3.1 + # See https://www.rfc-editor.org/rfc/rfc3986#section-3.1 # def safe_protocol?(scheme) return false unless scheme diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 6456ad08924..7e78363dae5 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -16,16 +16,6 @@ module Gitlab # include Gitlab::Utils::StrongMemoize # # def trigger_from_token - # strong_memoize(:trigger) do - # Ci::Trigger.find_by_token(params[:token].to_s) - # end - # end - # - # Or like: - # - # include Gitlab::Utils::StrongMemoize - # - # def trigger_from_token # Ci::Trigger.find_by_token(params[:token].to_s) # end # strong_memoize_attr :trigger_from_token @@ -99,6 +89,15 @@ module Gitlab def do_strong_memoize(klass, method_name, member_name) method = klass.instance_method(method_name) + unless method.arity == 0 + raise <<~ERROR + Using `strong_memoize_attr` on methods with parameters is not supported. + + Use `strong_memoize_with` instead. + See https://docs.gitlab.com/ee/development/utilities.html#strongmemoize + ERROR + end + # Methods defined within a class method are already public by default, so we don't need to # explicitly make them public. scope = %i[private protected].find do |scope| @@ -106,9 +105,9 @@ module Gitlab .include? method_name end - klass.define_method(method_name) do |*args, &block| + klass.define_method(method_name) do |&block| strong_memoize(member_name) do - method.bind_call(self, *args, &block) + method.bind_call(self, &block) end end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 7360585df43..8b016a09889 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -11,7 +11,7 @@ module Gitlab included do scope :public_only, -> { where(visibility_level: PUBLIC) } - scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } + scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL]) } scope :private_only, -> { where(visibility_level: PRIVATE) } scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } diff --git a/lib/gitlab/work_items/work_item_hierarchy.rb b/lib/gitlab/work_items/work_item_hierarchy.rb new file mode 100644 index 00000000000..e71bf2bce6a --- /dev/null +++ b/lib/gitlab/work_items/work_item_hierarchy.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module WorkItems + class WorkItemHierarchy < ObjectHierarchy + extend ::Gitlab::Utils::Override + + private + + def middle_table + ::WorkItems::ParentLink.arel_table + end + + def from_tables(cte) + [objects_table, cte.table, middle_table] + end + + override :parent_id_column + def parent_id_column(cte) + middle_table[:work_item_parent_id] + end + + override :ancestor_conditions + def ancestor_conditions(cte) + conditions = middle_table[:work_item_parent_id].eq(objects_table[:id]).and( + middle_table[:work_item_id].eq(cte.table[:id]) + ) + + with_type_filter(conditions, cte) + end + + override :descendant_conditions + def descendant_conditions(cte) + conditions = middle_table[:work_item_id].eq(objects_table[:id]).and( + middle_table[:work_item_parent_id].eq(cte.table[:id]) + ) + + with_type_filter(conditions, cte) + end + + def with_type_filter(conditions, cte) + return conditions unless options[:same_type] + + conditions.and(objects_table[:work_item_type_id].eq(cte.table[:work_item_type_id])) + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 0d5daeefe90..02418c45e73 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -33,7 +33,7 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags( + call_metadata: Feature::Gitaly.server_feature_flags( user: ::Feature::Gitaly.user_actor(user), repository: repository, project: ::Feature::Gitaly.project_actor(repository.container), @@ -48,6 +48,12 @@ module Gitlab attrs[:GitConfigOptions] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" end + attrs[:GitalyServer][:call_metadata].merge!( + 'user_id' => attrs[:GL_ID].presence, + 'username' => attrs[:GL_USERNAME].presence, + 'remote_ip' => Gitlab::ApplicationContext.current_context_attribute(:remote_ip).presence + ).compact! + attrs end @@ -257,7 +263,7 @@ module Gitlab { address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags( + call_metadata: Feature::Gitaly.server_feature_flags( user: ::Feature::Gitaly.user_actor, repository: repository, project: ::Feature::Gitaly.project_actor(repository.container), diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index f8a6980f208..d6bbb8bb2cb 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -6,6 +6,7 @@ module Gitlab module X509 class Signature include Gitlab::Utils::StrongMemoize + include SignatureType attr_reader :signature_text, :signed_text, :created_at @@ -16,14 +17,18 @@ module Gitlab @created_at = created_at end + def type + :x509 + end + def x509_certificate return if certificate_attributes.nil? X509Certificate.safe_create!(certificate_attributes) unless verified_signature.nil? end - def user - strong_memoize(:user) { User.find_by_any_email(@email) } + def signed_by_user + strong_memoize(:signed_by_user) { User.find_by_any_email(@email) } end def verified_signature @@ -33,11 +38,11 @@ module Gitlab def verification_status return :unverified if x509_certificate.nil? || - x509_certificate.revoked? || - !verified_signature || - user.nil? + x509_certificate.revoked? || + !verified_signature || + signed_by_user.nil? - if user.verified_emails.include?(@email.downcase) && certificate_email.casecmp?(@email) + if signed_by_user.verified_emails.include?(@email.downcase) && certificate_email.casecmp?(@email) :verified else :unverified diff --git a/lib/gitlab_edition.rb b/lib/gitlab_edition.rb index 5e3ed35ace4..2a668537534 100644 --- a/lib/gitlab_edition.rb +++ b/lib/gitlab_edition.rb @@ -49,7 +49,7 @@ module GitlabEdition # The behavior needs to be synchronised with # config/helpers/is_ee_env.js root.join('ee/app/models/license.rb').exist? && - !%w[true 1].include?(ENV['FOSS_ONLY'].to_s) + !%w[true 1].include?(ENV['FOSS_ONLY'].to_s) # rubocop:disable Rails/NegateInclude end def self.jh? @@ -57,8 +57,8 @@ module GitlabEdition @is_jh = ee? && - root.join('jh').exist? && - !%w[true 1].include?(ENV['EE_ONLY'].to_s) + root.join('jh').exist? && + !%w[true 1].include?(ENV['EE_ONLY'].to_s) # rubocop:disable Rails/NegateInclude end def self.ee diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 38a1a968aec..e37bd1f7606 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -237,8 +237,8 @@ module GoogleApi end def make_addons_config(enable_addons) - enable_addons.each_with_object({}) do |addon, hash| - hash[addon] = { disabled: false } + enable_addons.index_with do |addon| + { disabled: false } end end diff --git a/lib/kramdown/converter/commonmark.rb b/lib/kramdown/converter/commonmark.rb index 33ec9dd1fbc..a903d541d81 100644 --- a/lib/kramdown/converter/commonmark.rb +++ b/lib/kramdown/converter/commonmark.rb @@ -21,9 +21,9 @@ module Kramdown res = super if [:ul, :dl, :ol, :codeblock].include?(el.type) && opts[:next] && - ([el.type, :codeblock].include?(opts[:next].type) || - (opts[:next].type == :blank && opts[:nnext] && - [el.type, :codeblock].include?(opts[:nnext].type))) + ([el.type, :codeblock].include?(opts[:next].type) || + (opts[:next].type == :blank && opts[:nnext] && + [el.type, :codeblock].include?(opts[:nnext].type))) # replace the end of block character res.sub!(/\^\n\n\z/m, "#{END_OF_BLOCK}\n\n") end @@ -43,7 +43,7 @@ module Kramdown if el.children.first && el.children.first.type == :p && !el.children.first.options[:transparent] if el.children.size == 1 && @stack.last.children.last == el && - (@stack.last.children.any? { |c| c.children.first.type != :p } || @stack.last.children.size == 1) + (@stack.last.children.any? { |c| c.children.first.type != :p } || @stack.last.children.size == 1) # replace the end of block character res.sub!(/\^\n\z/m, "#{END_OF_BLOCK}\n") end diff --git a/lib/pager_duty/validator/schemas/message.json b/lib/pager_duty/validator/schemas/message.json index b1a3185cd1a..eeec6657587 100644 --- a/lib/pager_duty/validator/schemas/message.json +++ b/lib/pager_duty/validator/schemas/message.json @@ -1,44 +1,77 @@ { "type": "object", - "required": ["event", "incident"], + "required": [ + "event" + ], "properties": { - "event": { "type": "string" }, - "incident": { + "event": { "type": "object", "required": [ - "html_url", - "incident_number", - "title", - "status", - "created_at", - "urgency", - "incident_key" - ], - "properties": { - "html_url": { "type": "string" }, - "incindent_number": { "type": "integer" }, - "title": { "type": "string" }, - "status": { "type": "string" }, - "created_at": { "type": "string" }, - "urgency": { "type": "string", "enum": ["high", "low"] }, - "incident_key": { "type": ["string", "null"] }, - "assignments": { - "type": "array", - "items": { - "assignee": { - "type": "array", - "items": { - "summary": { "type": "string" }, - "html_url": { "type": "string" } + "data" + ] + }, + "properties": { + "data": { + "type": "object", + "required": [ + "html_url", + "number", + "title", + "status", + "created_at", + "urgency", + "incident_key" + ], + "properties": { + "html_url": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "title": { + "type": "string" + }, + "status": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "urgency": { + "type": "string", + "enum": [ + "high", + "low" + ] + }, + "incident_key": { + "type": [ + "string", + "null" + ] + }, + "assignee": { + "type": "array", + "items": { + "summary": { + "type": "string" + }, + "html_url": { + "type": "string" + } + } + }, + "service": { + "type": "object", + "items": { + "summary": { + "type": "string" + }, + "html_url": { + "type": "string" } } - } - }, - "impacted_services": { - "type": "array", - "items": { - "summary": { "type": "string" }, - "html_url": { "type": "string" } } } } diff --git a/lib/pager_duty/webhook_payload_parser.rb b/lib/pager_duty/webhook_payload_parser.rb index c17e3df1a72..e65341232f5 100644 --- a/lib/pager_duty/webhook_payload_parser.rb +++ b/lib/pager_duty/webhook_payload_parser.rb @@ -13,7 +13,7 @@ module PagerDuty end def call - Array(payload['messages']).map { |msg| parse_message(msg) }.reject(&:empty?) + parse_message(payload) end private @@ -24,41 +24,47 @@ module PagerDuty return {} unless valid_message?(message) { - 'event' => message['event'], - 'incident' => parse_incident(message['incident']) + 'event' => message.dig('event', 'event_type'), + 'incident' => parse_incident(message.dig('event', 'data')) } end def parse_incident(incident) + return {} unless incident + { 'url' => incident['html_url'], - 'incident_number' => incident['incident_number'], + 'incident_number' => incident['number'], 'title' => incident['title'], 'status' => incident['status'], 'created_at' => incident['created_at'], 'urgency' => incident['urgency'], 'incident_key' => incident['incident_key'], 'assignees' => reject_empty(parse_assignees(incident)), - 'impacted_services' => reject_empty(parse_impacted_services(incident)) + 'impacted_service' => parse_impacted_service(incident) } end def parse_assignees(incident) - Array(incident['assignments']).map do |a| + return [] unless incident + + Array(incident['assignees']).map do |a| { - 'summary' => a.dig('assignee', 'summary'), - 'url' => a.dig('assignee', 'html_url') + 'summary' => a['summary'], + 'url' => a['html_url'] } end end - def parse_impacted_services(incident) - Array(incident['impacted_services']).map do |is| - { - 'summary' => is['summary'], - 'url' => is['html_url'] - } - end + def parse_impacted_service(incident) + return {} unless incident + + return {} if incident.dig('service', 'summary').blank? && incident.dig('service', 'html_url').blank? + + { + 'summary' => incident.dig('service', 'summary'), + 'url' => incident.dig('service', 'html_url') + } end def reject_empty(entities) diff --git a/lib/security/ci_configuration/container_scanning_build_action.rb b/lib/security/ci_configuration/container_scanning_build_action.rb index 82f9f7d0320..f04c221fc40 100644 --- a/lib/security/ci_configuration/container_scanning_build_action.rb +++ b/lib/security/ci_configuration/container_scanning_build_action.rb @@ -12,7 +12,7 @@ module Security def template return 'Auto-DevOps.gitlab-ci.yml' if @auto_devops_enabled - 'Security/Container-Scanning.gitlab-ci.yml' + 'Jobs/Container-Scanning.gitlab-ci.yml' end def comment diff --git a/lib/security/ci_configuration/sast_build_action.rb b/lib/security/ci_configuration/sast_build_action.rb index 448d4fbeacb..2b1964f7c87 100644 --- a/lib/security/ci_configuration/sast_build_action.rb +++ b/lib/security/ci_configuration/sast_build_action.rb @@ -68,7 +68,7 @@ module Security end def auto_devops_stages - auto_devops_template = YAML.safe_load( Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content ) + auto_devops_template = YAML.safe_load(Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content) auto_devops_template['stages'] end diff --git a/lib/security/weak_passwords.rb b/lib/security/weak_passwords.rb index 42b02132933..0772ef42fea 100644 --- a/lib/security/weak_passwords.rb +++ b/lib/security/weak_passwords.rb @@ -9,6 +9,14 @@ module Security # random password. MINIMUM_SUBSTRING_SIZE = 4 + # Passwords of 64+ characters are more likely to randomly include a + # forbidden substring. + # + # This length was chosen somewhat arbitrarily, balancing security, + # usability, and skipping checks on `::User.random_password` which + # is 128 chars. See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105755 + PASSWORD_SUBSTRING_CHECK_MAX_LENGTH = 64 + class << self # Returns true when the password is on a list of weak passwords, # or contains predictable substrings derived from user attributes. @@ -72,7 +80,11 @@ module Security # Case-insensitively checks whether a password includes a dynamic # list of substrings. Substrings which are too short are not # predictable and may occur randomly, and therefore not checked. + # Similarly passwords which are long enough to inadvertently and + # randomly include a substring are not checked. def contains_predicatable_substring?(password, substrings) + return unless password.length < PASSWORD_SUBSTRING_CHECK_MAX_LENGTH + substrings = substrings.filter_map do |substring| substring.downcase if substring.length >= MINIMUM_SUBSTRING_SIZE end diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb deleted file mode 100644 index 6564f53d2da..00000000000 --- a/lib/serializers/json.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Serializers - # Make the resulting hash have deep indifferent access - class Json - class << self - def dump(obj) - obj - end - - def load(data) - return if data.nil? - - Gitlab::Utils.deep_indifferent_access(data) - end - end - end -end diff --git a/lib/service_ping/build_payload.rb b/lib/service_ping/build_payload.rb index 3553b624ae0..0f19cd55a4c 100644 --- a/lib/service_ping/build_payload.rb +++ b/lib/service_ping/build_payload.rb @@ -20,8 +20,8 @@ module ServicePing if has_metric_definition?(key_path) include_metric?(key_path) - else - filtered_usage_data(node, parents.dup << label) if node.is_a?(Hash) + elsif node.is_a?(Hash) + filtered_usage_data(node, parents.dup << label) end end end diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb index 873f11f8a5b..e115ca669d4 100644 --- a/lib/sidebars/groups/menus/packages_registries_menu.rb +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -51,8 +51,8 @@ module Sidebars def harbor_registry_menu_item if Feature.disabled?(:harbor_registry_integration) || - context.group.harbor_integration.nil? || - !context.group.harbor_integration.activated? + context.group.harbor_integration.nil? || + !context.group.harbor_integration.activated? return nil_menu_item(:harbor_registry) end @@ -66,7 +66,7 @@ module Sidebars def dependency_proxy_menu_item setting_does_not_exist_or_is_enabled = !context.group.dependency_proxy_setting || - context.group.dependency_proxy_setting.enabled + context.group.dependency_proxy_setting.enabled return nil_menu_item(:dependency_proxy) unless can?(context.current_user, :read_dependency_proxy, context.group) return nil_menu_item(:dependency_proxy) unless setting_does_not_exist_or_is_enabled diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb index d9d294ff982..dfd88c99a0c 100644 --- a/lib/sidebars/menu.rb +++ b/lib/sidebars/menu.rb @@ -43,7 +43,7 @@ module Sidebars # Value from menus is something like: [{ path: 'foo', path: 'bar', controller: :foo }] # This method filters the information and returns: { path: ['foo', 'bar'], controller: :foo } def all_active_routes - @all_active_routes ||= begin + @all_active_routes ||= ([active_routes] + renderable_items.map(&:active_routes)).flatten.each_with_object({}) do |pairs, hash| pairs.each do |k, v| hash[k] ||= [] @@ -53,7 +53,6 @@ module Sidebars hash end - end end # Returns whether the menu has any menu item, no diff --git a/lib/sidebars/projects/menus/analytics_menu.rb b/lib/sidebars/projects/menus/analytics_menu.rb index b9bcc3267d6..643b7ebcd5a 100644 --- a/lib/sidebars/projects/menus/analytics_menu.rb +++ b/lib/sidebars/projects/menus/analytics_menu.rb @@ -45,9 +45,9 @@ module Sidebars def ci_cd_analytics_menu_item if !context.project.feature_available?(:builds, context.current_user) || - !can?(context.current_user, :read_build, context.project) || - !can?(context.current_user, :read_ci_cd_analytics, context.project) || - context.project.empty_repo? + !can?(context.current_user, :read_build, context.project) || + !can?(context.current_user, :read_ci_cd_analytics, context.project) || + context.project.empty_repo? return ::Sidebars::NilMenuItem.new(item_id: :ci_cd_analytics) end diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb index 9904d533f47..5f789748288 100644 --- a/lib/sidebars/projects/menus/deployments_menu.rb +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -62,7 +62,7 @@ module Sidebars def releases_menu_item if !can?(context.current_user, :read_release, context.project) || - context.project.empty_repo? + context.project.empty_repo? return ::Sidebars::NilMenuItem.new(item_id: :releases) end diff --git a/lib/sidebars/projects/menus/hidden_menu.rb b/lib/sidebars/projects/menus/hidden_menu.rb index c273ee8b74f..5db46a1279c 100644 --- a/lib/sidebars/projects/menus/hidden_menu.rb +++ b/lib/sidebars/projects/menus/hidden_menu.rb @@ -29,8 +29,8 @@ module Sidebars end def graph_menu_item - if !can?(context.current_user, :download_code, context.project) || - context.project.empty_repo? + if !can?(context.current_user, :read_code, context.project) || + context.project.empty_repo? return ::Sidebars::NilMenuItem.new(item_id: :graph) end @@ -72,8 +72,8 @@ module Sidebars end def commits_menu_item - if !can?(context.current_user, :download_code, context.project) || - context.project.empty_repo? + if !can?(context.current_user, :read_code, context.project) || + context.project.empty_repo? return ::Sidebars::NilMenuItem.new(item_id: :commits) end diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index a8ac3d10f83..04c9ab77729 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -35,11 +35,7 @@ module Sidebars private def feature_enabled? - if ::Feature.enabled?(:split_operations_visibility_permissions, context.project) - context.project.feature_available?(:infrastructure, context.current_user) - else - context.project.feature_available?(:operations, context.current_user) - end + context.project.feature_available?(:infrastructure, context.current_user) end def kubernetes_menu_item diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index 035634702db..fea71e4aefd 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -41,11 +41,7 @@ module Sidebars private def feature_enabled? - if ::Feature.enabled?(:split_operations_visibility_permissions, context.project) - context.project.feature_available?(:monitor, context.current_user) - else - context.project.feature_available?(:operations, context.current_user) - end + context.project.feature_available?(:monitor, context.current_user) end def metrics_dashboard_menu_item diff --git a/lib/sidebars/projects/menus/repository_menu.rb b/lib/sidebars/projects/menus/repository_menu.rb index 1b46323089c..735be5a5133 100644 --- a/lib/sidebars/projects/menus/repository_menu.rb +++ b/lib/sidebars/projects/menus/repository_menu.rb @@ -6,7 +6,7 @@ module Sidebars class RepositoryMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - return false unless can?(context.current_user, :download_code, context.project) + return false unless can?(context.current_user, :read_code, context.project) return false if context.project.empty_repo? add_item(files_menu_item) @@ -56,9 +56,15 @@ module Sidebars end def commits_menu_item + link = if Feature.enabled?(:use_ref_type_parameter, context.project) + project_commits_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) + else + project_commits_path(context.project, context.current_ref) + end + ::Sidebars::MenuItem.new( title: _('Commits'), - link: project_commits_path(context.project, context.current_ref), + link: link, active_routes: { controller: %w(commit commits) }, item_id: :commits, container_html_options: { id: 'js-onboarding-commits-link' } @@ -87,18 +93,30 @@ module Sidebars def contributors_menu_item return false unless context.project.analytics_enabled? + link = if Feature.enabled?(:use_ref_type_parameter, context.project) + project_graph_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) + else + project_graph_path(context.project, context.current_ref) + end + ::Sidebars::MenuItem.new( title: _('Contributors'), - link: project_graph_path(context.project, context.current_ref), + link: link, active_routes: { path: 'graphs#show' }, item_id: :contributors ) end def graphs_menu_item + link = if Feature.enabled?(:use_ref_type_parameter, context.project) + project_network_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) + else + project_network_path(context.project, context.current_ref) + end + ::Sidebars::MenuItem.new( title: _('Graph'), - link: project_network_path(context.project, context.current_ref), + link: link, active_routes: { controller: :network }, item_id: :graphs ) @@ -112,6 +130,12 @@ module Sidebars item_id: :compare ) end + + def ref_type_from_context(context) + ref_type = context.try(:ref_type) + ref_type ||= 'heads' if context.current_ref == context.project.repository.root_ref + ref_type + end end end end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 1ad89fdc364..092c7d32ffb 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -65,11 +65,7 @@ if ! cd "$app_root" ; then echo "Failed to cd into $app_root, exiting!"; exit 1 fi -if [ -z "$SIDEKIQ_WORKERS" ]; then - sidekiq_pid_path="$pid_path/sidekiq.pid" -else - sidekiq_pid_path="$pid_path/sidekiq-cluster.pid" -fi +sidekiq_pid_path="$pid_path/sidekiq-cluster.pid" ### Init Script functions diff --git a/lib/support/systemd/gitlab-sidekiq.service b/lib/support/systemd/gitlab-sidekiq.service index cab741010ed..d8585a59085 100644 --- a/lib/support/systemd/gitlab-sidekiq.service +++ b/lib/support/systemd/gitlab-sidekiq.service @@ -10,13 +10,14 @@ Type=notify User=git WorkingDirectory=/home/git/gitlab Environment=RAILS_ENV=production -ExecStart=/usr/local/bin/bundle exec sidekiq --config /home/git/gitlab/config/sidekiq_queues.yml --environment production +Environment=SIDEKIQ_QUEUES=* +ExecStart=/home/git/gitlab/bin/sidekiq-cluster $SIDEKIQ_QUEUES -P /home/git/gitlab/tmp/pids/sidekiq.pid +NotifyAccess=all PIDFile=/home/git/gitlab/tmp/pids/sidekiq.pid Restart=on-failure RestartSec=1 SyslogIdentifier=gitlab-sidekiq Slice=gitlab.slice -WatchdogSec=10 [Install] WantedBy=gitlab.target diff --git a/lib/system_check/app/gitlab_cable_config_exists_check.rb b/lib/system_check/app/gitlab_cable_config_exists_check.rb new file mode 100644 index 00000000000..c13dade1b4c --- /dev/null +++ b/lib/system_check/app/gitlab_cable_config_exists_check.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module SystemCheck + module App + class GitlabCableConfigExistsCheck < SystemCheck::BaseCheck + set_name 'Cable config exists?' + + def check? + cable_config_file = Rails.root.join('config/cable.yml') + + File.exist?(cable_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/cable.yml.example to config/cable.yml', + 'Update config/cable.yml to match your setup' + ) + for_more_information( + see_installation_guide_section('GitLab') + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_resque_config_exists_check.rb b/lib/system_check/app/gitlab_resque_config_exists_check.rb new file mode 100644 index 00000000000..fb835553737 --- /dev/null +++ b/lib/system_check/app/gitlab_resque_config_exists_check.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module SystemCheck + module App + class GitlabResqueConfigExistsCheck < SystemCheck::BaseCheck + set_name 'Resque config exists?' + + def check? + resque_config_file = Rails.root.join('config/resque.yml') + + File.exist?(resque_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/resque.yml.example to config/resque.yml', + 'Update config/resque.yml to match your setup' + ) + for_more_information( + see_installation_guide_section('GitLab') + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb index 8fbfe7af713..3ead1cae8df 100644 --- a/lib/system_check/helpers.rb +++ b/lib/system_check/helpers.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true module SystemCheck + # Helpers used inside a SystemCheck instance to standardize output responses module Helpers include ::Gitlab::TaskHelpers diff --git a/lib/system_check/multi_check_helpers.rb b/lib/system_check/multi_check_helpers.rb new file mode 100644 index 00000000000..1b06864a63e --- /dev/null +++ b/lib/system_check/multi_check_helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module SystemCheck + # Helpers used inside a SystemCheck instance to standardize output responses + # when using a multi_check version + module MultiCheckHelpers + def print_skipped(reason) + $stdout.puts 'skipped'.color(:magenta) + + $stdout.puts ' Reason:'.color(:blue) + $stdout.puts " #{reason}" + end + + def print_warning(reason) + $stdout.puts 'warning'.color(:magenta) + + $stdout.puts ' Reason:'.color(:blue) + $stdout.puts " #{reason}" + end + + def print_failure(reason) + $stdout.puts 'no'.color(:red) + + $stdout.puts ' Reason:'.color(:blue) + $stdout.puts " #{reason}" + end + + def print_pass + $stdout.puts self.class.check_pass.color(:green) + end + end +end diff --git a/lib/system_check/rake_task/app_task.rb b/lib/system_check/rake_task/app_task.rb index 1eb7a35b40a..20332d4b24b 100644 --- a/lib/system_check/rake_task/app_task.rb +++ b/lib/system_check/rake_task/app_task.rb @@ -17,6 +17,8 @@ module SystemCheck SystemCheck::App::OrphanedGroupMembersCheck, SystemCheck::App::GitlabConfigExistsCheck, SystemCheck::App::GitlabConfigUpToDateCheck, + SystemCheck::App::GitlabCableConfigExistsCheck, + SystemCheck::App::GitlabResqueConfigExistsCheck, SystemCheck::App::LogWritableCheck, SystemCheck::App::TmpWritableCheck, SystemCheck::App::UploadsDirectoryExistsCheck, diff --git a/lib/system_check/sidekiq_check.rb b/lib/system_check/sidekiq_check.rb index ab048433b37..777e06f7501 100644 --- a/lib/system_check/sidekiq_check.rb +++ b/lib/system_check/sidekiq_check.rb @@ -5,6 +5,8 @@ module SystemCheck class SidekiqCheck < BaseCheck set_name 'Sidekiq:' + SYSTEMD_UNIT_PATH = '/run/systemd/units/invocation:gitlab-sidekiq.service' + def multi_check check_sidekiq_running only_one_sidekiq_running @@ -37,9 +39,9 @@ module SystemCheck $stdout.print 'Number of Sidekiq processes (cluster/worker) ... ' - if (cluster_count == 1 && worker_count > 0) || (cluster_count == 0 && worker_count == 1) + if cluster_count == 1 && worker_count >= 1 $stdout.puts "#{cluster_count}/#{worker_count}".color(:green) - elsif File.symlink?('/run/systemd/units/invocation:gitlab-sidekiq.service') + elsif File.symlink?(SYSTEMD_UNIT_PATH) $stdout.puts "#{cluster_count}/#{worker_count}".color(:red) try_fixing_it( 'sudo systemctl restart gitlab-sidekiq.service' diff --git a/lib/tasks/contracts/merge_requests.rake b/lib/tasks/contracts/merge_requests.rake index 2ee7ec07a96..61823f0cf1a 100644 --- a/lib/tasks/contracts/merge_requests.rake +++ b/lib/tasks/contracts/merge_requests.rake @@ -4,29 +4,35 @@ return if Rails.env.production? require 'pact/tasks/verification_task' -contracts = File.expand_path('../../../spec/contracts/contracts/project/merge_request', __dir__) provider = File.expand_path('../../../spec/contracts/provider', __dir__) namespace :contracts do + require_relative "../../../spec/contracts/provider/helpers/contract_source_helper" + namespace :merge_requests do - Pact::VerificationTask.new(:diffs_batch) do |pact| + Pact::VerificationTask.new(:get_diffs_batch) do |pact| + pact_helper_location = "pact_helpers/project/merge_requests/show/get_diffs_batch_helper.rb" + pact.uri( - "#{contracts}/show/mergerequest#show-merge_request_diffs_batch_endpoint.json", - pact_helper: "#{provider}/pact_helpers/project/merge_request/show/diffs_batch_helper.rb" + Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + pact_helper: "#{provider}/#{pact_helper_location}" ) end - Pact::VerificationTask.new(:diffs_metadata) do |pact| + Pact::VerificationTask.new(:get_diffs_metadata) do |pact| + pact_helper_location = "pact_helpers/project/merge_requests/show/get_diffs_metadata_helper.rb" pact.uri( - "#{contracts}/show/mergerequest#show-merge_request_diffs_metadata_endpoint.json", - pact_helper: "#{provider}/pact_helpers/project/merge_request/show/diffs_metadata_helper.rb" + Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + pact_helper: "#{provider}/#{pact_helper_location}" ) end - Pact::VerificationTask.new(:discussions) do |pact| + Pact::VerificationTask.new(:get_discussions) do |pact| + pact_helper_location = "pact_helpers/project/merge_requests/show/get_discussions_helper.rb" + pact.uri( - "#{contracts}/show/mergerequest#show-merge_request_discussions_endpoint.json", - pact_helper: "#{provider}/pact_helpers/project/merge_request/show/discussions_helper.rb" + Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + pact_helper: "#{provider}/#{pact_helper_location}" ) end diff --git a/lib/tasks/contracts/pipeline_schedules.rake b/lib/tasks/contracts/pipeline_schedules.rake index 75080d41ebe..b4c87d2e3c9 100644 --- a/lib/tasks/contracts/pipeline_schedules.rake +++ b/lib/tasks/contracts/pipeline_schedules.rake @@ -4,15 +4,18 @@ return if Rails.env.production? require 'pact/tasks/verification_task' -contracts = File.expand_path('../../../spec/contracts/contracts/project/pipeline_schedule', __dir__) provider = File.expand_path('../../../spec/contracts/provider', __dir__) namespace :contracts do + require_relative "../../../spec/contracts/provider/helpers/contract_source_helper" + namespace :pipeline_schedules do Pact::VerificationTask.new(:update_pipeline_schedule) do |pact| + pact_helper_location = "pact_helpers/project/pipeline_schedules/edit/put_edit_a_pipeline_schedule_helper.rb" + pact.uri( - "#{contracts}/edit/pipelineschedules#edit-put_edit_a_pipeline_schedule.json", - pact_helper: "#{provider}/pact_helpers/project/pipeline_schedule/update_pipeline_schedule_helper.rb" + Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + pact_helper: "#{provider}/#{pact_helper_location}" ) end diff --git a/lib/tasks/contracts/pipelines.rake b/lib/tasks/contracts/pipelines.rake index 5a8d7791233..55a7baa4539 100644 --- a/lib/tasks/contracts/pipelines.rake +++ b/lib/tasks/contracts/pipelines.rake @@ -4,40 +4,45 @@ return if Rails.env.production? require 'pact/tasks/verification_task' -contracts = File.expand_path('../../../spec/contracts/contracts/project/pipeline', __dir__) provider = File.expand_path('../../../spec/contracts/provider', __dir__) namespace :contracts do + require_relative "../../../spec/contracts/provider/helpers/contract_source_helper" + namespace :pipelines do Pact::VerificationTask.new(:create_a_new_pipeline) do |pact| + pact_helper_location = "pact_helpers/project/pipelines/new/post_create_a_new_pipeline_helper.rb" + pact.uri( - "#{contracts}/new/pipelines#new-post_create_a_new_pipeline.json", - pact_helper: "#{provider}/pact_helpers/project/pipeline/index/create_a_new_pipeline_helper.rb" + Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + pact_helper: "#{provider}/#{pact_helper_location}" ) end Pact::VerificationTask.new(:get_list_project_pipelines) do |pact| + pact_helper_location = "pact_helpers/project/pipelines/index/get_list_project_pipelines_helper.rb" + pact.uri( - "#{contracts}/index/pipelines#index-get_list_project_pipelines.json", - pact_helper: "#{provider}/pact_helpers/project/pipeline/index/get_list_project_pipelines_helper.rb" + Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + pact_helper: "#{provider}/#{pact_helper_location}" ) end Pact::VerificationTask.new(:get_pipeline_header_data) do |pact| - # pact.uri( - # "http://localhost:9292/pacts/provider/GET%20pipeline%20header%20data/consumer/Pipelines%23show/latest", - # pact_helper: "#{provider}/pact_helpers/project/pipeline/show/get_pipeline_header_data_helper.rb" - # ) + pact_helper_location = "pact_helpers/project/pipelines/show/get_pipeline_header_data_helper.rb" + pact.uri( - "#{contracts}/show/pipelines#show-get_pipeline_header_data.json", - pact_helper: "#{provider}/pact_helpers/project/pipeline/show/get_pipeline_header_data_helper.rb" + Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + pact_helper: "#{provider}/#{pact_helper_location}" ) end Pact::VerificationTask.new(:delete_pipeline) do |pact| + pact_helper_location = "pact_helpers/project/pipelines/show/delete_pipeline_helper.rb" + pact.uri( - "#{contracts}/show/pipelines#show-delete_pipeline.json", - pact_helper: "#{provider}/pact_helpers/project/pipeline/show/delete_pipeline_helper.rb" + Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + pact_helper: "#{provider}/#{pact_helper_location}" ) end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 129f4c0ff0e..22ca5d9039c 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -76,13 +76,9 @@ namespace :dev do namespace :copy_db do ALLOWED_DATABASES = %w[ci].freeze - defined_copy_db_tasks = [] - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| next unless ALLOWED_DATABASES.include?(name) - defined_copy_db_tasks << name - desc "Copies the #{name} database from the main database" task name => :environment do Rake::Task["dev:terminate_all_connections"].invoke @@ -94,16 +90,5 @@ namespace :dev do warn "Database '#{db_config.database}' already exists" end end - - ALLOWED_DATABASES.each do |name| - next if defined_copy_db_tasks.include?(name) - - # :nocov: we cannot mock ActiveRecord::Tasks::DatabaseTasks in time - # Workaround for GDK issue, see - # https://gitlab.com/gitlab-org/gitlab-development-kit/-/issues/1464 - desc "No-op task" - task name - # :nocov: - end end end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 12a8cb01e9e..d8c0b1007e6 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -137,7 +137,7 @@ namespace :gitlab do File.open(gzip, 'wb+') do |f| gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION) gz.mtime = mtime - gz.write IO.binread(file) + gz.write File.binread(file) gz.close File.utime(mtime, mtime, f.path) @@ -154,7 +154,9 @@ namespace :gitlab do desc 'GitLab | Assets | Check that scss mixins do not introduce any sideffects' task :check_page_bundle_mixins_css_for_sideeffects do - system('./scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js') + unless system('./scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js') + abort 'Error: At least one CSS changes introduces an unwanted sideeffect'.color(:red) + end end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index f908a7606fa..49d2d9fed03 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -15,13 +15,11 @@ namespace :gitlab do if Gitlab::Auth::Ldap::Access.allowed?(user) puts " [OK]".color(:green) + elsif block_flag + user.block! unless user.blocked? + puts " [BLOCKED]".color(:red) else - if block_flag - user.block! unless user.blocked? - puts " [BLOCKED]".color(:red) - else - puts " [NOT IN LDAP]".color(:yellow) - end + puts " [NOT IN LDAP]".color(:yellow) end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 4ef0c396f4a..f0264456201 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -316,6 +316,7 @@ namespace :gitlab do all_databases.each do |db| desc "Run migrations on #{db} with instrumentation" task db => :environment do + Gitlab::Database::Migrations::Runner.batched_migrations_last_id(db).store Gitlab::Database::Migrations::Runner.up(database: db).run end end @@ -406,7 +407,7 @@ namespace :gitlab do Rails.application.eager_load! tables = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.tables } - classes = tables.to_h { |t| [t, []] } + classes = tables.index_with { [] } Gitlab::Database.database_base_models.each do |_, model_class| model_class diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake index 421c6a90fdd..a856aa77abc 100644 --- a/lib/tasks/gitlab/db/lock_writes.rake +++ b/lib/tasks/gitlab/db/lock_writes.rake @@ -6,7 +6,8 @@ namespace :gitlab do task lock_writes: [:environment, 'gitlab:db:validate_config'] do Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name| schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) - Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + + Gitlab::Database::LockWritesManager.tables_to_lock(connection) do |table_name, schema_name| # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 next if schema_name == :gitlab_geo @@ -30,7 +31,7 @@ namespace :gitlab do desc "GitLab | DB | Remove all triggers that prevents writes from all databases" task unlock_writes: :environment do Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| - Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + Gitlab::Database::LockWritesManager.tables_to_lock(connection) do |table_name, schema_name| # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 next if schema_name == :gitlab_geo diff --git a/lib/tasks/gitlab/feature_categories.rake b/lib/tasks/gitlab/feature_categories.rake new file mode 100644 index 00000000000..cecfaf3cb36 --- /dev/null +++ b/lib/tasks/gitlab/feature_categories.rake @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :feature_categories do + desc 'GitLab | Feature categories | Build index page for groups' + task index: :environment do + require 'pathname' + + controller_actions = Gitlab::RequestEndpoints + .all_controller_actions + .each_with_object({}) do |(controller, action), hash| + feature_category = controller.feature_category_for_action(action).to_s + + hash[feature_category] ||= [] + hash[feature_category] << { + klass: controller.to_s, + action: action, + source_location: source_location(controller, action) + } + end + + endpoints = Gitlab::RequestEndpoints.all_api_endpoints.each_with_object({}) do |route, hash| + klass = route.app.options[:for] + path = API::Base.path_for_app(route.app) + feature_category = klass.feature_category_for_action(path).to_s + + hash[feature_category] ||= [] + hash[feature_category] << { + klass: klass.to_s, + action: path, + source_location: source_location(klass) + } + end + + workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml.flatten.each_with_object({}) do |worker, hash| + feature_category = worker.get_feature_category.to_s + + next unless worker.klass.name + + hash[feature_category] ||= [] + hash[feature_category] << { + klass: worker.klass.name, + source_location: source_location(worker.klass.name) + } + end + + database_tables = Dir['db/docs/*.yml'].each_with_object({}) do |file, hash| + yaml = YAML.safe_load(File.read(file)) + table_name = yaml['table_name'] + + yaml['feature_categories'].each do |feature_category| + hash[feature_category] ||= [] + hash[feature_category] << table_name + end + end + + puts YAML.dump('controller_actions' => controller_actions, + 'api_endpoints' => endpoints, + 'sidekiq_workers' => workers, + 'database_tables' => database_tables) + end + + def source_location(klass, method = nil) + file, line = + if method && klass.method_defined?(method) + klass.instance_method(method).source_location + else + Kernel.const_source_location(klass.to_s) + end + + relative = Pathname.new(file).relative_path_from(Rails.root).to_s + + if relative.starts_with?('../') || relative.starts_with?('/') + nil + else + [relative, line] + end + end + end +end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 161c7dd38ac..4f7053b7629 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -85,11 +85,9 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{Gitlab::Shell.version || "unknown".color(:red)}" - puts "Repository storage paths:" - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab.config.repositories.storages.each do |name, repository_storage| - puts "- #{name}: \t#{repository_storage.legacy_disk_path}" - end + puts "Repository storages:" + Gitlab.config.repositories.storages.each do |name, repository_storage| + puts "- #{name}: \t#{repository_storage.gitaly_address}" end puts "GitLab Shell path:\t\t#{Gitlab.config.gitlab_shell.path}" end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index cf9876366aa..a5dcb23450f 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -53,13 +53,11 @@ namespace :gitlab do path_to_repo = project.repository.path_to_repo if File.exist?(path_to_repo) print '-' - else - if Gitlab::Shell.new.create_repository(project.repository_storage, + elsif Gitlab::Shell.new.create_repository(project.repository_storage, project.disk_path) - print '.' - else - print 'F' - end + print '.' + else + print 'F' end end end @@ -81,7 +79,7 @@ namespace :gitlab do authorized_keys.clear - Key.find_in_batches(batch_size: 1000) do |keys| + Key.auth.find_in_batches(batch_size: 1000) do |keys| unless authorized_keys.batch_add_keys(keys) puts "Failed to add keys...".color(:red) exit 1 diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index dc472305304..34ef4b139c3 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -10,21 +10,21 @@ namespace :gitlab do desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names' task schedule: :environment do ::Gitlab::SidekiqMigrateJobs - .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout)) .migrate_set('schedule') end desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names' task retry: :environment do ::Gitlab::SidekiqMigrateJobs - .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout)) .migrate_set('retry') end desc 'GitLab | Sidekiq | Migrate jobs in queues outside of routing rules' task queued: :environment do ::Gitlab::SidekiqMigrateJobs - .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout) ) + .new(::Gitlab::SidekiqConfig.worker_queue_mappings, logger: Logger.new($stdout)) .migrate_queues end end @@ -130,5 +130,10 @@ namespace :gitlab do end end end + + namespace :queues do + desc 'GitLab | Sidekiq | Validate all_queues.yml and sidekiq_queues.yml match worker definitions' + task check: ['gitlab:sidekiq:all_queues_yml:check', 'gitlab:sidekiq:sidekiq_queues_yml:check', :environment] + end end end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 7a2dee3e2e4..ec2ea623e02 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -81,6 +81,10 @@ namespace :tw do CodeOwnerRule.new('Workspace', '@lciutacu') ].freeze + ERRORS_EXCLUDED_FILES = [ + '/doc/architecture' + ].freeze + CODEOWNERS_BLOCK_BEGIN = "# Begin rake-managed-docs-block" CODEOWNERS_BLOCK_END = "# End rake-managed-docs-block" @@ -105,16 +109,17 @@ namespace :tw do Dir.glob(path) do |file| yaml_data = YAML.load_file(file) document = Document.new(yaml_data['group'], yaml_data['redirect_to']) + relative_file = file.delete_prefix(Dir.pwd) if document.missing_metadata? - errors << file + errors << relative_file unless ERRORS_EXCLUDED_FILES.any? { |element| relative_file.starts_with?(element) } next end writer = writer_for_group(document.group) next unless writer - mappings << DocumentOwnerMapping.new(file.delete_prefix(Dir.pwd), writer) if document.has_a_valid_group? + mappings << DocumentOwnerMapping.new(relative_file, writer) if document.has_a_valid_group? end deduplicated_mappings = Set.new @@ -139,10 +144,16 @@ namespace :tw do File.write(codeowners_path, new_codeowners_content) + if current_codeowners_content == new_codeowners_content + puts "~ CODEOWNERS already up to date".color(:yellow) + else + puts "✓ CODEOWNERS updated".color(:green) + end + if errors.present? - puts "-----" - puts "ERRORS - the following files are missing the correct metadata:" - errors.map { |file| puts file.gsub(Dir.pwd, ".") } + puts "" + puts "✘ Files with missing metadata found:".color(:red) + errors.map { |file| puts file } end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index d67ad340007..e87f478ac42 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -24,10 +24,11 @@ namespace :gitlab do raise "This rake task is not meant for production instances" end - admin = User.find_by(admin: true) + # Find an admin user with an SSH key + admin = User.where(admin: true).joins(:keys).where.not(keys: { id: nil }).take unless admin - raise "No admin user could be found" + raise "No admin user with SSH key could be found" end tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}" @@ -73,6 +74,8 @@ namespace :gitlab do Commit SHA: #{commit_sha} MSG + local_remote = project.ssh_url_to_repo + Dir.mktmpdir do |tmpdir| Dir.chdir(tmpdir) do Gitlab::TaskHelpers.run_command!(['wget', project_archive_uri, '-O', 'archive.tar.gz']) @@ -80,13 +83,9 @@ namespace :gitlab do extracted_project_basename = Dir['*/'].first Dir.chdir(extracted_project_basename) do Gitlab::TaskHelpers.run_command!(%w(git init --initial-branch=master)) + Gitlab::TaskHelpers.run_command!(%W(git remote add origin #{local_remote})) Gitlab::TaskHelpers.run_command!(%w(git add .)) Gitlab::TaskHelpers.run_command!(['git', 'commit', '--author', 'GitLab <root@localhost>', '--message', commit_message]) - - # Hacky workaround to push to the project in a way that works with both GDK and the test environment - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab::TaskHelpers.run_command!(['git', 'remote', 'add', 'origin', "file://#{project.repository.raw.path}"]) - end Gitlab::TaskHelpers.run_command!(['git', 'push', '-u', 'origin', 'master']) end end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 159b70cd673..32db5e2dff6 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -51,9 +51,12 @@ namespace :gitlab do desc 'GitLab | UsageDataMetrics | Generate raw SQL metrics queries for RSpec' task generate_sql_metrics_queries: :environment do + require 'active_support/testing/time_helpers' + include ActiveSupport::Testing::TimeHelpers + path = Rails.root.join('tmp', 'test') - queries = Timecop.freeze(2021, 1, 1) do + queries = travel_to(Time.utc(2021, 1, 1)) do Gitlab::Usage::ServicePingReport.for(output: :metrics_queries) end diff --git a/lib/version_check.rb b/lib/version_check.rb index 35014f3ddf0..9b7ab440328 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -5,6 +5,9 @@ require "base64" class VersionCheck include ReactiveCaching + # Increment when format of cache value is changed + CACHE_VERSION = 1 + ## Version Check Reactive Caching ## This cache stores the external API response from https://version.gitlab.com ## @@ -61,7 +64,7 @@ class VersionCheck end def id - Gitlab::VERSION + [Gitlab::VERSION, Gitlab.revision, CACHE_VERSION].join('-') end def calculate_reactive_cache(*) @@ -69,13 +72,19 @@ class VersionCheck case response&.code when 200 - response.body + Gitlab::Json.parse(response.body) + else + { error: 'version check failed', status: response&.code } end + rescue JSON::ParserError + { error: 'parsing version check response failed', status: response&.code } end def response with_reactive_cache do |data| - Gitlab::Json.parse(data) if data + raise InvalidateReactiveCache if !data.is_a?(Hash) || data[:error] + + data end end end |
