diff options
Diffstat (limited to 'lib')
343 files changed, 4550 insertions, 1337 deletions
diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index b724d3a38dc..4aebd9c0d40 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -9,6 +9,7 @@ module API before do authenticated_as_admin! + ensure_feature_enabled! end namespace 'admin' do @@ -133,6 +134,10 @@ module API def update_cluster_params declared_params(include_missing: false).without(:cluster_id) end + + def ensure_feature_enabled! + not_found! unless Feature.enabled?(:certificate_based_clusters, clusterable_instance, default_enabled: :yaml, type: :ops) + end end end end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 0762c276aad..e081265b418 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -36,6 +36,8 @@ module API optional :ends_at, type: DateTime, desc: 'Ending time', default: -> { 1.hour.from_now } optional :color, type: String, desc: 'Background color' optional :font, type: String, desc: 'Foreground color' + optional :target_access_levels, type: Array[Integer], coerce_with: Validations::Types::CommaSeparatedToIntegerArray.coerce, + values: BroadcastMessage::ALLOWED_TARGET_ACCESS_LEVELS, desc: 'Target user roles' optional :target_path, type: String, desc: 'Target path' optional :broadcast_type, type: String, values: BroadcastMessage.broadcast_types.keys, desc: 'Broadcast type. Defaults to banner', default: -> { 'banner' } optional :dismissable, type: Boolean, desc: 'Is dismissable' @@ -76,6 +78,8 @@ module API optional :ends_at, type: DateTime, desc: 'Ending time' optional :color, type: String, desc: 'Background color' optional :font, type: String, desc: 'Foreground color' + optional :target_access_levels, type: Array[Integer], coerce_with: Validations::Types::CommaSeparatedToIntegerArray.coerce, + values: BroadcastMessage::ALLOWED_TARGET_ACCESS_LEVELS, desc: 'Target user roles' optional :target_path, type: String, desc: 'Target path' optional :broadcast_type, type: String, values: BroadcastMessage.broadcast_types.keys, desc: 'Broadcast Type' optional :dismissable, type: Boolean, desc: 'Is dismissable' diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index 30ce1454419..d9d0da2e4d1 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -38,7 +38,7 @@ module API use :pagination end # rubocop: disable CodeReuse/ActiveRecord - get ':id/jobs', feature_category: :continuous_integration do + get ':id/jobs', urgency: :low, feature_category: :continuous_integration do authorize_read_builds! builds = user_project.builds.order('id DESC') @@ -55,7 +55,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/jobs/:job_id', feature_category: :continuous_integration do + get ':id/jobs/:job_id', urgency: :low, feature_category: :continuous_integration do authorize_read_builds! build = find_build!(params[:job_id]) @@ -70,7 +70,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/jobs/:job_id/trace', feature_category: :continuous_integration do + get ':id/jobs/:job_id/trace', urgency: :low, feature_category: :continuous_integration do authorize_read_builds! build = find_build!(params[:job_id]) @@ -92,7 +92,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a job' end - post ':id/jobs/:job_id/cancel', feature_category: :continuous_integration do + post ':id/jobs/:job_id/cancel', urgency: :low, feature_category: :continuous_integration do authorize_update_builds! build = find_build!(params[:job_id]) @@ -109,7 +109,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a build' end - post ':id/jobs/:job_id/retry', feature_category: :continuous_integration do + post ':id/jobs/:job_id/retry', urgency: :low, feature_category: :continuous_integration do authorize_update_builds! build = find_build!(params[:job_id]) @@ -127,7 +127,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a build' end - post ':id/jobs/:job_id/erase', feature_category: :continuous_integration do + post ':id/jobs/:job_id/erase', urgency: :low, feature_category: :continuous_integration do authorize_update_builds! build = find_build!(params[:job_id]) @@ -144,9 +144,14 @@ module API end params do requires :job_id, type: Integer, desc: 'The ID of a Job' + optional :job_variables_attributes, type: Array, + desc: 'User defined variables that will be included when running the job' do + requires :key, type: String, desc: 'The name of the variable' + requires :value, type: String, desc: 'The value of the variable' + end end - post ":id/jobs/:job_id/play", feature_category: :continuous_integration do + post ':id/jobs/:job_id/play', urgency: :low, feature_category: :continuous_integration do authorize_read_builds! job = find_job!(params[:job_id]) @@ -155,7 +160,7 @@ module API bad_request!("Unplayable Job") unless job.playable? - job.play(current_user) + job.play(current_user, params[:job_variables_attributes]) status 200 @@ -168,11 +173,11 @@ module API end resource :job do - desc 'Get current project using job token' do + desc 'Get current job using job token' do success Entities::Ci::Job end route_setting :authentication, job_token_allowed: true - get '', feature_category: :continuous_integration do + get '', feature_category: :continuous_integration, urgency: :low do validate_current_authenticated_job present current_authenticated_job, with: Entities::Ci::Job diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index e0086f624a8..2d7a437ca08 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -123,7 +123,7 @@ module API use :pagination end - get ':id/pipelines/:pipeline_id/jobs', feature_category: :continuous_integration do + get ':id/pipelines/:pipeline_id/jobs', urgency: :low, feature_category: :continuous_integration do authorize!(:read_pipeline, user_project) pipeline = user_project.all_pipelines.find(params[:pipeline_id]) @@ -223,9 +223,13 @@ module API post ':id/pipelines/:pipeline_id/retry', feature_category: :continuous_integration do authorize! :update_pipeline, pipeline - pipeline.retry_failed(current_user) + response = pipeline.retry_failed(current_user) - present pipeline, with: Entities::Ci::Pipeline + if response.success? + present pipeline, with: Entities::Ci::Pipeline + else + render_api_error!(response.errors.join(', '), response.http_status) + end end desc 'Cancel all builds in the pipeline' do diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 4df9600322c..0e3b295396b 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -38,7 +38,7 @@ module API attributes[:maintenance_note] ||= deprecated_note if deprecated_note attributes[:active] = !attributes.delete(:paused) if attributes.include?(:paused) - @runner = ::Ci::RegisterRunnerService.new.execute(params[:token], attributes) + @runner = ::Ci::Runners::RegisterRunnerService.new.execute(params[:token], attributes) forbidden! unless @runner if @runner.persisted? @@ -57,7 +57,7 @@ module API delete '/', feature_category: :runner do authenticate_runner! - destroy_conditionally!(current_runner) { ::Ci::UnregisterRunnerService.new(current_runner).execute } + destroy_conditionally!(current_runner) { ::Ci::Runners::UnregisterRunnerService.new(current_runner, params[:token]).execute } end desc 'Validates authentication credentials' do @@ -71,6 +71,19 @@ module API status 200 body "200" end + + desc 'Reset runner authentication token with current token' do + success Entities::Ci::ResetTokenResult + end + params do + requires :token, type: String, desc: 'The current authentication token of the runner' + end + post '/reset_authentication_token', feature_category: :runner do + authenticate_runner! + + current_runner.reset_token! + present current_runner.token_with_expiration, with: Entities::Ci::ResetTokenResult + end end resource :jobs do @@ -118,7 +131,7 @@ module API formatter :build_json, ->(object, _) { object } parser :build_json, ::Grape::Parser::Json - post '/request', feature_category: :continuous_integration do + post '/request', urgency: :low, feature_category: :continuous_integration do authenticate_runner! unless current_runner.active? @@ -172,7 +185,7 @@ module API end optional :exit_code, type: Integer, desc: %q(Job's exit code) end - put '/:id', feature_category: :continuous_integration do + put '/:id', urgency: :low, feature_category: :continuous_integration do job = authenticate_job!(heartbeat_runner: true) Gitlab::Metrics.add_event(:update_build) @@ -199,7 +212,7 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) end - patch '/:id/trace', feature_category: :continuous_integration do + patch '/:id/trace', urgency: :default, feature_category: :continuous_integration do job = authenticate_job!(heartbeat_runner: true) error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 8a7ffab97dd..3c9e887e751 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -90,7 +90,7 @@ module API runner = get_runner(params.delete(:id)) authenticate_update_runner!(runner) params[:active] = !params.delete(:paused) if params.include?(:paused) - update_service = ::Ci::UpdateRunnerService.new(runner) + update_service = ::Ci::Runners::UpdateRunnerService.new(runner) if update_service.update(declared_params(include_missing: false)) present runner, with: Entities::Ci::RunnerDetails, current_user: current_user @@ -110,7 +110,7 @@ module API authenticate_delete_runner!(runner) - destroy_conditionally!(runner) { ::Ci::UnregisterRunnerService.new(runner).execute } + destroy_conditionally!(runner) { ::Ci::Runners::UnregisterRunnerService.new(runner, current_user).execute } end desc 'List jobs running on a runner' do @@ -187,7 +187,7 @@ module API runner = get_runner(params[:runner_id]) authenticate_enable_runner!(runner) - if runner.assign_to(user_project) + if ::Ci::Runners::AssignRunnerService.new(runner, user_project, current_user).execute present runner, with: Entities::Ci::Runner else render_validation_error!(runner) @@ -246,9 +246,9 @@ module API success Entities::Ci::ResetTokenResult end post 'reset_registration_token' do - authorize! :update_runners_registration_token + authorize! :update_runners_registration_token, ApplicationSetting.current - ApplicationSetting.current.reset_runners_registration_token! + ::Ci::Runners::ResetRegistrationTokenService.new(ApplicationSetting.current, current_user).execute present ApplicationSetting.current_without_cache.runners_registration_token_with_expiration, with: Entities::Ci::ResetTokenResult end end diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index 715a8b37fae..d5b21e2ef29 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -7,8 +7,8 @@ module API before do authenticate! - authorize! :admin_build, user_project feature_flag_enabled? + authorize! :read_secure_files, user_project end feature_category :pipeline_authoring @@ -52,39 +52,44 @@ module API body secure_file.file.read end - desc 'Upload a Secure File' - params do - requires :name, type: String, desc: 'The name of the file' - requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The secure file to be uploaded' - optional :permissions, type: String, desc: 'The file permissions', default: 'read_only', values: %w[read_only read_write execute] - end - - route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true - post ':id/secure_files' do - secure_file = user_project.secure_files.new( - name: params[:name], - permissions: params[:permissions] || :read_only - ) - - secure_file.file = params[:file] - - file_too_large! unless secure_file.file.size < ::Ci::SecureFile::FILE_SIZE_LIMIT.to_i + resource do + before do + authorize! :admin_secure_files, user_project + end - if secure_file.save - present secure_file, with: Entities::Ci::SecureFile - else - render_validation_error!(secure_file) + desc 'Upload a Secure File' + params do + requires :name, type: String, desc: 'The name of the file' + requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The secure file to be uploaded' + optional :permissions, type: String, desc: 'The file permissions', default: 'read_only', values: %w[read_only read_write execute] + end + route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true + post ':id/secure_files' do + secure_file = user_project.secure_files.new( + name: params[:name], + permissions: params[:permissions] || :read_only + ) + + secure_file.file = params[:file] + + file_too_large! unless secure_file.file.size < ::Ci::SecureFile::FILE_SIZE_LIMIT.to_i + + if secure_file.save + present secure_file, with: Entities::Ci::SecureFile + else + render_validation_error!(secure_file) + end end - end - desc 'Delete an individual Secure File' - route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true - delete ':id/secure_files/:secure_file_id' do - secure_file = user_project.secure_files.find(params[:secure_file_id]) + desc 'Delete an individual Secure File' + route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true + delete ':id/secure_files/:secure_file_id' do + secure_file = user_project.secure_files.find(params[:secure_file_id]) - secure_file.destroy! + ::Ci::DestroySecureFileService.new(user_project, current_user).execute(secure_file) - no_content! + no_content! + end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 8b8d8192524..dedda82091f 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -44,6 +44,8 @@ module API use :pagination end get ':id/repository/commits', urgency: :low do + not_found! 'Repository' unless user_project.repository_exists? + path = params[:path] before = params[:until] after = params[:since] diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index edf20b6aebe..e241633fa8b 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -38,6 +38,10 @@ module API helpers ::API::Helpers::Packages::Conan::ApiHelpers helpers ::API::Helpers::RelatedResourcesHelpers + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + before do require_packages_enabled! @@ -285,6 +289,7 @@ module API params do requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES end + namespace 'export/:file_name', requirements: FILE_NAME_REQUIREMENTS do desc 'Download recipe files' do detail 'This feature was introduced in GitLab 12.6' diff --git a/lib/api/container_repositories.rb b/lib/api/container_repositories.rb index 9cd3e449687..17d667fb6df 100644 --- a/lib/api/container_repositories.rb +++ b/lib/api/container_repositories.rb @@ -23,11 +23,17 @@ module API params do optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' optional :tags_count, type: Boolean, default: false, desc: 'Determines if the tags count should be included' + optional :size, type: Boolean, default: false, desc: 'Determines if the size should be included' end get ':id' do authorize!(:read_container_image, repository) - present repository, with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count], user: current_user + present repository, + with: Entities::ContainerRegistry::Repository, + tags: params[:tags], + tags_count: params[:tags_count], + size: params[:size], + user: current_user end end end diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb index e9beeb18d62..074c307e881 100644 --- a/lib/api/deploy_tokens.rb +++ b/lib/api/deploy_tokens.rb @@ -93,6 +93,21 @@ module API end end + desc 'Get a project deploy token' do + detail 'This feature was introduced in GitLab 14.9' + success Entities::DeployToken + end + params do + requires :token_id, type: Integer, desc: 'The deploy token ID' + end + get ':id/deploy_tokens/:token_id' do + authorize!(:read_deploy_token, user_project) + + deploy_token = user_project.deploy_tokens.find(params[:token_id]) + + present deploy_token, with: Entities::DeployToken + end + desc 'Delete a project deploy token' do detail 'This feature was introduced in GitLab 12.9' end @@ -159,6 +174,21 @@ module API end end + desc 'Get a group deploy token' do + detail 'This feature was introduced in GitLab 14.9' + success Entities::DeployToken + end + params do + requires :token_id, type: Integer, desc: 'The deploy token ID' + end + get ':id/deploy_tokens/:token_id' do + authorize!(:read_deploy_token, user_group) + + deploy_token = user_group.deploy_tokens.find(params[:token_id]) + + present deploy_token, with: Entities::DeployToken + end + desc 'Delete a group deploy token' do detail 'This feature was introduced in GitLab 12.9' end diff --git a/lib/api/entities/broadcast_message.rb b/lib/api/entities/broadcast_message.rb index e42b110adbe..5a31d64fd86 100644 --- a/lib/api/entities/broadcast_message.rb +++ b/lib/api/entities/broadcast_message.rb @@ -3,7 +3,7 @@ module API module Entities class BroadcastMessage < Grape::Entity - expose :id, :message, :starts_at, :ends_at, :color, :font, :target_path, :broadcast_type, :dismissable + expose :id, :message, :starts_at, :ends_at, :color, :font, :target_access_levels, :target_path, :broadcast_type, :dismissable expose :active?, as: :active end end diff --git a/lib/api/entities/ci/runner.rb b/lib/api/entities/ci/runner.rb index a6944b8c925..e29d55771f2 100644 --- a/lib/api/entities/ci/runner.rb +++ b/lib/api/entities/ci/runner.rb @@ -7,7 +7,7 @@ module API expose :id expose :description expose :ip_address - expose :active # TODO Remove in %15.0 in favor of `paused` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/351109 + expose :active # TODO Remove in %16.0 in favor of `paused` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/351109 expose :paused do |runner| !runner.active end @@ -16,7 +16,7 @@ module API expose :name expose :online?, as: :online # DEPRECATED - # TODO Remove in %15.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + # TODO Remove in %16.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 expose :deprecated_rest_status, as: :status end end diff --git a/lib/api/entities/ci/secure_file.rb b/lib/api/entities/ci/secure_file.rb index 041c864156b..b60a1a6ac90 100644 --- a/lib/api/entities/ci/secure_file.rb +++ b/lib/api/entities/ci/secure_file.rb @@ -9,6 +9,7 @@ module API expose :permissions expose :checksum expose :checksum_algorithm + expose :created_at end end end diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index c9c2c5156cc..2fdfac40c32 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -22,6 +22,7 @@ module API expose :tags_count, if: -> (_, options) { options[:tags_count] } expose :tags, using: Tag, if: -> (_, options) { options[:tags] } expose :delete_api_path, if: ->(object, options) { Ability.allowed?(options[:user], :admin_container_image, object) } + expose :size, if: -> (_, options) { options[:size] } private diff --git a/lib/api/entities/error_tracking.rb b/lib/api/entities/error_tracking.rb index b55cba05ea0..163bda92680 100644 --- a/lib/api/entities/error_tracking.rb +++ b/lib/api/entities/error_tracking.rb @@ -9,6 +9,12 @@ module API expose :sentry_external_url expose :api_url expose :integrated + + def integrated + return false unless ::Feature.enabled?(:integrated_error_tracking, object.project) + + object.integrated_client? + end end class ClientKey < Grape::Entity diff --git a/lib/api/entities/issuable_time_stats.rb b/lib/api/entities/issuable_time_stats.rb index 7c3452a10a1..f93b4651b1f 100644 --- a/lib/api/entities/issuable_time_stats.rb +++ b/lib/api/entities/issuable_time_stats.rb @@ -18,7 +18,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def total_time_spent # Avoids an N+1 query since timelogs are preloaded - object.timelogs.map(&:time_spent).sum + object.timelogs.sum(&:time_spent) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/entities/label_basic.rb b/lib/api/entities/label_basic.rb index ed52688638e..7c846180558 100644 --- a/lib/api/entities/label_basic.rb +++ b/lib/api/entities/label_basic.rb @@ -3,7 +3,11 @@ module API module Entities class LabelBasic < Grape::Entity - expose :id, :name, :color, :description, :description_html, :text_color + expose :id, :name, :description, :description_html, :text_color + + expose :color do |label, options| + label.color.to_s + end end end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 74097dc2883..8f9a8add938 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -74,6 +74,7 @@ module API expose(:operations_access_level) { |project, options| project.project_feature.string_access_level(:operations) } expose(:analytics_access_level) { |project, options| project.project_feature.string_access_level(:analytics) } expose(:container_registry_access_level) { |project, options| project.project_feature.string_access_level(:container_registry) } + expose(:security_and_compliance_access_level) { |project, options| project.project_feature.string_access_level(:security_and_compliance) } expose :emails_disabled expose :shared_runners_enabled diff --git a/lib/api/entities/project_integration.rb b/lib/api/entities/project_integration.rb index 649e4d015b8..155136d2f80 100644 --- a/lib/api/entities/project_integration.rb +++ b/lib/api/entities/project_integration.rb @@ -5,19 +5,8 @@ module API class ProjectIntegration < Entities::ProjectIntegrationBasic # Expose serialized properties expose :properties do |integration, options| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - - attributes = - if integration.data_fields_present? - integration.data_fields.as_json.keys - else - integration.properties.keys - end - - attributes &= integration.api_field_names - - attributes.each_with_object({}) do |attribute, hash| - hash[attribute] = integration.public_send(attribute) # rubocop:disable GitlabSecurity/PublicSend + integration.api_field_names.to_h do |name| + [name, integration.public_send(name)] # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/api/entities/user_safe.rb b/lib/api/entities/user_safe.rb index c7349026a88..fb99c2e960d 100644 --- a/lib/api/entities/user_safe.rb +++ b/lib/api/entities/user_safe.rb @@ -5,14 +5,7 @@ module API class UserSafe < Grape::Entity expose :id, :username expose :name do |user| - next user.name unless user.project_bot? - - next user.name if options[:current_user]&.can?(:read_project, user.projects.first) - - # If the requester does not have permission to read the project bot name, - # the API returns an arbitrary string. UI changes will be addressed in a follow up issue: - # https://gitlab.com/gitlab-org/gitlab/-/issues/346058 - '****' + user.redacted_name(options[:current_user]) end end end diff --git a/lib/api/entities/wiki_page.rb b/lib/api/entities/wiki_page.rb index a8ef0bd857c..43af6a336d2 100644 --- a/lib/api/entities/wiki_page.rb +++ b/lib/api/entities/wiki_page.rb @@ -3,7 +3,15 @@ module API module Entities class WikiPage < WikiPageBasic - expose :content + include ::MarkupHelper + + expose :content do |wiki_page, options| + options[:render_html] ? render_wiki_content(wiki_page, ref: wiki_page.version.id) : wiki_page.content + end + + expose :encoding do |wiki_page| + wiki_page.content.encoding.name + end end end end diff --git a/lib/api/error_tracking/collector.rb b/lib/api/error_tracking/collector.rb index 13fda356257..22a4e04a91c 100644 --- a/lib/api/error_tracking/collector.rb +++ b/lib/api/error_tracking/collector.rb @@ -28,8 +28,8 @@ module API end def feature_enabled? - project.error_tracking_setting&.enabled? && - project.error_tracking_setting&.integrated_client? + Feature.enabled?(:integrated_error_tracking, project) && + project.error_tracking_setting&.integrated_enabled? end def find_client_key(public_key) diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 8cca3378eec..97230976482 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -14,8 +14,6 @@ module API before do require_packages_enabled! authenticate_non_get! - - require_generic_packages_available! end params do @@ -113,10 +111,6 @@ module API include ::API::Helpers::PackagesHelpers include ::API::Helpers::Packages::BasicAuthHelpers - def require_generic_packages_available! - not_found! unless Feature.enabled?(:generic_packages, project, default_enabled: true) - end - def project authorized_user_project end diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index 81944a653c8..a5a60ce8741 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -4,7 +4,10 @@ module API class GroupClusters < ::API::Base include PaginationParams - before { authenticate! } + before do + authenticate! + ensure_feature_enabled! + end feature_category :kubernetes_management @@ -133,6 +136,10 @@ module API def update_cluster_params declared_params(include_missing: false).without(:cluster_id) end + + def ensure_feature_enabled! + not_found! unless Feature.enabled?(:certificate_based_clusters, user_group, default_enabled: :yaml, type: :ops) + end end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 184fe7868a5..de9d42bdce7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -119,7 +119,7 @@ module API def find_project(id) return unless id - projects = Project.without_deleted + projects = Project.without_deleted.not_hidden if id.is_a?(Integer) || id =~ /^\d+$/ projects.find_by(id: id) @@ -474,17 +474,22 @@ module API model.errors.messages end - def render_spam_error! - render_api_error!({ error: 'Spam detected' }, 400) + def render_api_error!(message, status) + render_structured_api_error!({ 'message' => message }, status) end - def render_api_error!(message, status) + def render_structured_api_error!(hash, status) + # Use this method instead of `render_api_error!` when you have additional top-level + # hash entries in addition to 'message' which need to be passed to `#error!` + set_status_code_in_env(status) + error!(hash, status, header) + end + + def set_status_code_in_env(status) # grape-logging doesn't pass the status code, so this is a # workaround for getting that information in the loggers: # https://github.com/aserafin/grape_logging/issues/71 env[API_RESPONSE_STATUS_CODE] = Rack::Utils.status_code(status) - - error!({ 'message' => message }, status, header) end def handle_api_exception(exception) diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 86dedc12fca..0fbd0e6be44 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -440,6 +440,32 @@ module API }, chat_notification_events ].flatten, + 'harbor' => [ + { + required: true, + name: :url, + type: String, + desc: 'The base URL to the Harbor instance which is being linked to this GitLab project. For example, https://demo.goharbor.io.' + }, + { + required: true, + name: :project_name, + type: String, + desc: 'The Project name to the Harbor instance. For example, testproject.' + }, + { + required: true, + name: :username, + type: String, + desc: 'The username created from Harbor interface.' + }, + { + required: true, + name: :password, + type: String, + desc: 'The password of the user.' + } + ], 'irker' => [ { required: true, @@ -856,6 +882,7 @@ module API ::Integrations::ExternalWiki, ::Integrations::Flowdock, ::Integrations::HangoutsChat, + ::Integrations::Harbor, ::Integrations::Irker, ::Integrations::Jenkins, ::Integrations::Jira, diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 00f745067e7..f1125899f8c 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -36,6 +36,7 @@ module API optional :operations_access_level, type: String, values: %w(disabled private enabled), desc: 'Operations access level. One of `disabled`, `private` or `enabled`' optional :analytics_access_level, type: String, values: %w(disabled private enabled), desc: 'Analytics access level. One of `disabled`, `private` or `enabled`' optional :container_registry_access_level, type: String, values: %w(disabled private enabled), desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry' + optional :security_and_compliance_access_level, type: String, values: %w(disabled private enabled), desc: 'Security and compliance access level. One of `disabled`, `private` or `enabled`' optional :emails_disabled, type: Boolean, desc: 'Disable email notifications' optional :show_default_award_emojis, type: Boolean, desc: 'Show default award emojis' @@ -118,6 +119,7 @@ module API def self.update_params_at_least_one_of [ :allow_merge_on_skipped_pipeline, + :analytics_access_level, :autoclose_referenced_issues, :auto_devops_enabled, :auto_devops_deploy_strategy, @@ -145,6 +147,7 @@ module API :name, :only_allow_merge_if_all_discussions_are_resolved, :only_allow_merge_if_pipeline_succeeds, + :operations_access_level, :pages_access_level, :path, :printing_merge_request_link_enabled, @@ -154,6 +157,7 @@ module API :request_access_enabled, :resolve_outdated_diff_discussions, :restrict_user_defined_variables, + :security_and_compliance_access_level, :squash_option, :shared_runners_enabled, :snippets_access_level, diff --git a/lib/api/helpers/wikis_helpers.rb b/lib/api/helpers/wikis_helpers.rb index 4a14dc1f40a..a9cd0e2919d 100644 --- a/lib/api/helpers/wikis_helpers.rb +++ b/lib/api/helpers/wikis_helpers.rb @@ -13,8 +13,8 @@ module API raise "Unknown wiki container #{kind}" end - def wiki_page - Wiki.for_container(container, current_user).find_page(params[:slug]) || not_found!('Wiki Page') + def wiki_page(version = nil) + Wiki.for_container(container, current_user).find_page(params[:slug], version.presence) || not_found!('Wiki Page') end def commit_params(attrs) diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 48157a91477..9c527f28d44 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -92,6 +92,8 @@ module API payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" end + send_git_audit_streaming_event(protocol: params[:protocol], action: params[:action]) + response_with_status(**payload) when ::Gitlab::GitAccessResult::CustomAction response_with_status(code: 300, payload: check_result.payload, gl_console_messages: check_result.console_messages) @@ -100,6 +102,10 @@ module API end end + def send_git_audit_streaming_event(msg) + # Defined in EE + end + def access_check!(actor, params) access_checker = access_checker_for(actor, params[:protocol]) access_checker.check(params[:action], params[:changes]).tap do |result| diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 3977da4bda4..df887a83c4f 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -39,6 +39,7 @@ module API def gitaly_repository(project) { + default_branch: project.default_branch_or_main, storage_name: project.repository_storage, relative_path: project.disk_path + '.git', gl_repository: repo_type.identifier_for_container(project), diff --git a/lib/api/issues.rb b/lib/api/issues.rb index a5d6a6d7cf3..e9bb9fe7a97 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -4,6 +4,7 @@ module API class Issues < ::API::Base include PaginationParams helpers Helpers::IssuesHelpers + helpers SpammableActions::CaptchaCheck::RestApiActionsSupport before { authenticate_non_get! } @@ -262,8 +263,6 @@ module API post ':id/issues' do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21140') - check_rate_limit!(:issues_create, scope: current_user) if Feature.disabled?("rate_limited_service_issues_create", user_project, default_enabled: :yaml) - authorize! :create_issue, user_project issue_params = declared_params(include_missing: false) @@ -277,14 +276,12 @@ module API params: issue_params, spam_params: spam_params).execute - if issue.spam? - render_api_error!({ error: 'Spam detected' }, 400) - end - if issue.valid? present issue, with: Entities::Issue, current_user: current_user, project: user_project else - render_validation_error!(issue) + with_captcha_check_rest_api(spammable: issue) do + render_validation_error!(issue) + end end rescue ::ActiveRecord::RecordNotUnique render_api_error!('Duplicated issue', 409) @@ -322,12 +319,12 @@ module API params: update_params, spam_params: spam_params).execute(issue) - render_spam_error! if issue.spam? - if issue.valid? present issue, with: Entities::Issue, current_user: current_user, project: user_project else - render_validation_error!(issue) + with_captcha_check_rest_api(spammable: issue) do + render_validation_error!(issue) + end end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index f7df8d33418..de9a2a198d9 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -304,10 +304,6 @@ module API end get ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review, urgency: :high do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - project = merge_request.project - - not_found! unless project.context_commits_enabled? - context_commits = paginate(merge_request.merge_request_context_commits).map(&:to_commit) @@ -328,9 +324,6 @@ module API end merge_request = find_merge_request_with_access(params[:merge_request_iid]) - project = merge_request.project - - not_found! unless project.context_commits_enabled? authorize!(:update_merge_request, merge_request) @@ -351,9 +344,6 @@ module API delete ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do commit_ids = params[:commits] merge_request = find_merge_request_with_access(params[:merge_request_iid]) - project = merge_request.project - - not_found! unless project.context_commits_enabled? authorize!(:destroy_merge_request, merge_request) project = merge_request.target_project diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 93ef77d5a62..b260f5289b3 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -75,6 +75,7 @@ module API requires :body, type: String, desc: 'The content of a note' optional :confidential, type: Boolean, desc: 'Confidentiality note flag, default is false' optional :created_at, type: String, desc: 'The creation date of the note' + optional :merge_request_diff_head_sha, type: String, desc: 'The SHA of the head commit' end post ":id/#{noteables_str}/:noteable_id/notes", feature_category: feature_category do allowlist = @@ -87,7 +88,8 @@ module API noteable_type: noteables_str.classify, noteable_id: noteable.id, confidential: params[:confidential], - created_at: params[:created_at] + created_at: params[:created_at], + merge_request_diff_head_sha: params[:merge_request_diff_head_sha] } note = create_note(noteable, opts) diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index e80355e80c7..4861c0c740e 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -29,7 +29,7 @@ module API .new(user_project, params[:package_id]).execute package_files = package.installable_package_files - .preload_pipelines + .preload_pipelines.order_id_asc present paginate(package_files), with: ::API::Entities::PackageFile end diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index 6785b28ddef..8bba67a53af 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -4,7 +4,10 @@ module API class ProjectClusters < ::API::Base include PaginationParams - before { authenticate! } + before do + authenticate! + ensure_feature_enabled! + end feature_category :kubernetes_management @@ -138,6 +141,10 @@ module API def update_cluster_params declared_params(include_missing: false).without(:cluster_id) end + + def ensure_feature_enabled! + not_found! unless Feature.enabled?(:certificate_based_clusters, user_project, default_enabled: :yaml, type: :ops) + end end end end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index a3d76e571a9..fae170d638b 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -87,14 +87,16 @@ module API validate_file! - response = ::Import::GitlabProjects::CreateProjectFromUploadedFileService.new( + response = ::Import::GitlabProjects::CreateProjectService.new( current_user, - path: import_params[:path], - namespace: namespace_from(import_params, current_user), - name: import_params[:name], - file: import_params[:file], - overwrite: import_params[:overwrite], - override: filtered_override_params(import_params) + params: { + path: import_params[:path], + namespace: namespace_from(import_params, current_user), + name: import_params[:name], + file: import_params[:file], + overwrite: import_params[:overwrite], + override: filtered_override_params(import_params) + } ).execute if response.success? @@ -137,14 +139,66 @@ module API check_rate_limit! :project_import, scope: [current_user, :project_import] - response = ::Import::GitlabProjects::CreateProjectFromRemoteFileService.new( + response = ::Import::GitlabProjects::CreateProjectService.new( current_user, - path: import_params[:path], - namespace: namespace_from(import_params, current_user), - name: import_params[:name], - remote_import_url: import_params[:url], - overwrite: import_params[:overwrite], - override: filtered_override_params(import_params) + params: { + path: import_params[:path], + namespace: namespace_from(import_params, current_user), + name: import_params[:name], + remote_import_url: import_params[:url], + overwrite: import_params[:overwrite], + override: filtered_override_params(import_params) + }, + file_acquisition_strategy: ::Import::GitlabProjects::FileAcquisitionStrategies::RemoteFile + ).execute + + if response.success? + present(response.payload, with: Entities::ProjectImportStatus) + else + render_api_error!(response.message, response.http_status) + end + end + + params do + requires :region, type: String, desc: 'AWS region' + requires :bucket_name, type: String, desc: 'Bucket name' + requires :file_key, type: String, desc: 'File key' + requires :access_key_id, type: String, desc: 'Access key id' + requires :secret_access_key, type: String, desc: 'Secret access key' + requires :path, type: String, desc: 'The new project path and name' + optional :name, type: String, desc: 'The name of the project to be imported. Defaults to the path of the project if not provided.' + optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." + optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it' + optional :override_params, + type: Hash, + desc: 'New project params to override values in the export' do + use :optional_project_params + end + end + desc 'Create a new project import using a file from AWS S3' do + detail 'This feature was introduced in GitLab 14.9.' + success Entities::ProjectImportStatus + end + post 'remote-import-s3' do + not_found! unless ::Feature.enabled?(:import_project_from_remote_file_s3, default_enabled: :yaml) + + check_rate_limit! :project_import, scope: [current_user, :project_import] + + response = ::Import::GitlabProjects::CreateProjectService.new( + current_user, + params: { + path: import_params[:path], + namespace: namespace_from(import_params, current_user), + name: import_params[:name], + overwrite: import_params[:overwrite], + override: filtered_override_params(import_params), + region: import_params[:region], + bucket_name: import_params[:bucket_name], + file_key: import_params[:file_key], + access_key_id: import_params[:access_key_id], + secret_access_key: import_params[:secret_access_key] + }, + file_acquisition_strategy: ::Import::GitlabProjects::FileAcquisitionStrategies::RemoteFileS3 ).execute if response.success? diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index fdbfdf1f7a9..a80e45637dc 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -13,6 +13,7 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers Helpers::SnippetsHelpers + helpers SpammableActions::CaptchaCheck::RestApiActionsSupport helpers do def check_snippets_enabled forbidden! unless user_project.feature_available?(:snippets, current_user) @@ -82,9 +83,9 @@ module API if service_response.success? present snippet, with: Entities::ProjectSnippet, current_user: current_user else - render_spam_error! if snippet.spam? - - render_api_error!({ error: service_response.message }, service_response.http_status) + with_captcha_check_rest_api(spammable: snippet) do + render_api_error!({ error: service_response.message }, service_response.http_status) + end end end @@ -124,9 +125,9 @@ module API if service_response.success? present snippet, with: Entities::ProjectSnippet, current_user: current_user else - render_spam_error! if snippet.spam? - - render_api_error!({ error: service_response.message }, service_response.http_status) + with_captcha_check_rest_api(spammable: snippet) do + render_api_error!({ error: service_response.message }, service_response.http_status) + end end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 706c0702fce..86f36d4fc00 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -170,9 +170,9 @@ module API params do requires :content, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' - requires :requires_python, type: String requires :name, type: String requires :version, type: String + optional :requires_python, type: String optional :md5_digest, type: String optional :sha256_digest, type: String end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index c3632c812f3..2e21f591667 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -248,6 +248,8 @@ module API changelog = service.execute(commit_to_changelog: false) present changelog, with: Entities::Changelog + rescue Gitlab::Changelog::Error => ex + render_api_error!("Failed to generate the changelog: #{ex.message}", 422) end desc 'Generates a changelog section for a release and commits it in a changelog file' do diff --git a/lib/api/search.rb b/lib/api/search.rb index 60a7e944b43..4ef8fef329c 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -7,7 +7,7 @@ module API before do authenticate! - check_rate_limit!(:user_email_lookup, scope: [current_user]) if search_service.params.email_lookup? + check_rate_limit!(:search_rate_limit, scope: [current_user]) end feature_category :global_search diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index c4b17a62b59..9a3c68bc854 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -9,6 +9,7 @@ module API resource :snippets do helpers Helpers::SnippetsHelpers + helpers SpammableActions::CaptchaCheck::RestApiActionsSupport helpers do def snippets_for_current_user SnippetsFinder.new(current_user, author: current_user).execute @@ -91,9 +92,9 @@ module API if service_response.success? present snippet, with: Entities::PersonalSnippet, current_user: current_user else - render_spam_error! if snippet.spam? - - render_api_error!({ error: service_response.message }, service_response.http_status) + with_captcha_check_rest_api(spammable: snippet) do + render_api_error!({ error: service_response.message }, service_response.http_status) + end end end @@ -135,9 +136,9 @@ module API if service_response.success? present snippet, with: Entities::PersonalSnippet, current_user: current_user else - render_spam_error! if snippet.spam? - - render_api_error!({ error: service_response.message }, service_response.http_status) + with_captcha_check_rest_api(spammable: snippet) do + render_api_error!({ error: service_response.message }, service_response.http_status) + end end end diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb index 6818c04fd2e..a12a2ed08d7 100644 --- a/lib/api/statistics.rb +++ b/lib/api/statistics.rb @@ -12,7 +12,7 @@ module API desc 'Get the current application statistics' do success Entities::ApplicationStatistics end - get "application/statistics" do + get "application/statistics", urgency: :low do counts = Gitlab::Database::Count.approximate_counts(COUNTED_ITEMS) present counts, with: Entities::ApplicationStatistics end diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index e4133713c1f..7c91fbd36d9 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -22,6 +22,18 @@ module API present paginate(SystemHook.all), with: Entities::Hook end + desc 'Get a hook' do + success Entities::Hook + end + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end + get ":id" do + hook = SystemHook.find(params[:id]) + + present hook, with: Entities::Hook + end + desc 'Create a new system hook' do success Entities::Hook end diff --git a/lib/api/topics.rb b/lib/api/topics.rb index b9c2bcc2da8..e4a1fa2367e 100644 --- a/lib/api/topics.rb +++ b/lib/api/topics.rb @@ -77,5 +77,19 @@ module API render_validation_error!(topic) end end + + desc 'Delete a topic' do + detail 'This feature was introduced in GitLab 14.9.' + end + params do + requires :id, type: Integer, desc: 'ID of project topic' + end + delete 'topics/:id' do + authenticated_as_admin! + + topic = ::Projects::Topic.find(params[:id]) + + destroy_conditionally!(topic) + end end end diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb index 634dd0f2179..e5dfac3b1a1 100644 --- a/lib/api/user_counts.rb +++ b/lib/api/user_counts.rb @@ -11,13 +11,19 @@ module API get do unauthorized! unless current_user - { + counts = { merge_requests: current_user.assigned_open_merge_requests_count, # @deprecated assigned_issues: current_user.assigned_open_issues_count, assigned_merge_requests: current_user.assigned_open_merge_requests_count, review_requested_merge_requests: current_user.review_requested_open_merge_requests_count, todos: current_user.todos_pending_count } + + if Feature.enabled?(:mr_attention_requests, default_enabled: :yaml) + counts[:attention_requests] = current_user.attention_requested_open_merge_requests_count + end + + counts end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 6d4f12d80f8..0f710e0a307 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -142,13 +142,11 @@ module API get ":id", feature_category: :users do forbidden!('Not authorized!') unless current_user - if Feature.enabled?(:rate_limit_user_by_id_endpoint, type: :development) - unless current_user.admin? - check_rate_limit!(:users_get_by_id, - scope: current_user, - users_allowlist: Gitlab::CurrentSettings.current_application_settings.users_get_by_id_limit_allowlist - ) - end + unless current_user.admin? + check_rate_limit!(:users_get_by_id, + scope: current_user, + users_allowlist: Gitlab::CurrentSettings.current_application_settings.users_get_by_id_limit_allowlist + ) end user = User.find_by(id: params[:id]) @@ -383,6 +381,23 @@ module API present paginate(keys), with: Entities::SSHKey end + desc 'Get a SSH key of a specified user.' do + success Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + get ':id/keys/:key_id', requirements: API::USER_REQUIREMENTS, feature_category: :authentication_and_authorization do + user = find_user(params[:id]) + not_found!('User') unless user && can?(current_user, :read_user, user) + + key = user.keys.find_by(id: params[:key_id]) # rubocop: disable CodeReuse/ActiveRecord + not_found!('Key') unless key + + present key, with: Entities::SSHKey + end + desc 'Delete an existing SSH key from a specified user. Available only for admins.' do success Entities::SSHKey end @@ -687,6 +702,8 @@ module API if user.ldap_blocked? forbidden!('LDAP blocked users cannot be modified by the API') + elsif current_user == user + forbidden!('The API initiating user cannot be blocked by the API') end break if user.blocked? diff --git a/lib/api/validations/validators/file_path.rb b/lib/api/validations/validators/file_path.rb index 246c445658f..268ddc29d4e 100644 --- a/lib/api/validations/validators/file_path.rb +++ b/lib/api/validations/validators/file_path.rb @@ -8,8 +8,7 @@ module API options = @option.is_a?(Hash) ? @option : {} path_allowlist = options.fetch(:allowlist, []) path = params[attr_name] - path = Gitlab::Utils.check_path_traversal!(path) - Gitlab::Utils.check_allowed_absolute_path!(path, path_allowlist) + Gitlab::Utils.check_allowed_absolute_path_and_path_traversal!(path, path_allowlist) rescue StandardError raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index fdce3c5ce18..e90d88940a5 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -45,11 +45,13 @@ module API end params do requires :slug, type: String, desc: 'The slug of a wiki page' + optional :version, type: String, desc: 'The version hash of a wiki page' + optional :render_html, type: Boolean, default: false, desc: 'Render content to HTML' end get ':id/wikis/:slug' do authorize! :read_wiki, container - present wiki_page, with: Entities::WikiPage + present wiki_page(params[:version]), with: Entities::WikiPage, render_html: params[:render_html] end desc 'Create a wiki page' do diff --git a/lib/atlassian/jira_connect.rb b/lib/atlassian/jira_connect.rb index 7f693eff59b..595cf0ac465 100644 --- a/lib/atlassian/jira_connect.rb +++ b/lib/atlassian/jira_connect.rb @@ -8,7 +8,10 @@ module Atlassian end def app_key - "gitlab-jira-connect-#{gitlab_host}" + # App key must be <= 64 characters. + # See: https://developer.atlassian.com/cloud/jira/platform/connect-app-descriptor/#app-descriptor-structure + + "gitlab-jira-connect-#{gitlab_host}"[..63] end private diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index dc37465744b..b8aa2cc8ea0 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -127,16 +127,21 @@ module Atlassian def handle_response(response, name, &block) data = response.parsed_response - case response.code - when 200 then yield data - when 400 then { 'errorMessages' => data.map { |e| e['message'] } } - when 401 then { 'errorMessages' => ['Invalid JWT'] } - when 403 then { 'errorMessages' => ["App does not support #{name}"] } - when 413 then { 'errorMessages' => ['Data too large'] + data.map { |e| e['message'] } } - when 429 then { 'errorMessages' => ['Rate limit exceeded'] } - when 503 then { 'errorMessages' => ['Service unavailable'] } + if [200, 202].include?(response.code) + yield data else - { 'errorMessages' => ['Unknown error'], 'response' => data } + message = case response.code + when 400 then { 'errorMessages' => data.map { |e| e['message'] } } + when 401 then { 'errorMessages' => ['Invalid JWT'] } + when 403 then { 'errorMessages' => ["App does not support #{name}"] } + when 413 then { 'errorMessages' => ['Data too large'] + data.map { |e| e['message'] } } + when 429 then { 'errorMessages' => ['Rate limit exceeded'] } + when 503 then { 'errorMessages' => ['Service unavailable'] } + else + { 'errorMessages' => ['Unknown error'], 'response' => data } + end + + message.merge('responseCode' => response.code) end end diff --git a/lib/atlassian/jira_connect/serializers/build_entity.rb b/lib/atlassian/jira_connect/serializers/build_entity.rb index a3434c529a4..10e4bb0e709 100644 --- a/lib/atlassian/jira_connect/serializers/build_entity.rb +++ b/lib/atlassian/jira_connect/serializers/build_entity.rb @@ -26,7 +26,7 @@ module Atlassian # merge request title. @issue_keys ||= begin pipeline.all_merge_requests.flat_map do |mr| - src = "#{mr.source_branch} #{mr.title}" + src = "#{mr.source_branch} #{mr.title} #{mr.description}" JiraIssueKeyExtractor.new(src).issue_keys end.uniq end diff --git a/lib/atlassian/jira_connect/serializers/environment_entity.rb b/lib/atlassian/jira_connect/serializers/environment_entity.rb index b6b5db40ba6..67ac93473c3 100644 --- a/lib/atlassian/jira_connect/serializers/environment_entity.rb +++ b/lib/atlassian/jira_connect/serializers/environment_entity.rb @@ -20,18 +20,7 @@ module Atlassian end def type - case environment.name - when /\A(.*[^a-z0-9])?(staging|stage|stg|preprod|pre-prod|model|internal)([^a-z0-9].*)?\z/i - 'staging' - when /\A(.*[^a-z0-9])?(prod|production|prd|live)([^a-z0-9].*)?\z/i - 'production' - when /\A(.*[^a-z0-9])?(test|testing|tests|tst|integration|integ|intg|int|acceptance|accept|acpt|qa|qc|control|quality)([^a-z0-9].*)?\z/i - 'testing' - when /\A(.*[^a-z0-9])?(dev|review|development)([^a-z0-9].*)?\z/i - 'development' - else - 'unmapped' - end + environment.tier == 'other' ? 'unmapped' : environment.tier end end end diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 163446998e9..4ef76b0aaf3 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -2,14 +2,11 @@ module Backup class Artifacts < Backup::Files - attr_reader :progress - def initialize(progress) - @progress = progress - - super('artifacts', JobArtifactUploader.root, excludes: ['tmp']) + super(progress, 'artifacts', JobArtifactUploader.root, excludes: ['tmp']) end + override :human_name def human_name _('artifacts') end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 51a68ca933d..fbf932e3f6b 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -2,14 +2,11 @@ module Backup class Builds < Backup::Files - attr_reader :progress - def initialize(progress) - @progress = progress - - super('builds', Settings.gitlab_ci.builds_path) + super(progress, 'builds', Settings.gitlab_ci.builds_path) end + override :human_name def human_name _('builds') end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index de26dbab038..afc84a4b913 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -3,10 +3,10 @@ require 'yaml' module Backup - class Database + class Database < Task + extend ::Gitlab::Utils::Override include Backup::Helper - attr_reader :progress - attr_reader :config, :db_file_name + attr_reader :force, :config IGNORED_ERRORS = [ # Ignore warnings @@ -18,13 +18,14 @@ module Backup ].freeze IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze - def initialize(progress, filename: nil) - @progress = progress + def initialize(progress, force:) + super(progress) @config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash - @db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') + @force = force end - def dump + override :dump + def dump(db_file_name) FileUtils.mkdir_p(File.dirname(db_file_name)) FileUtils.rm_f(db_file_name) compress_rd, compress_wr = IO.pipe @@ -64,12 +65,24 @@ module Backup raise DatabaseBackupError.new(config, db_file_name) unless success end - def restore + override :restore + def restore(db_file_name) + unless force + progress.puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) + sleep(5) + end + + # Drop all tables Load the schema to ensure we don't have any newer tables + # hanging out from a failed upgrade + puts_time 'Cleaning the database ... '.color(:blue) + Rake::Task['gitlab:db:drop_tables'].invoke + puts_time 'done'.color(:green) + decompress_rd, decompress_wr = IO.pipe decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) decompress_wr.close - status, errors = + status, @errors = case config[:adapter] when "postgresql" then progress.print "Restoring PostgreSQL database #{database} ... " @@ -81,33 +94,47 @@ module Backup Process.waitpid(decompress_pid) success = $?.success? && status.success? - if errors.present? + if @errors.present? progress.print "------ BEGIN ERRORS -----\n".color(:yellow) - progress.print errors.join.color(:yellow) + progress.print @errors.join.color(:yellow) progress.print "------ END ERRORS -------\n".color(:yellow) end report_success(success) raise Backup::Error, 'Restore failed' unless success + end - if errors.present? - warning = <<~MSG - There were errors in restoring the schema. This may cause - issues if this results in missing indexes, constraints, or - columns. Please record the errors above and contact GitLab - Support if you have questions: - https://about.gitlab.com/support/ - MSG - - warn warning.color(:red) - Gitlab::TaskHelpers.ask_to_continue - end + override :pre_restore_warning + def pre_restore_warning + return if force + + <<-MSG.strip_heredoc + Be sure to stop Puma, Sidekiq, and any other process that + connects to the database before proceeding. For Omnibus + installs, see the following link for more information: + https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations + + Before restoring the database, we will remove all existing + tables to avoid future upgrade problems. Be aware that if you have + custom tables in the GitLab database these tables and all data will be + removed. + MSG end - def enabled - true + override :post_restore_warning + def post_restore_warning + return unless @errors.present? + + <<-MSG.strip_heredoc + There were errors in restoring the schema. This may cause + issues if this results in missing indexes, constraints, or + columns. Please record the errors above and contact GitLab + Support if you have questions: + https://about.gitlab.com/support/ + MSG end + override :human_name def human_name _('database') end diff --git a/lib/backup/files.rb b/lib/backup/files.rb index db6278360a3..7fa07e40cee 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -1,25 +1,27 @@ # frozen_string_literal: true require 'open3' -require_relative 'helper' module Backup - class Files + class Files < Task + extend ::Gitlab::Utils::Override include Backup::Helper DEFAULT_EXCLUDE = 'lost+found' - attr_reader :name, :backup_tarball, :excludes + attr_reader :name, :excludes + + def initialize(progress, name, app_files_dir, excludes: []) + super(progress) - def initialize(name, app_files_dir, excludes: []) @name = name @app_files_dir = app_files_dir - @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') @excludes = [DEFAULT_EXCLUDE].concat(excludes) end # Copy files from public/files to backup/files - def dump + override :dump + def dump(backup_tarball) FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.rm_f(backup_tarball) @@ -35,7 +37,7 @@ module Backup unless status == 0 puts output - raise_custom_error + raise_custom_error(backup_tarball) end tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{backup_files_realpath} -cf - .]].flatten @@ -47,11 +49,12 @@ module Backup end unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output) - raise_custom_error + raise_custom_error(backup_tarball) end end - def restore + override :restore + def restore(backup_tarball) backup_existing_files_dir cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]] @@ -61,10 +64,6 @@ module Backup end end - def enabled - true - end - def tar if system(*%w[gtar --version], out: '/dev/null') # It looks like we can get GNU tar by running 'gtar' @@ -146,7 +145,7 @@ module Backup end end - def raise_custom_error + def raise_custom_error(backup_tarball) raise FileBackupError.new(app_files_realpath, backup_tarball) end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index 8ac09e94004..b688ff7f13b 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -9,13 +9,16 @@ module Backup # @param [StringIO] progress IO interface to output progress # @param [Integer] max_parallelism max parallelism when running backups # @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism) - def initialize(progress, max_parallelism: nil, storage_parallelism: nil) + # @param [String] backup_id unique identifier for the backup + def initialize(progress, max_parallelism: nil, storage_parallelism: nil, incremental: false, backup_id: nil) @progress = progress @max_parallelism = max_parallelism @storage_parallelism = storage_parallelism + @incremental = incremental + @backup_id = backup_id end - def start(type) + def start(type, backup_repos_path) raise Error, 'already started' if started? command = case type @@ -30,6 +33,13 @@ module Backup args = [] args += ['-parallel', @max_parallelism.to_s] if @max_parallelism args += ['-parallel-storage', @storage_parallelism.to_s] if @storage_parallelism + if Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml) + args += ['-layout', 'pointer'] + if type == :create + args += ['-incremental'] if @incremental + args += ['-id', @backup_id] if @backup_id + end + end @input_stream, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args) @@ -93,10 +103,6 @@ module Backup @thread.present? end - def backup_repos_path - File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories')) - end - def bin_path File.absolute_path(Gitlab.config.backup.gitaly_backup_path) end diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb index bbd83cd2157..89ed27cfa13 100644 --- a/lib/backup/gitaly_rpc_backup.rb +++ b/lib/backup/gitaly_rpc_backup.rb @@ -7,10 +7,11 @@ module Backup @progress = progress end - def start(type) + def start(type, backup_repos_path) raise Error, 'already started' if @type @type = type + @backup_repos_path = backup_repos_path case type when :create FileUtils.rm_rf(backup_repos_path) @@ -31,7 +32,7 @@ module Backup backup_restore = BackupRestore.new( progress, repository_type.repository_for(container), - backup_repos_path + @backup_repos_path ) case @type @@ -52,10 +53,6 @@ module Backup attr_reader :progress - def backup_repos_path - @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories') - end - class BackupRestore attr_accessor :progress, :repository, :backup_repos_path diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 17f7b8bf8b0..e92f235a2d7 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -2,14 +2,11 @@ module Backup class Lfs < Backup::Files - attr_reader :progress - def initialize(progress) - @progress = progress - - super('lfs', Settings.lfs.storage_path) + super(progress, 'lfs', Settings.lfs.storage_path) end + override :human_name def human_name _('lfs objects') end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 5b393cf9477..6e90824fce2 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -2,43 +2,84 @@ module Backup class Manager - ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs terraform_state registry packages].freeze - FOLDERS_TO_BACKUP = %w[repositories db].freeze FILE_NAME_SUFFIX = '_gitlab_backup.tar' + MANIFEST_NAME = 'backup_information.yml' + + TaskDefinition = Struct.new( + :destination_path, # Where the task should put its backup file/dir. + :destination_optional, # `true` if the destination might not exist on a successful backup. + :cleanup_path, # Path to remove after a successful backup. Uses `destination_path` when not specified. + :task, + keyword_init: true + ) attr_reader :progress - def initialize(progress) + def initialize(progress, definitions: nil) @progress = progress max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i - - @tasks = { - 'db' => Database.new(progress), - 'repositories' => Repositories.new(progress, - strategy: repository_backup_strategy, - max_concurrency: max_concurrency, - max_storage_concurrency: max_storage_concurrency), - 'uploads' => Uploads.new(progress), - 'builds' => Builds.new(progress), - 'artifacts' => Artifacts.new(progress), - 'pages' => Pages.new(progress), - 'lfs' => Lfs.new(progress), - 'terraform_state' => TerraformState.new(progress), - 'registry' => Registry.new(progress), - 'packages' => Packages.new(progress) + force = ENV['force'] == 'yes' + incremental = Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false) + + @definitions = definitions || { + 'db' => TaskDefinition.new( + destination_path: 'db/database.sql.gz', + cleanup_path: 'db', + task: Database.new(progress, force: force) + ), + 'repositories' => TaskDefinition.new( + destination_path: 'repositories', + destination_optional: true, + task: Repositories.new(progress, + strategy: repository_backup_strategy(incremental), + max_concurrency: max_concurrency, + max_storage_concurrency: max_storage_concurrency) + ), + 'uploads' => TaskDefinition.new( + destination_path: 'uploads.tar.gz', + task: Uploads.new(progress) + ), + 'builds' => TaskDefinition.new( + destination_path: 'builds.tar.gz', + task: Builds.new(progress) + ), + 'artifacts' => TaskDefinition.new( + destination_path: 'artifacts.tar.gz', + task: Artifacts.new(progress) + ), + 'pages' => TaskDefinition.new( + destination_path: 'pages.tar.gz', + task: Pages.new(progress) + ), + 'lfs' => TaskDefinition.new( + destination_path: 'lfs.tar.gz', + task: Lfs.new(progress) + ), + 'terraform_state' => TaskDefinition.new( + destination_path: 'terraform_state.tar.gz', + task: TerraformState.new(progress) + ), + 'registry' => TaskDefinition.new( + destination_path: 'registry.tar.gz', + task: Registry.new(progress) + ), + 'packages' => TaskDefinition.new( + destination_path: 'packages.tar.gz', + task: Packages.new(progress) + ) }.freeze end def create - @tasks.keys.each do |task_name| + @definitions.keys.each do |task_name| run_create_task(task_name) end - write_info + write_backup_information - if ENV['SKIP'] && ENV['SKIP'].include?('tar') + if skipped?('tar') upload else pack @@ -54,21 +95,23 @@ module Backup end def run_create_task(task_name) - task = @tasks[task_name] + definition = @definitions[task_name] - puts_time "Dumping #{task.human_name} ... ".color(:blue) + build_backup_information + puts_time "Dumping #{definition.task.human_name} ... ".color(:blue) - unless task.enabled + unless definition.task.enabled puts_time "[DISABLED]".color(:cyan) return end - if ENV["SKIP"] && ENV["SKIP"].include?(task_name) + if skipped?(task_name) puts_time "[SKIPPED]".color(:cyan) return end - task.dump + definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path)) + puts_time "done".color(:green) rescue Backup::DatabaseBackupError, Backup::FileBackupError => e @@ -77,42 +120,11 @@ module Backup def restore cleanup_required = unpack + read_backup_information verify_backup_version - unless skipped?('db') - begin - unless ENV['force'] == 'yes' - warning = <<-MSG.strip_heredoc - Be sure to stop Puma, Sidekiq, and any other process that - connects to the database before proceeding. For Omnibus - installs, see the following link for more information: - https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations - - Before restoring the database, we will remove all existing - tables to avoid future upgrade problems. Be aware that if you have - custom tables in the GitLab database these tables and all data will be - removed. - MSG - puts warning.color(:red) - Gitlab::TaskHelpers.ask_to_continue - puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) - sleep(5) - end - - # Drop all tables Load the schema to ensure we don't have any newer tables - # hanging out from a failed upgrade - puts_time 'Cleaning the database ... '.color(:blue) - Rake::Task['gitlab:db:drop_tables'].invoke - puts_time 'done'.color(:green) - run_restore_task('db') - rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".color(:red) - exit 1 - end - end - - @tasks.except('db').keys.each do |task_name| - run_restore_task(task_name) unless skipped?(task_name) + @definitions.keys.each do |task_name| + run_restore_task(task_name) if !skipped?(task_name) && enabled_task?(task_name) end Rake::Task['gitlab:shell:setup'].invoke @@ -130,30 +142,71 @@ module Backup end def run_restore_task(task_name) - task = @tasks[task_name] + definition = @definitions[task_name] - puts_time "Restoring #{task.human_name} ... ".color(:blue) + read_backup_information + puts_time "Restoring #{definition.task.human_name} ... ".color(:blue) - unless task.enabled + unless definition.task.enabled puts_time "[DISABLED]".color(:cyan) return end - task.restore + warning = definition.task.pre_restore_warning + if warning.present? + puts_time warning.color(:red) + Gitlab::TaskHelpers.ask_to_continue + end + + definition.task.restore(File.join(Gitlab.config.backup.path, definition.destination_path)) + puts_time "done".color(:green) + + warning = definition.task.post_restore_warning + if warning.present? + puts_time warning.color(:red) + Gitlab::TaskHelpers.ask_to_continue + end + + rescue Gitlab::TaskAbortedByUserError + puts_time "Quitting...".color(:red) + exit 1 end - def write_info + private + + def read_backup_information + @backup_information ||= YAML.load_file(File.join(backup_path, MANIFEST_NAME)) + end + + def write_backup_information # Make sure there is a connection ActiveRecord::Base.connection.reconnect! Dir.chdir(backup_path) do - File.open("#{backup_path}/backup_information.yml", "w+") do |file| + File.open("#{backup_path}/#{MANIFEST_NAME}", "w+") do |file| file << backup_information.to_yaml.gsub(/^---\n/, '') end end end + def build_backup_information + @backup_information ||= { + db_version: ActiveRecord::Migrator.current_version.to_s, + backup_created_at: Time.now, + gitlab_version: Gitlab::VERSION, + tar_version: tar_version, + installation_type: Gitlab::INSTALLATION_TYPE, + skipped: ENV["SKIP"] + } + end + + def backup_information + raise Backup::Error, "#{MANIFEST_NAME} not yet loaded" unless @backup_information + + @backup_information + end + def pack Dir.chdir(backup_path) do # create archive @@ -182,8 +235,11 @@ module Backup upload = directory.files.create(create_attributes) if upload - progress.puts "done".color(:green) - upload + if upload.respond_to?(:encryption) && upload.encryption + progress.puts "done (encrypted with #{upload.encryption})".color(:green) + else + progress.puts "done".color(:green) + end else puts "uploading backup to #{remote_directory} failed".color(:red) raise Backup::Error, 'Backup failed' @@ -193,18 +249,19 @@ module Backup def cleanup progress.print "Deleting tmp directories ... " - backup_contents.each do |dir| - next unless File.exist?(File.join(backup_path, dir)) - - if FileUtils.rm_rf(File.join(backup_path, dir)) - progress.puts "done".color(:green) - else - puts "deleting tmp directory '#{dir}' failed".color(:red) - raise Backup::Error, 'Backup failed' - end + remove_backup_path(MANIFEST_NAME) + @definitions.each do |_, definition| + remove_backup_path(definition.cleanup_path || definition.destination_path) end end + def remove_backup_path(path) + return unless File.exist?(File.join(backup_path, path)) + + FileUtils.rm_rf(File.join(backup_path, path)) + progress.puts "done".color(:green) + end + def remove_tmp # delete tmp inside backups progress.print "Deleting backups/tmp ... " @@ -255,15 +312,15 @@ module Backup def verify_backup_version Dir.chdir(backup_path) do # restoring mismatching backups can lead to unexpected problems - if settings[:gitlab_version] != Gitlab::VERSION + if backup_information[:gitlab_version] != Gitlab::VERSION progress.puts(<<~HEREDOC.color(:red)) GitLab version mismatch: Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup! Please switch to the following version and try again: - version: #{settings[:gitlab_version]} + version: #{backup_information[:gitlab_version]} HEREDOC progress.puts - progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" + progress.puts "Hint: git checkout v#{backup_information[:gitlab_version]}" exit 1 end end @@ -319,13 +376,11 @@ module Backup end def skipped?(item) - settings[:skipped] && settings[:skipped].include?(item) || !enabled_task?(item) + backup_information[:skipped] && backup_information[:skipped].include?(item) end - private - def enabled_task?(task_name) - @tasks[task_name].enabled + @definitions[task_name].task.enabled end def backup_file?(file) @@ -333,7 +388,7 @@ module Backup end def non_tarred_backup? - File.exist?(File.join(backup_path, 'backup_information.yml')) + File.exist?(File.join(backup_path, MANIFEST_NAME)) end def backup_path @@ -380,19 +435,10 @@ module Backup end def backup_contents - folders_to_backup + archives_to_backup + ["backup_information.yml"] - end - - def archives_to_backup - ARCHIVES_TO_BACKUP.map { |name| (name + ".tar.gz") unless skipped?(name) }.compact - end - - def folders_to_backup - FOLDERS_TO_BACKUP.select { |name| !skipped?(name) && Dir.exist?(File.join(backup_path, name)) } - end - - def settings - @settings ||= YAML.load_file("backup_information.yml") + [MANIFEST_NAME] + @definitions.reject do |name, definition| + skipped?(name) || !enabled_task?(name) || + (definition.destination_optional && !File.exist?(File.join(backup_path, definition.destination_path))) + end.values.map(&:destination_path) end def tar_file @@ -403,17 +449,6 @@ module Backup end end - def backup_information - @backup_information ||= { - db_version: ActiveRecord::Migrator.current_version.to_s, - backup_created_at: Time.now, - gitlab_version: Gitlab::VERSION, - tar_version: tar_version, - installation_type: Gitlab::INSTALLATION_TYPE, - skipped: ENV["SKIP"] - } - end - def create_attributes attrs = { key: remote_target, @@ -447,11 +482,11 @@ module Backup Gitlab.config.backup.upload.connection&.provider&.downcase == 'google' end - def repository_backup_strategy + def repository_backup_strategy(incremental) if Feature.enabled?(:gitaly_backup, default_enabled: :yaml) max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence - Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) + Backup::GitalyBackup.new(progress, incremental: incremental, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) else Backup::GitalyRpcBackup.new(progress) end diff --git a/lib/backup/packages.rb b/lib/backup/packages.rb index 037ff31fd9b..9384e007162 100644 --- a/lib/backup/packages.rb +++ b/lib/backup/packages.rb @@ -2,14 +2,11 @@ module Backup class Packages < Backup::Files - attr_reader :progress - def initialize(progress) - @progress = progress - - super('packages', Settings.packages.storage_path, excludes: ['tmp']) + super(progress, 'packages', Settings.packages.storage_path, excludes: ['tmp']) end + override :human_name def human_name _('packages') end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index 724972d212d..ebed6820724 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -6,14 +6,11 @@ module Backup # if some of these files are still there, we don't need them in the backup LEGACY_PAGES_TMP_PATH = '@pages.tmp' - attr_reader :progress - def initialize(progress) - @progress = progress - - super('pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH]) + super(progress, 'pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH]) end + override :human_name def human_name _('pages') end diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index 7ba3a9e9c60..68ea635034d 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -2,18 +2,16 @@ module Backup class Registry < Backup::Files - attr_reader :progress - def initialize(progress) - @progress = progress - - super('registry', Settings.registry.path) + super(progress, 'registry', Settings.registry.path) end + override :human_name def human_name _('container registry images') end + override :enabled def enabled Gitlab.config.registry.enabled end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index e7c3e869928..3633ebd661e 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -3,16 +3,20 @@ require 'yaml' module Backup - class Repositories + class Repositories < Task + extend ::Gitlab::Utils::Override + def initialize(progress, strategy:, max_concurrency: 1, max_storage_concurrency: 1) - @progress = progress + super(progress) + @strategy = strategy @max_concurrency = max_concurrency @max_storage_concurrency = max_storage_concurrency end - def dump - strategy.start(:create) + override :dump + def dump(path) + strategy.start(:create, path) # gitaly-backup is designed to handle concurrency on its own. So we want # to avoid entering the buggy concurrency code here when gitaly-backup @@ -50,8 +54,9 @@ module Backup strategy.finish! end - def restore - strategy.start(:restore) + override :restore + def restore(path) + strategy.start(:restore, path) enqueue_consecutive ensure @@ -61,17 +66,14 @@ module Backup restore_object_pools end - def enabled - true - end - + override :human_name def human_name _('repositories') end private - attr_reader :progress, :strategy, :max_concurrency, :max_storage_concurrency + attr_reader :strategy, :max_concurrency, :max_storage_concurrency def check_valid_storages! repository_storage_klasses.each do |klass| diff --git a/lib/backup/task.rb b/lib/backup/task.rb new file mode 100644 index 00000000000..15cd2aa64d3 --- /dev/null +++ b/lib/backup/task.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Backup + class Task + def initialize(progress) + @progress = progress + end + + # human readable task name used for logging + def human_name + raise NotImplementedError + end + + # dump task backup to `path` + def dump(path) + raise NotImplementedError + end + + # restore task backup from `path` + def restore(path) + raise NotImplementedError + end + + # a string returned here will be displayed to the user before calling #restore + def pre_restore_warning + end + + # a string returned here will be displayed to the user after calling #restore + def post_restore_warning + end + + # returns `true` when the task should be used + def enabled + true + end + + private + + attr_reader :progress + + def puts_time(msg) + progress.puts "#{Time.zone.now} -- #{msg}" + Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") + end + end +end diff --git a/lib/backup/terraform_state.rb b/lib/backup/terraform_state.rb index be82793fe03..05f61d248be 100644 --- a/lib/backup/terraform_state.rb +++ b/lib/backup/terraform_state.rb @@ -2,14 +2,11 @@ module Backup class TerraformState < Backup::Files - attr_reader :progress - def initialize(progress) - @progress = progress - - super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp']) + super(progress, 'terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp']) end + override :human_name def human_name _('terraform states') end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 7048a9a8ff5..700f2af4415 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -2,14 +2,11 @@ module Backup class Uploads < Backup::Files - attr_reader :progress - def initialize(progress) - @progress = progress - - super('uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp']) + super(progress, 'uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp']) end + override :human_name def human_name _('uploads') end diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb index 705400a5497..c788137e122 100644 --- a/lib/banzai/filter/front_matter_filter.rb +++ b/lib/banzai/filter/front_matter_filter.rb @@ -9,7 +9,10 @@ module Banzai html.sub(Gitlab::FrontMatter::PATTERN) do |_match| lang = $~[:lang].presence || lang_mapping[$~[:delim]] - ["```#{lang}:frontmatter", $~[:front_matter].strip!, "```", "\n"].join("\n") + before = $~[:before] + before = "\n#{before}" if $~[:encoding].presence + + "#{before}```#{lang}:frontmatter\n#{$~[:front_matter]}```\n" end end end diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index ed0a01e6277..44acc7805b4 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -8,11 +8,17 @@ module Banzai # Find every image that isn't already wrapped in an `a` tag, create # a new node (a link to the image source), copy the image as a child # of the anchor, and then replace the img with the link-wrapped version. + # + # If `link_replaces_image` context parameter is provided, the image is going + # to be replaced with a link to an image. def call doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| + link_replaces_image = !!context[:link_replaces_image] + html_class = link_replaces_image ? 'with-attachment-icon' : 'no-attachment-icon' + link = doc.document.create_element( 'a', - class: 'no-attachment-icon', + class: html_class, href: img['data-src'] || img['src'], target: '_blank', rel: 'noopener noreferrer' @@ -21,7 +27,11 @@ module Banzai # make sure the original non-proxied src carries over to the link link['data-canonical-src'] = img['data-canonical-src'] if img['data-canonical-src'] - link.children = img.clone + link.children = if link_replaces_image + img['alt'] || img['data-src'] || img['src'] + else + img.clone + end img.replace(link) end diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index c6b402575cb..896f67cb875 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -9,6 +9,9 @@ require 'task_list/filter' module Banzai module Filter class TaskListFilter < TaskList::Filter + def render_item_checkbox(item) + "<task-button></task-button>#{super}" + end end end end diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index eb3d551d1d7..037da5e0816 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -123,7 +123,7 @@ module BulkImports def with_error_handling response = yield - raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}", response: response) unless response.success? + raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}. Body: #{response.parsed_response}", response: response) unless response.success? response rescue *Gitlab::HTTP::HTTP_ERRORS => e diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index add238350dd..4b2250d089d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -10,6 +10,21 @@ module ContainerRegistry REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features' REGISTRY_TAG_DELETE_FEATURE = 'tag_delete' + ALLOWED_REDIRECT_SCHEMES = %w[http https].freeze + REDIRECT_OPTIONS = { + clear_authorization_header: true, + limit: 3, + cookies: [], + callback: -> (response_env, request_env) do + request_env.request_headers.delete(::FaradayMiddleware::FollowRedirects::AUTH_HEADER) + + redirect_to = request_env.url + unless redirect_to.scheme.in?(ALLOWED_REDIRECT_SCHEMES) + raise ArgumentError, "Invalid scheme for #{redirect_to}" + end + end + }.freeze + def self.supports_tag_delete? with_dummy_client(return_value_if_disabled: false) do |client| client.supports_tag_delete? @@ -136,6 +151,10 @@ module ContainerRegistry def faraday_blob @faraday_blob ||= faraday_base do |conn| initialize_connection(conn, @options) + + if Feature.enabled?(:container_registry_follow_redirects_middleware, default_enabled: :yaml) + conn.use ::FaradayMiddleware::FollowRedirects, REDIRECT_OPTIONS + end end end end diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index 20b8e1d419b..3cd7003d1f8 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -31,8 +31,10 @@ module ContainerRegistry registry_features = Gitlab::CurrentSettings.container_registry_features || [] next true if ::Gitlab.com? && registry_features.include?(REGISTRY_GITLAB_V1_API_FEATURE) - response = faraday.get('/gitlab/v1/') - response.success? || response.status == 401 + with_token_faraday do |faraday_client| + response = faraday_client.get('/gitlab/v1/') + response.success? || response.status == 401 + end end end @@ -50,15 +52,46 @@ module ContainerRegistry # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#get-repository-import-status def import_status(path) - body_hash = response_body(faraday.get(import_url_for(path))) - body_hash['status'] || 'error' + with_import_token_faraday do |faraday_client| + body_hash = response_body(faraday_client.get(import_url_for(path))) + body_hash['status'] || 'error' + end + end + + def repository_details(path, with_size: false) + with_token_faraday do |faraday_client| + req = faraday_client.get("/gitlab/v1/repositories/#{path}/") do |req| + req.params['size'] = 'self' if with_size + end + + break {} unless req.success? + + response_body(req) + end end private def start_import_for(path, pre:) - faraday.put(import_url_for(path)) do |req| - req.params['pre'] = pre.to_s + with_import_token_faraday do |faraday_client| + faraday_client.put(import_url_for(path)) do |req| + req.params['import_type'] = pre ? 'pre' : 'final' + end + end + end + + def with_token_faraday + yield faraday + end + + def with_import_token_faraday + yield faraday_with_import_token + end + + def faraday_with_import_token(timeout_enabled: true) + @faraday_with_import_token ||= faraday_base(timeout_enabled: timeout_enabled) do |conn| + # initialize the connection with the :import_token instead of :token + initialize_connection(conn, @options.merge(token: @options[:import_token]), &method(:configure_connection)) end end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 710f8169a00..b377f7d0ac3 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -2,26 +2,16 @@ module ContainerRegistry class Registry - include Gitlab::Utils::StrongMemoize - - attr_reader :uri, :client, :path + attr_reader :uri, :client, :gitlab_api_client, :path def initialize(uri, options = {}) @uri = uri @options = options @path = @options[:path] || default_path @client = ContainerRegistry::Client.new(@uri, @options) - end - - def gitlab_api_client - strong_memoize(:gitlab_api_client) do - token = Auth::ContainerRegistryAuthenticationService.import_access_token - - url = Gitlab.config.registry.api_url - host_port = Gitlab.config.registry.host_port - ContainerRegistry::GitlabApiClient.new(url, token: token, path: host_port) - end + import_token = Auth::ContainerRegistryAuthenticationService.import_access_token + @gitlab_api_client = ContainerRegistry::GitlabApiClient.new(@uri, @options.merge(import_token: import_token)) end private diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 2a32f950457..04a8e1d2e8f 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -104,7 +104,7 @@ module ContainerRegistry def total_size return unless layers - layers.map(&:size).sum if v2? + layers.sum(&:size) if v2? end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 2449554d3c0..d33120575a2 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -49,9 +49,15 @@ module Gitlab INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze HTTP_PROXY_ENV_VARS = %w(http_proxy https_proxy HTTP_PROXY HTTPS_PROXY).freeze + def self.simulate_com? + return false unless Rails.env.development? + + Gitlab::Utils.to_boolean(ENV['GITLAB_SIMULATE_SAAS']) + end + def self.com? # Check `gl_subdomain?` as well to keep parity with gitlab.com - Gitlab.config.gitlab.url == Gitlab::Saas.com_url || gl_subdomain? + simulate_com? || Gitlab.config.gitlab.url == Gitlab::Saas.com_url || gl_subdomain? end def self.com @@ -82,12 +88,8 @@ module Gitlab Gitlab::Saas.subdomain_regex === Gitlab.config.gitlab.url end - def self.dev_env_org_or_com? - dev_env_or_com? || org? - end - - def self.dev_env_or_com? - Rails.env.development? || com? + def self.org_or_com? + org? || com? end def self.dev_or_test_env? diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index bc9d94ef09c..af695c5cfa4 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -80,12 +80,13 @@ module Gitlab direction: direction&.to_sym, page: page, end_event_filter: end_event_filter.to_sym, - use_aggregated_data_collector: Feature.enabled?(:use_vsa_aggregated_tables, group || project, default_enabled: :yaml) + use_aggregated_data_collector: use_aggregated_backend? }.merge(attributes.symbolize_keys.slice(*FINDER_PARAM_NAMES)) end def to_data_attributes {}.tap do |attrs| + attrs[:aggregation] = aggregation_attributes if group attrs[:group] = group_data_attributes if group attrs[:value_stream] = value_stream_data_attributes.to_json if value_stream attrs[:created_after] = created_after.to_date.iso8601 @@ -103,6 +104,24 @@ module Gitlab private + def use_aggregated_backend? + group.present? && # for now it's only available on the group-level + aggregation.enabled && + Feature.enabled?(:use_vsa_aggregated_tables, group, default_enabled: :yaml) + end + + def aggregation_attributes + { + enabled: aggregation.enabled.to_s, + last_run_at: aggregation.last_incremental_run_at&.iso8601, + next_run_at: aggregation.estimated_next_run_at&.iso8601 + } + end + + def aggregation + @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_group(group) + end + def group_data_attributes { id: group.id, diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index d2a31938e89..0b0aaacbaff 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -39,7 +39,8 @@ module Gitlab profile_update_username: { threshold: 10, interval: 1.minute }, update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, - user_email_lookup: { threshold: -> { application_settings.user_email_lookup_limit }, interval: 1.minute }, + search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, + search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, gitlab_shell_operation: { threshold: 600, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index ecda96af403..7adaaef86e4 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -12,6 +12,7 @@ module Gitlab class InsufficientScopeError < AuthenticationError attr_reader :scopes + def initialize(scopes) @scopes = scopes.map { |s| s.try(:name) || s } end diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index d134350775d..56c2af1910e 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -11,9 +11,6 @@ module Gitlab module Ldap class User < Gitlab::Auth::OAuth::User extend ::Gitlab::Utils::Override - def save - super('LDAP') - end # instance methods def find_user @@ -44,6 +41,10 @@ module Gitlab def auth_hash=(auth_hash) @auth_hash = Gitlab::Auth::Ldap::AuthHash.new(auth_hash) end + + def protocol_name + 'LDAP' + end end end end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 2ec75669d24..a45778159c7 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -7,6 +7,7 @@ module Gitlab module OAuth class AuthHash attr_reader :auth_hash + def initialize(auth_hash) @auth_hash = auth_hash end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 9f142727ebb..200f1a843e6 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -46,7 +46,7 @@ module Gitlab valid? && persisted? end - def save(provider = 'OAuth') + def save(provider = protocol_name) raise SigninDisabledForProviderError if oauth_provider_disabled? raise SignupDisabledError unless gl_user @@ -55,6 +55,7 @@ module Gitlab Users::UpdateService.new(gl_user, user: gl_user).execute! gl_user.block_pending_approval if block_after_save + activate_user_if_user_cap_not_reached log.info "(#{provider}) saving user #{auth_hash.email} from login with admin => #{gl_user.admin}, extern_uid => #{auth_hash.uid}" gl_user @@ -96,8 +97,16 @@ module Gitlab end end + def protocol_name + 'OAuth' + end + protected + def activate_user_if_user_cap_not_reached + nil + end + def should_save? true end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index b6ed6bbf2df..0948663b4b3 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -42,6 +42,10 @@ module Gitlab nil end + def can_sign_in_bot?(user) + user&.project_bot? && api_request? + end + # To prevent Rack Attack from incorrectly rate limiting # authenticated Git activity, we need to authenticate the user # from other means (e.g. HTTP Basic Authentication) only if the diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index 205d5fe0015..d14da41deb6 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -11,10 +11,6 @@ module Gitlab class User < Gitlab::Auth::OAuth::User extend ::Gitlab::Utils::Override - def save - super('SAML') - end - def find_user user = find_by_uid_and_provider @@ -40,6 +36,10 @@ module Gitlab saml_config.upstream_two_factor_authn_contexts&.include?(auth_hash.authn_context) end + def protocol_name + 'SAML' + end + protected def saml_config diff --git a/lib/gitlab/background_migration/backfill_issue_search_data.rb b/lib/gitlab/background_migration/backfill_issue_search_data.rb new file mode 100644 index 00000000000..ec206cbfd41 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_issue_search_data.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Backfills the new `issue_search_data` table, which contains + # the tsvector from the issue title and description. + class BackfillIssueSearchData + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms) + define_batchable_model(batch_table, connection: ActiveRecord::Base.connection).where(batch_column => start_id..stop_id).each_batch(of: sub_batch_size) do |sub_batch| + update_search_data(sub_batch) + + sleep(pause_ms * 0.001) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + update_search_data_individually(sub_batch, pause_ms) + end + end + + private + + def update_search_data(relation) + relation.klass.connection.execute( + <<~SQL + INSERT INTO issue_search_data (project_id, issue_id, search_vector, created_at, updated_at) + SELECT + project_id, + id, + setweight(to_tsvector('english', LEFT(title, 255)), 'A') || setweight(to_tsvector('english', LEFT(REGEXP_REPLACE(description, '[A-Za-z0-9+/@]{50,}', ' ', 'g'), 1048576)), 'B'), + NOW(), + NOW() + FROM issues + WHERE issues.id IN (#{relation.select(:id).to_sql}) + ON CONFLICT DO NOTHING + SQL + ) + end + + def update_search_data_individually(relation, pause_ms) + relation.pluck(:id).each do |issue_id| + update_search_data(relation.klass.where(id: issue_id)) + + sleep(pause_ms * 0.001) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + logger.error( + message: 'Error updating search data: string is too long for tsvector', + class: relation.klass.name, + model_id: issue_id + ) + end + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb index 61145f6a445..669e5338dd1 100644 --- a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb +++ b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb @@ -79,7 +79,7 @@ module Gitlab end def mark_jobs_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name.demodulize, arguments) end end end diff --git a/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb b/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb new file mode 100644 index 00000000000..1ed147d67c7 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `members.member_namespace_id` column for `type=GroupMember` + class BackfillMemberNamespaceForGroupMembers + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:update_all) do + sub_batch.update_all('member_namespace_id=source_id') + end + + pause_ms = [0, pause_ms].max + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + .joins('INNER JOIN namespaces ON members.source_id = namespaces.id') + .where(source_key_column => start_id..stop_id) + .where(type: 'GroupMember') + .where(source_type: 'Namespace') + .where(member_namespace_id: nil) + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/base_strategy.rb b/lib/gitlab/background_migration/batching_strategies/base_strategy.rb new file mode 100644 index 00000000000..37bddea4f61 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/base_strategy.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Simple base class for batching strategy job classes. + # + # Any strategy class that inherits from the base class will have connection to the tracking database set on + # initialization. + class BaseStrategy + def initialize(connection:) + @connection = connection + end + + def next_batch(*arguments) + raise NotImplementedError, + "#{self.class} does not implement #{__method__}" + end + + private + + attr_reader :connection + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index 09700438d47..5569bac0e19 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -8,7 +8,7 @@ module Gitlab # values for the next batch as an array. # # If no more batches exist in the table, returns nil. - class PrimaryKeyBatchingStrategy + class PrimaryKeyBatchingStrategy < BaseStrategy include Gitlab::Database::DynamicModelHelpers # Finds and returns the next batch in the table. @@ -19,7 +19,7 @@ module Gitlab # batch_size - The size of the next batch # job_arguments - The migration job arguments def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:) - model_class = define_batchable_model(table_name, connection: ActiveRecord::Base.connection) + model_class = define_batchable_model(table_name, connection: connection) quoted_column_name = model_class.connection.quote_column_name(column_name) relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb new file mode 100644 index 00000000000..3843356af69 --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates the integration.properties column from plaintext to encrypted text. + class EncryptIntegrationProperties + # The Integration model, with just the relevant bits. + class Integration < ActiveRecord::Base + include EachBatch + + ALGORITHM = 'aes-256-gcm' + + self.table_name = 'integrations' + self.inheritance_column = :_type_disabled + + scope :with_properties, -> { where.not(properties: nil) } + scope :not_already_encrypted, -> { where(encrypted_properties: nil) } + scope :for_batch, ->(range) { where(id: range) } + + attr_encrypted :encrypted_properties_tmp, + attribute: :encrypted_properties, + mode: :per_attribute_iv, + key: ::Settings.attr_encrypted_db_key_base_32, + algorithm: ALGORITHM, + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false + + # See 'Integration#reencrypt_properties' + def encrypt_properties + data = ::Gitlab::Json.parse(properties) + iv = generate_iv(ALGORITHM) + ep = self.class.encrypt(:encrypted_properties_tmp, data, { iv: iv }) + + [ep, iv] + end + end + + def perform(start_id, stop_id) + batch_query = Integration.with_properties.not_already_encrypted.for_batch(start_id..stop_id) + encrypt_batch(batch_query) + mark_job_as_succeeded(start_id, stop_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + + # represent binary string as a PSQL binary literal: + # https://www.postgresql.org/docs/9.4/datatype-binary.html + def bytea(value) + "'\\x#{value.unpack1('H*')}'::bytea" + end + + def encrypt_batch(batch_query) + values = batch_query.select(:id, :properties).map do |record| + encrypted_properties, encrypted_properties_iv = record.encrypt_properties + "(#{record.id}, #{bytea(encrypted_properties)}, #{bytea(encrypted_properties_iv)})" + end + + return if values.empty? + + Integration.connection.execute(<<~SQL.squish) + WITH cte(cte_id, cte_encrypted_properties, cte_encrypted_properties_iv) + AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT * + FROM (VALUES #{values.join(',')}) AS t (id, encrypted_properties, encrypted_properties_iv) + ) + UPDATE #{Integration.table_name} + SET encrypted_properties = cte_encrypted_properties + , encrypted_properties_iv = cte_encrypted_properties_iv + FROM cte + WHERE cte_id = id + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb index 2b049ea2d2f..a34e923545c 100644 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -59,7 +59,7 @@ module Gitlab private def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( 'FixVulnerabilityOccurrencesWithHashesAsRawMetadata', arguments ) diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index b7d47c389df..acbb5f76ad8 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -50,34 +50,41 @@ module Gitlab Gitlab::Database::SharedModel.using_connection(connection, &block) end - def steal(steal_class, retry_dead_jobs: false) - with_shared_connection do + def pending_jobs(include_dead_jobs: false) + Enumerator.new do |y| queues = [ Sidekiq::ScheduledSet.new, Sidekiq::Queue.new(self.queue) ] - if retry_dead_jobs + if include_dead_jobs queues << Sidekiq::RetrySet.new queues << Sidekiq::DeadSet.new end queues.each do |queue| queue.each do |job| - migration_class, migration_args = job.args + y << job if job.klass == worker_class.name + end + end + end + end + + def steal(steal_class, retry_dead_jobs: false) + with_shared_connection do + pending_jobs(include_dead_jobs: retry_dead_jobs).each do |job| + migration_class, migration_args = job.args - next unless job.klass == worker_class.name - next unless migration_class == steal_class - next if block_given? && !(yield job) + next unless migration_class == steal_class + next if block_given? && !(yield job) - begin - perform(migration_class, migration_args) if job.delete - rescue Exception # rubocop:disable Lint/RescueException - worker_class # enqueue this migration again - .perform_async(migration_class, migration_args) + begin + perform(migration_class, migration_args) if job.delete + rescue Exception # rubocop:disable Lint/RescueException + worker_class # enqueue this migration again + .perform_async(migration_class, migration_args) - raise - end + raise end end end diff --git a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb b/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb new file mode 100644 index 00000000000..49eff6e2771 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates personal namespace project `maintainer` memberships (for the associated user only) to OWNER + # Does not create any missing records, simply migrates existing ones + class MigratePersonalNamespaceProjectMaintainerToOwner + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:update_all) do + sub_batch.update_all('access_level = 50') + end + + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + # members of projects within their own personal namespace + + # rubocop: disable CodeReuse/ActiveRecord + define_batchable_model(:members, connection: ApplicationRecord.connection) + .where(source_key_column => start_id..stop_id) + .joins("INNER JOIN projects ON members.source_id = projects.id") + .joins("INNER JOIN namespaces ON projects.namespace_id = namespaces.id") + .where(type: 'ProjectMember') + .where("namespaces.type = 'User'") + .where('members.access_level < 50') + .where('namespaces.owner_id = members.user_id') + end + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb new file mode 100644 index 00000000000..78e897d9ae1 --- /dev/null +++ b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify orphan runner_id on ci_builds table + class NullifyOrphanRunnerIdOnCiBuilds + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + pause_ms = 0 if pause_ms < 0 + + batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| + batch_metrics.time_operation(:update_all) do + sub_batch.update_all(runner_id: nil) + end + + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def connection + ActiveRecord::Base.connection + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: connection) + .joins('LEFT OUTER JOIN ci_runners ON ci_runners.id = ci_builds.runner_id') + .where('ci_builds.runner_id IS NOT NULL AND ci_runners.id IS NULL') + .where(source_key_column => start_id..stop_id) + end + end + end +end diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb index ba3f7c47047..c34cc57ce60 100644 --- a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -34,8 +34,11 @@ module Gitlab def backfill_project_namespaces(namespace_id) project_ids.each_slice(sub_batch_size) do |project_ids| - ActiveRecord::Base.connection.execute("select gin_clean_pending_list('index_namespaces_on_name_trigram')") - ActiveRecord::Base.connection.execute("select gin_clean_pending_list('index_namespaces_on_path_trigram')") + # cleanup gin indexes on namespaces table + cleanup_gin_index('namespaces') + + # cleanup gin indexes on projects table + cleanup_gin_index('projects') # We need to lock these project records for the period when we create project namespaces # and link them to projects so that if a project is modified in the time between creating @@ -53,6 +56,14 @@ module Gitlab end end + def cleanup_gin_index(table_name) + index_names = ActiveRecord::Base.connection.select_values("select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%gin%'") + + index_names.each do |index_name| + ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')") + end + end + def cleanup_backfilled_project_namespaces(namespace_id) project_ids.each_slice(sub_batch_size) do |project_ids| # IMPORTANT: first nullify project_namespace_id in projects table to avoid removing projects when records diff --git a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb new file mode 100644 index 00000000000..d47aa76f24b --- /dev/null +++ b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Removing expire_at timestamps that shouldn't have + # been written to traces on gitlab.com. + class RemoveAllTraceExpirationDates + include Gitlab::Database::MigrationHelpers + + BATCH_SIZE = 1_000 + + # Stubbed class to connect to the CI database + # connects_to has to be called in abstract classes. + class MultiDbAdaptableClass < ActiveRecord::Base + self.abstract_class = true + + if Gitlab::Database.has_config?(:ci) + connects_to database: { writing: :ci, reading: :ci } + end + end + + # Stubbed class to access the ci_job_artifacts table + class JobArtifact < MultiDbAdaptableClass + include EachBatch + + self.table_name = 'ci_job_artifacts' + + TARGET_TIMESTAMPS = [ + Date.new(2021, 04, 22).midnight.utc, + Date.new(2021, 05, 22).midnight.utc, + Date.new(2021, 06, 22).midnight.utc, + Date.new(2022, 01, 22).midnight.utc, + Date.new(2022, 02, 22).midnight.utc, + Date.new(2022, 03, 22).midnight.utc, + Date.new(2022, 04, 22).midnight.utc + ].freeze + + scope :traces, -> { where(file_type: 3) } + scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } + scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) } + end + + def perform(start_id, end_id) + return unless Gitlab.com? + + JobArtifact.traces + .between(start_id, end_id) + .in_targeted_timestamps + .each_batch(of: BATCH_SIZE) { |batch| batch.update_all(expire_at: nil) } + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb new file mode 100644 index 00000000000..80ca76ef37f --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate runners_token_encrypted values in projects table in batches + class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects + class Project < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'projects' + + scope :base_query, -> do + where.not(runners_token_encrypted: nil) + end + end + + def perform(start_id, end_id) + # Reset duplicate runner tokens that would prevent creating an unique index. + duplicate_tokens = Project.base_query + .where(id: start_id..end_id) + .group(:runners_token_encrypted) + .having('COUNT(*) > 1') + .pluck(:runners_token_encrypted) + + Project.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects', arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb new file mode 100644 index 00000000000..d87ce6c88d3 --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate ci_runners_token values in projects table in batches + class ResetDuplicateCiRunnersTokenValuesOnProjects + class Project < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'projects' + + scope :base_query, -> do + where.not(runners_token: nil) + end + end + + def perform(start_id, end_id) + # Reset duplicate runner tokens that would prevent creating an unique index. + duplicate_tokens = Project.base_query + .where(id: start_id..end_id) + .group(:runners_token) + .having('COUNT(*) > 1') + .pluck(:runners_token) + + Project.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnerValuesTokensOnProjects', arguments) + end + end + end +end diff --git a/lib/gitlab/checks/base_bulk_checker.rb b/lib/gitlab/checks/base_bulk_checker.rb index 46a68fdf485..e2a016a9907 100644 --- a/lib/gitlab/checks/base_bulk_checker.rb +++ b/lib/gitlab/checks/base_bulk_checker.rb @@ -4,6 +4,7 @@ module Gitlab module Checks class BaseBulkChecker < BaseChecker attr_reader :changes_access + delegate(*ChangesAccess::ATTRIBUTES, to: :changes_access) def initialize(changes_access) diff --git a/lib/gitlab/checks/base_single_checker.rb b/lib/gitlab/checks/base_single_checker.rb index 06519833d7c..435f4ccf5ba 100644 --- a/lib/gitlab/checks/base_single_checker.rb +++ b/lib/gitlab/checks/base_single_checker.rb @@ -4,6 +4,7 @@ module Gitlab module Checks class BaseSingleChecker < BaseChecker attr_reader :change_access + delegate(*SingleChangeAccess::ATTRIBUTES, to: :change_access) def initialize(change_access) diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 7ade9ca5085..2e5f6611e73 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -36,7 +36,7 @@ module Gitlab # the pattern matching does not work for merge requests pipelines if pipeline.branch? || pipeline.tag? regexp = Gitlab::UntrustedRegexp::RubySyntax - .fabricate(pattern, fallback: true, project: pipeline.project) + .fabricate(pattern, project: pipeline.project) if regexp regexp.match?(pipeline.ref) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 8dd1f686132..06c81fd65dd 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -37,10 +37,12 @@ module Gitlab next unless dependencies.present? next unless needs_value.present? - missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) + if needs_value[:job].nil? && needs_value[:cross_dependency].present? + errors.add(:needs, "corresponding to dependencies must be from the same pipeline") + else + missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) - if missing_needs.any? - errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") + errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any? end end end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 7b14218d3ea..adc3660d950 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -17,7 +17,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable validations do - validates :config, array_of_strings_or_regexps_with_fallback: true + validates :config, array_of_strings_or_regexps: true end def value @@ -38,7 +38,7 @@ module Gitlab validate :variables_expressions_syntax with_options allow_nil: true do - validates :refs, array_of_strings_or_regexps_with_fallback: true + validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] validates :variables, array_of_strings: true validates :changes, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index e45dbfa243f..f8fce1abc06 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -8,6 +8,7 @@ module Gitlab # Entry that represents a configuration of job artifacts. # class Reports < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable @@ -15,10 +16,13 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications - requirements coverage_fuzzing api_fuzzing cluster_image_scanning].freeze + requirements coverage_fuzzing api_fuzzing cluster_image_scanning + coverage_report].freeze attributes ALLOWED_KEYS + entry :coverage_report, Reports::CoverageReport, description: 'Coverage report configuration.' + validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS @@ -47,10 +51,18 @@ module Gitlab validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 validates :requirements, array_of_strings_or_string: true end + + validates :config, mutually_exclusive_keys: [:coverage_report, :cobertura] end def value - @config.transform_values { |v| Array(v) } + @config.transform_values do |value| + if value.is_a?(Hash) + value + else + Array(value) + end + end end end end diff --git a/lib/gitlab/ci/config/entry/reports/coverage_report.rb b/lib/gitlab/ci/config/entry/reports/coverage_report.rb new file mode 100644 index 00000000000..98119c7fd53 --- /dev/null +++ b/lib/gitlab/ci/config/entry/reports/coverage_report.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Reports + class CoverageReport < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[coverage_format path].freeze + SUPPORTED_COVERAGE = %w[cobertura].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + with_options(presence: true) do + validates :coverage_format, inclusion: { in: SUPPORTED_COVERAGE, message: "must be one of supported formats: #{SUPPORTED_COVERAGE.join(', ')}." } + validates :path, type: String + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 840f2d6f31a..4722f2e9a61 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -24,7 +24,7 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay? validates :start_in, presence: true, if: :specifies_delay? - validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay? + validates :start_in, duration: { limit: '1 week' }, if: :specifies_delay? with_options allow_nil: true do validates :if, expression: true diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index c6ba53adfd7..0f94b3f94fe 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -5,12 +5,13 @@ module Gitlab class Config module Entry ## - # Entry that represents a cross-project downstream trigger. + # Entry that represents a parent-child or cross-project downstream trigger. # class Trigger < ::Gitlab::Config::Entry::Simplifiable strategy :SimpleTrigger, if: -> (config) { config.is_a?(String) } strategy :ComplexTrigger, if: -> (config) { config.is_a?(Hash) } + # cross-project class SimpleTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -28,11 +29,13 @@ module Gitlab config.key?(:include) end + # cross-project class CrossProjectTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[project branch strategy].freeze + ALLOWED_KEYS = %i[project branch strategy forward].freeze attributes :project, :branch, :strategy validations do @@ -42,15 +45,26 @@ module Gitlab validates :branch, type: String, allow_nil: true validates :strategy, type: String, inclusion: { in: %w[depend], message: 'should be depend' }, allow_nil: true end + + entry :forward, ::Gitlab::Ci::Config::Entry::Trigger::Forward, + description: 'List what to forward to downstream pipelines' + + def value + { project: project, + branch: branch, + strategy: strategy, + forward: forward_value }.compact + end end + # parent-child class SameProjectTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable INCLUDE_MAX_SIZE = 3 - ALLOWED_KEYS = %i[strategy include].freeze + ALLOWED_KEYS = %i[strategy include forward].freeze attributes :strategy validations do @@ -64,8 +78,13 @@ module Gitlab reserved: true, metadata: { max_size: INCLUDE_MAX_SIZE } + entry :forward, ::Gitlab::Ci::Config::Entry::Trigger::Forward, + description: 'List what to forward to downstream pipelines' + def value - @config + { include: @config[:include], + strategy: strategy, + forward: forward_value }.compact end end diff --git a/lib/gitlab/ci/config/entry/trigger/forward.rb b/lib/gitlab/ci/config/entry/trigger/forward.rb new file mode 100644 index 00000000000..f80f018f149 --- /dev/null +++ b/lib/gitlab/ci/config/entry/trigger/forward.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the configuration for passing attributes to the downstream pipeline + # + class Trigger + class Forward < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[yaml_variables pipeline_variables].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :yaml_variables, boolean: true + validates :pipeline_variables, boolean: true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index fdb3e1b00f9..3839c43bd53 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -33,6 +33,10 @@ module Gitlab def fetch_local_content context.project.repository.blob_data_at(context.sha, location) + rescue GRPC::InvalidArgument + errors.push("Sha #{context.sha} is not valid!") + + nil end override :expand_context_attrs diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb index 22822614b67..45787077c91 100644 --- a/lib/gitlab/ci/config/yaml/tags/reference.rb +++ b/lib/gitlab/ci/config/yaml/tags/reference.rb @@ -27,7 +27,7 @@ module Gitlab override :_resolve def _resolve(resolver) - object = resolver.config.dig(*location) + object = config_at_location(resolver) value = resolver.deep_resolve(object) raise MissingReferenceError, missing_ref_error_message unless value @@ -35,6 +35,12 @@ module Gitlab value end + def config_at_location(resolver) + resolver.config.dig(*location) + rescue TypeError + raise MissingReferenceError, missing_ref_error_message + end + def missing_ref_error_message "#{data[:tag]} #{data[:seq].inspect} could not be found" end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index d6b3af674a6..6041907ef78 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -8,140 +8,8 @@ module Gitlab InvalidXMLError = Class.new(Gitlab::Ci::Parsers::ParserError) InvalidLineInformationError = Class.new(Gitlab::Ci::Parsers::ParserError) - GO_SOURCE_PATTERN = '/usr/local/go/src' - MAX_SOURCES = 100 - def parse!(xml_data, coverage_report, project_path: nil, worktree_paths: nil) - root = Hash.from_xml(xml_data) - - context = { - project_path: project_path, - paths: worktree_paths&.to_set, - sources: [] - } - - parse_all(root, coverage_report, context) - rescue Nokogiri::XML::SyntaxError - raise InvalidXMLError, "XML parsing failed" - end - - private - - def parse_all(root, coverage_report, context) - return unless root.present? - - root.each do |key, value| - parse_node(key, value, coverage_report, context) - end - end - - def parse_node(key, value, coverage_report, context) - if key == 'sources' && value && value['source'].present? - parse_sources(value['source'], context) - elsif key == 'package' - Array.wrap(value).each do |item| - parse_package(item, coverage_report, context) - end - elsif key == 'class' - # This means the cobertura XML does not have classes within package nodes. - # This is possible in some cases like in simple JS project structures - # running Jest. - Array.wrap(value).each do |item| - parse_class(item, coverage_report, context) - end - elsif value.is_a?(Hash) - parse_all(value, coverage_report, context) - elsif value.is_a?(Array) - value.each do |item| - parse_all(item, coverage_report, context) - end - end - end - - def parse_sources(sources, context) - return unless context[:project_path] && context[:paths] - - sources = Array.wrap(sources) - - # TODO: Go cobertura has a different format with how their packages - # are included in the filename. So we can't rely on the sources. - # We'll deal with this later. - return if sources.include?(GO_SOURCE_PATTERN) - - sources.each do |source| - source = build_source_path(source, context) - context[:sources] << source if source.present? - end - end - - def build_source_path(source, context) - # | raw source | extracted | - # |-----------------------------|------------| - # | /builds/foo/test/SampleLib/ | SampleLib/ | - # | /builds/foo/test/something | something | - # | /builds/foo/test/ | nil | - # | /builds/foo/test | nil | - source.split("#{context[:project_path]}/", 2)[1] - end - - def parse_package(package, coverage_report, context) - classes = package.dig('classes', 'class') - return unless classes.present? - - matched_filenames = Array.wrap(classes).map do |item| - parse_class(item, coverage_report, context) - end - - # Remove these filenames from the paths to avoid conflict - # with other packages that may contain the same class filenames - remove_matched_filenames(matched_filenames, context) - end - - def remove_matched_filenames(filenames, context) - return unless context[:paths] - - filenames.each { |f| context[:paths].delete(f) } - end - - def parse_class(file, coverage_report, context) - return unless file["filename"].present? && file["lines"].present? - - parsed_lines = parse_lines(file["lines"]) - filename = determine_filename(file["filename"], context) - - coverage_report.add_file(filename, Hash[parsed_lines]) if filename - - filename - end - - def parse_lines(lines) - line_array = Array.wrap(lines["line"]) - - line_array.map do |line| - # Using `Integer()` here to raise exception on invalid values - [Integer(line["number"]), Integer(line["hits"])] - end - rescue StandardError - raise InvalidLineInformationError, "Line information had invalid values" - end - - def determine_filename(filename, context) - return filename unless context[:sources].any? - - full_filename = nil - - context[:sources].each_with_index do |source, index| - break if index >= MAX_SOURCES - break if full_filename = check_source(source, filename, context) - end - - full_filename - end - - def check_source(source, filename, context) - full_path = File.join(source, filename) - - return full_path if context[:paths].include?(full_path) + Nokogiri::XML::SAX::Parser.new(SaxDocument.new(coverage_report, project_path, worktree_paths)).parse(xml_data) end end end diff --git a/lib/gitlab/ci/parsers/coverage/sax_document.rb b/lib/gitlab/ci/parsers/coverage/sax_document.rb new file mode 100644 index 00000000000..27cce0e3a3b --- /dev/null +++ b/lib/gitlab/ci/parsers/coverage/sax_document.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Coverage + class SaxDocument < Nokogiri::XML::SAX::Document + GO_SOURCE_PATTERN = '/usr/local/go/src' + MAX_SOURCES = 100 + + def initialize(coverage_report, project_path, worktree_paths) + @coverage_report = coverage_report + @project_path = project_path + @paths = worktree_paths&.to_set + + @matched_filenames = [] + @parsed_lines = [] + @sources = [] + end + + def error(error) + raise Cobertura::InvalidXMLError, "XML parsing failed with error: #{error}" + end + + def start_element(node_name, attrs = []) + return unless node_name + + self.node_name = node_name + node_attrs = Hash[attrs] + + if node_name == 'class' && node_attrs["filename"].present? + self.filename = determine_filename(node_attrs["filename"]) + self.matched_filenames << filename if filename + elsif node_name == 'line' + self.parsed_lines << parse_line(node_attrs) + end + end + + def characters(node_content) + if node_name == 'source' + parse_source(node_content) + end + end + + def end_element(node_name) + if node_name == "package" + remove_matched_filenames + elsif node_name == "class" && filename && parsed_lines.present? + coverage_report.add_file(filename, Hash[parsed_lines]) + self.filename = nil + self.parsed_lines = [] + end + end + + private + + attr_accessor :coverage_report, :project_path, :paths, :sources, :node_name, :filename, :parsed_lines, :matched_filenames + + def parse_line(line) + [Integer(line["number"]), Integer(line["hits"])] + rescue StandardError + raise Cobertura::InvalidLineInformationError, "Line information had invalid values" + end + + def parse_source(node) + return unless project_path && paths && !node.include?(GO_SOURCE_PATTERN) + + source = build_source_path(node) + self.sources << source if source.present? + end + + def build_source_path(node) + # | raw source | extracted | + # |-----------------------------|------------| + # | /builds/foo/test/SampleLib/ | SampleLib/ | + # | /builds/foo/test/something | something | + # | /builds/foo/test/ | nil | + # | /builds/foo/test | nil | + node.split("#{project_path}/", 2)[1] + end + + def remove_matched_filenames + return unless paths + + matched_filenames.each { |f| paths.delete(f) } + end + + def determine_filename(filename) + return filename unless sources.any? + + full_filename = nil + + sources.each_with_index do |source, index| + break if index >= MAX_SOURCES + break if full_filename = check_source(source, filename) + end + + full_filename + end + + def check_source(source, filename) + full_path = File.join(source, filename) + + return full_path if paths.include?(full_path) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 9aec615d012..7baae2f53d7 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -19,6 +19,8 @@ module Gitlab end def parse! + set_report_version + return report_data unless valid? raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash) @@ -26,7 +28,6 @@ module Gitlab create_scanner create_scan create_analyzer - set_report_version create_findings @@ -42,14 +43,19 @@ module Gitlab attr_reader :json_data, :report, :validate def valid? - if Feature.enabled?(:enforce_security_report_validation) - if !validate || schema_validator.valid? - report.schema_validation_status = :valid_schema - true + if Feature.enabled?(:show_report_validation_warnings, default_enabled: :yaml) + # We want validation to happen regardless of VALIDATE_SCHEMA CI variable + schema_validation_passed = schema_validator.valid? + + if validate + schema_validator.errors.each { |error| report.add_error('Schema', error) } unless schema_validation_passed + + schema_validation_passed else - report.schema_validation_status = :invalid_schema - schema_validator.errors.each { |error| report.add_error('Schema', error) } - false + # We treat all schema validation errors as warnings + schema_validator.errors.each { |error| report.add_warning('Schema', error) } + + true end else return true if !validate || schema_validator.valid? @@ -61,7 +67,7 @@ module Gitlab end def schema_validator - @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data) + @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data, report.version) end def report_data @@ -99,6 +105,7 @@ module Gitlab flags = create_flags(data['flags']) links = create_links(data['links']) location = create_location(data['location'] || {}) + evidence = create_evidence(data['evidence']) signatures = create_signatures(tracking_data(data)) if @vulnerability_finding_signatures_enabled && !signatures.empty? @@ -117,6 +124,7 @@ module Gitlab name: finding_name(data, identifiers, location), compare_key: data['cve'] || '', location: location, + evidence: evidence, severity: parse_severity_level(data['severity']), confidence: parse_confidence_level(data['confidence']), scanner: create_scanner(data['scanner']), @@ -253,6 +261,12 @@ module Gitlab raise NotImplementedError end + def create_evidence(evidence_data) + return unless evidence_data.is_a?(Hash) + + ::Gitlab::Ci::Reports::Security::Evidence.new(data: evidence_data) + end + def finding_name(data, identifiers, location) return data['message'] if data['message'].present? return data['name'] if data['name'].present? diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 651ed23eb25..0ab1a128052 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -6,20 +6,56 @@ module Gitlab module Security module Validators class SchemaValidator + # https://docs.gitlab.com/ee/update/deprecations.html#147 + SUPPORTED_VERSIONS = { + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0] + }.freeze + + # https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/tags + PREVIOUS_RELEASES = %w[10.0.0 12.0.0 12.1.0 13.0.0 + 13.1.0 2.3.0-rc1 2.3.0-rc1 2.3.1-rc1 2.3.2-rc1 2.3.3-rc1 + 2.4.0-rc1 3.0.0 3.0.0-rc1 3.1.0-rc1 4.0.0-rc1 5.0.0-rc1 + 5.0.1-rc1 6.0.0-rc1 6.0.1-rc1 6.1.0-rc1 7.0.0-rc1 7.0.1-rc1 + 8.0.0-rc1 8.0.1-rc1 8.1.0-rc1 9.0.0-rc1].freeze + + # These come from https://app.periscopedata.com/app/gitlab/895813/Secure-Scan-metrics?widget=12248944&udv=1385516 + KNOWN_VERSIONS_TO_DEPRECATE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze + + VERSIONS_TO_DEPRECATE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_DEPRECATE).freeze + + DEPRECATED_VERSIONS = { + cluster_image_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + container_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + coverage_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, + dast: VERSIONS_TO_DEPRECATE_IN_15_0, + api_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, + dependency_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + sast: VERSIONS_TO_DEPRECATE_IN_15_0, + secret_detection: VERSIONS_TO_DEPRECATE_IN_15_0 + }.freeze + class Schema def root_path File.join(__dir__, 'schemas') end - def initialize(report_type) + def initialize(report_type, report_version) @report_type = report_type.to_sym + @report_version = report_version.to_s end delegate :validate, to: :schemer private - attr_reader :report_type + attr_reader :report_type, :report_version def schemer JSONSchemer.schema(pathname) @@ -30,7 +66,19 @@ module Gitlab end def schema_path - File.join(root_path, file_name) + # We can't exactly error out here pre-15.0. + # If the report itself doesn't specify the schema version, + # it will be considered invalid post-15.0 but for now we will + # validate against earliest supported version. + # https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_801479803 + # describes the indended behavior in detail + # TODO: After 15.0 - pass report_type and report_data here and + # error out if no version. + report_declared_version = File.join(root_path, report_version, file_name) + return report_declared_version if File.file?(report_declared_version) + + earliest_supported_version = SUPPORTED_VERSIONS[report_type].min + File.join(root_path, earliest_supported_version, file_name) end def file_name @@ -38,9 +86,10 @@ module Gitlab end end - def initialize(report_type, report_data) + def initialize(report_type, report_data, report_version = nil) @report_type = report_type @report_data = report_data + @report_version = report_version end def valid? @@ -53,10 +102,10 @@ module Gitlab private - attr_reader :report_type, :report_data + attr_reader :report_type, :report_data, :report_version def schema - Schema.new(report_type) + Schema.new(report_type, report_version) end end end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 54b54bd0514..71dfc1a676c 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -14,7 +14,7 @@ module Gitlab with_bulk_insert_tags do pipeline.transaction do pipeline.save! - CommitStatus.bulk_insert_tags!(statuses) if bulk_insert_tags? + CommitStatus.bulk_insert_tags!(statuses) end end end @@ -29,15 +29,9 @@ module Gitlab private - def bulk_insert_tags? - strong_memoize(:bulk_insert_tags) do - ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) - end - end - def with_bulk_insert_tags previous = Thread.current['ci_bulk_insert_tags'] - Thread.current['ci_bulk_insert_tags'] = bulk_insert_tags? + Thread.current['ci_bulk_insert_tags'] = true yield ensure Thread.current['ci_bulk_insert_tags'] = previous diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 10c0fe295f8..ee6c3898592 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -94,6 +94,7 @@ module Gitlab private attr_reader :project, :destination, :started_at, :log_conditions + delegate :current_monotonic_time, to: :class def age diff --git a/lib/gitlab/ci/reports/security/evidence.rb b/lib/gitlab/ci/reports/security/evidence.rb new file mode 100644 index 00000000000..a19f52f7195 --- /dev/null +++ b/lib/gitlab/ci/reports/security/evidence.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Evidence + attr_reader :data + + def initialize(data:) + @data = data + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 69fb8474cde..911a7f5d358 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -13,6 +13,7 @@ module Gitlab attr_reader :flags attr_reader :links attr_reader :location + attr_reader :evidence attr_reader :metadata_version attr_reader :name attr_reader :old_location @@ -33,13 +34,14 @@ module Gitlab alias_method :cve, :compare_key - def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers @flags = flags @links = links @location = location + @evidence = evidence @metadata_version = metadata_version @name = name @original_data = original_data @@ -65,6 +67,7 @@ module Gitlab flags links location + evidence metadata_version name project_fingerprint diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index fbf8c81ac36..8c528056d0c 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -6,7 +6,7 @@ module Gitlab module Security class Report attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers - attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status + attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings delegate :project_id, to: :pipeline @@ -19,6 +19,7 @@ module Gitlab @identifiers = {} @scanned_resources = [] @errors = [] + @warnings = [] end def commit_sha @@ -29,6 +30,10 @@ module Gitlab errors << { type: type, message: message } end + def add_warning(type, message) + warnings << { type: type, message: message } + end + def errored? errors.present? end diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index 287a03cefe2..7fa744d047c 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -106,7 +106,7 @@ module Gitlab private def max_tests(*used) - [DEFAULT_MAX_TESTS - used.map(&:count).sum, DEFAULT_MIN_TESTS].max + [DEFAULT_MAX_TESTS - used.sum(&:count), DEFAULT_MIN_TESTS].max end end end diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb index 59869a947a9..ac3f5838d26 100644 --- a/lib/gitlab/ci/status/build/waiting_for_approval.rb +++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb @@ -9,11 +9,35 @@ module Gitlab { image: 'illustrations/manual_action.svg', size: 'svg-394', - title: 'Waiting for approval', - content: "This job deploys to the protected environment \"#{subject.deployment&.environment&.name}\" which requires approvals. Use the Deployments API to approve or reject the deployment." + title: _('Waiting for approval'), + content: _("This job deploys to the protected environment \"%{environment}\" which requires approvals.") % { environment: subject.deployment&.environment&.name } } end + def has_action? + true + end + + def action_icon + nil + end + + def action_title + nil + end + + def action_button_title + _('Go to environments page to approve or reject') + end + + def action_path + project_environments_path(subject.project) + end + + def action_method + :get + end + def self.matches?(build, user) build.waiting_for_deployment_approval? end diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 64e3b695e27..bbe1b0a4b82 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -4,8 +4,14 @@ # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml # Read more about how to use this script on this blog post https://about.gitlab.com/2019/01/28/android-publishing-with-gitlab-and-fastlane/ -# You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. # If you are looking for a simpler template that does not publish, see the Android template. +# You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. + +# The following environment variables also need to be defined via the CI/CD settings: +# +# - $signing_jks_file_hex: A hex-encoded Java keystore file containing your signing keys. +# To encode this file, use `xxd -p <your-keystore-file>.jks` and save the output as `$signing_jks_file_hex` +# - $google_play_service_account_api_key_json: Your Google Play service account credentials - https://docs.fastlane.tools/getting-started/android/setup/#collect-your-google-credentials stages: - environment @@ -41,20 +47,21 @@ ensureContainer: before_script: - "mkdir -p ~/.docker && echo '{\"experimental\": \"enabled\"}' > ~/.docker/config.json" - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # Skip update container `script` if the container already exists - # via https://gitlab.com/gitlab-org/gitlab-foss/issues/26866#note_97609397 -> https://stackoverflow.com/a/52077071/796832 - - docker manifest inspect $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG > /dev/null && exit || true - + - | + if docker manifest inspect $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG > /dev/null; then + echo 'Skipping job since there is already an image with this tag' + exit 0 + fi .build_job: image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG stage: build before_script: - # We store this binary file in a variable as hex with this command: `xxd -p android-app.jks` + # We store this binary file in a project variable as hex with this command: `xxd -p android-app.jks` # Then we convert the hex back to a binary file - echo "$signing_jks_file_hex" | xxd -r -p - > android-signing-keystore.jks - - "export VERSION_CODE=$CI_PIPELINE_IID && echo $VERSION_CODE" - - "export VERSION_SHA=`echo ${CI_COMMIT_SHA:0:8}` && echo $VERSION_SHA" + - export VERSION_CODE="$CI_PIPELINE_IID" && echo "$VERSION_CODE" + - export VERSION_SHA="${CI_COMMIT_SHA:0:8}" && echo "$VERSION_SHA" after_script: - rm -f android-signing-keystore.jks || true artifacts: diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index a50e722f18a..6354db38f58 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -18,7 +18,7 @@ cache: - .pub-cache/global_packages before_script: - - export PATH="$PATH":"~/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" - pub get --no-precompile test: diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index d176ce19299..a5c261e367a 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: image: "cirrusci/flutter:1.22.5" before_script: - pub global activate dart_code_metrics - - export PATH="$PATH":"$HOME/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" script: - metrics lib -r codeclimate > gl-code-quality-report.json artifacts: @@ -20,7 +20,7 @@ test: image: "cirrusci/flutter:1.22.5" before_script: - pub global activate junitreport - - export PATH="$PATH":"$HOME/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" script: - flutter test --machine --coverage | tojunit -o report.xml - lcov --summary coverage/lcov.info diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index b5dd0005013..19e4ffdbe1e 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -16,9 +16,9 @@ variables: # repository in /go/src/gitlab.com/namespace/project # Thus, making a symbolic link corrects this. before_script: - - mkdir -p $GOPATH/src/$(dirname $REPO_NAME) - - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME - - cd $GOPATH/src/$REPO_NAME + - mkdir -p "$GOPATH/src/$(dirname $REPO_NAME)" + - ln -svf "$CI_PROJECT_DIR" "$GOPATH/src/$REPO_NAME" + - cd "$GOPATH/src/$REPO_NAME" stages: - test diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 76f0c9f8427..08dc10d34b7 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -17,7 +17,8 @@ variables: GRADLE_OPTS: "-Dorg.gradle.daemon=false" before_script: - - export GRADLE_USER_HOME=`pwd`/.gradle + - GRADLE_USER_HOME="$(pwd)/.gradle" + - export GRADLE_USER_HOME build: stage: build diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 3c514d7b0c6..7e59354c4a1 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -23,8 +23,8 @@ variables: before_script: - apt-get update -qq && apt-get install -y -qq unzip - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > /root/.sdkman/etc/config - - source /root/.sdkman/bin/sdkman-init.sh + - echo sdkman_auto_answer=true > ~/.sdkman/etc/config + - source ~/.sdkman/bin/sdkman-init.sh - sdk install gradle $GRADLE_VERSION < /dev/null - sdk use gradle $GRADLE_VERSION # As it's not a good idea to version gradle.properties feel free to add your @@ -36,7 +36,7 @@ before_script: # Be aware that if you are using Angular profile, # Bower cannot be run as root if you don't allow it before. # Feel free to remove next line if you are not using Bower - - echo {\"allow_root\":true} > /root/.bowerrc + - echo '{"allow_root":true}' > ~/.bowerrc # This build job does the full grails pipeline # (compile, test, integrationTest, war, assemble). diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 99fd9870b1d..d1018f1e769 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -2,7 +2,7 @@ browser_performance: stage: performance - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,19 +10,21 @@ browser_performance: SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi - - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - CI_ENVIRONMENT_URL="$(cat environment_url.txt)" + - export CI_ENVIRONMENT_URL - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 99fd9870b1d..bb7e020b159 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -2,7 +2,7 @@ browser_performance: stage: performance - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,11 +10,12 @@ browser_performance: SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi @@ -22,7 +23,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index d5ca93a0a3b..f3d2e293c86 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.5.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' build: stage: build @@ -7,7 +7,7 @@ build: variables: DOCKER_TLS_CERTDIR: '' services: - - name: 'docker:20.10.6-dind' + - name: 'docker:20.10.12-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index d5ca93a0a3b..f3d2e293c86 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.5.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' build: stage: build @@ -7,7 +7,7 @@ build: variables: DOCKER_TLS_CERTDIR: '' services: - - name: 'docker:20.10.6-dind' + - name: 'docker:20.10.12-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 6942631a97f..6a95d042842 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -1,9 +1,10 @@ code_quality: stage: test - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" @@ -13,7 +14,7 @@ code_quality: - export SOURCE_CODE=$PWD - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 28ac627f103..cc204207f84 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.17.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 65c9232f3b9..1a99db67441 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 075e13e87f0..bc4f2099d94 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.18.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index e9c5d970c21..ce584091eab 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.18.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml index fc51f5adb3c..89a44eddefd 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. LICENSE_MANAGEMENT_VERSION: 3 diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 8e34388893a..eea1c397108 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -1,6 +1,6 @@ load_performance: stage: performance - image: docker:19.03.11 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,11 +10,12 @@ load_performance: K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: - - docker:19.03.11-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index fa7f6ffa2b7..5ddfb2a54be 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" iac-sast: diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 25d20563010..8cc9ea0200c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 4e4f96bc7c7..0ef6f63bb94 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -5,7 +5,7 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECRETS_ANALYZER_VERSION: "3" SECRET_DETECTION_EXCLUDED_PATHS: "" @@ -30,24 +30,43 @@ secret_detection: - if: $CI_COMMIT_BRANCH script: - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi + # Historic scan - | - # we don't need the whole history when excluding in the next `git fetch` line, - # so git depth=1 - git fetch origin --depth=1 $CI_DEFAULT_BRANCH - # shallow clone $CI_COMMIT_REF_NAME to get commits associated with MR or push - git fetch --shallow-exclude=${CI_DEFAULT_BRANCH} origin $CI_COMMIT_REF_NAME - # determine what commits we need to scan using "git log A..B" - git log --no-merges --pretty=format:"%H" refs/remotes/origin/${CI_DEFAULT_BRANCH}..refs/remotes/origin/${CI_COMMIT_REF_NAME} >${CI_COMMIT_SHA}_commit_list.txt - - # we need to extend the git fetch depth to the number of commits + 2 for the following reasons: - # because busybox wc only counts \n and there is no trailing \n (+1) - # include the parent commit of the base commit in this MR/Push event. This is needed because - # `git diff -p` needs something to compare changes in that commit against (+1) - git fetch --depth=$(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 2)) origin $CI_COMMIT_REF_NAME + if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ] + then + echo "historic scan" + git fetch --unshallow origin $CI_COMMIT_REF_NAME + /analyzer run + exit + fi + # Default branch scan + - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit; fi + # Push event + - | + if [ "$CI_COMMIT_BEFORE_SHA" == "0000000000000000000000000000000000000000" ]; + then + # first commit on a new branch + echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt + git fetch --depth=2 origin $CI_COMMIT_REF_NAME + else + # determine commit range so that we can fetch the appropriate depth + # check the exit code to determine if we need to limit the commit_list.txt to CI_COMMIT_SHA. + if ! git log --pretty=format:"%H" ${CI_COMMIT_BEFORE_SHA}..${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt; + then + echo "unable to determine commit range, limiting to ${CI_COMMIT_SHA}" + echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt + else + # append newline to to list since `git log` does not end with a + # newline, this is to keep the log messages consistent + echo >> ${CI_COMMIT_SHA}_commit_list.txt + fi - # +1 because busybox wc only counts \n and there is no trailing \n - echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) commits" + # we need to extend the git fetch depth to the number of commits + 1 for the following reasons: + # to include the parent commit of the base commit in this MR/Push event. This is needed because + # `git diff -p` needs something to compare changes in that commit against + git fetch --depth=$(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) origin $CI_COMMIT_REF_NAME + fi + echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt))) commits for a push event" export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt - /analyzer run - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index d32444833fb..85f90984045 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -8,7 +8,7 @@ pages: stage: deploy script: - mkdir .public - - cp -r * .public + - cp -r ./* .public - rm -rf public - mv .public public artifacts: diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 4917abf6ae9..6ed5e05ed4c 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -47,7 +47,8 @@ run: pages: script: - pip install sphinx sphinx-rtd-theme - - cd doc ; make html + - cd doc + - make html - mv build/html/ ../public/ artifacts: paths: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 1660a9250e3..33c0928db6f 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -52,7 +52,7 @@ rails: # This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk # are supported too: https://github.com/travis-ci/dpl deploy: - type: deploy + stage: deploy environment: production script: - gem install dpl diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 009061ce844..aff8b6cb7fa 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: FUZZAPI_VERSION: "1" - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} apifuzzer_fuzz: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 01041f4f056..bd8ba71effe 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: FUZZAPI_VERSION: "1" - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" FUZZAPI_IMAGE: api-fuzzing apifuzzer_fuzz: diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index a2933085d4e..d82f9f06f8d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # DAST_API_VERSION: "1" DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index 57f1993921d..0e0afa489a3 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # DAST_API_VERSION: "1" DAST_API_IMAGE: api-fuzzing diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 7ffec7d2e6b..3f9c87b7abf 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -5,7 +5,7 @@ stages: - dast variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" DAST_API_VERSION: "1" DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 3e7ab9b5c3b..998425aa141 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -13,7 +13,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 0ecbe5e14b8..e8e7fe62e70 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 3d07674c377..c755211ec11 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 82c7bfd0620..a6fd070ec34 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -14,8 +14,11 @@ # Docs: https://docs.gitlab.com/ee/topics/airgap/ variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, license-finder, dast, dast-runner-validation, api-fuzzing @@ -40,7 +43,7 @@ variables: script: - docker info - env - - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"registry.gitlab.com/gitlab-org/security-products/analyzers/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi + - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"${SECURE_ANALYZERS_PREFIX}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi - docker pull --quiet ${SECURE_BINARIES_IMAGE} - mkdir -p output/$(dirname ${CI_JOB_NAME}) - | diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index e696c75253e..84a962e1541 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -24,19 +24,19 @@ cache: .init: &init stage: init script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform init .validate: &validate stage: validate script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform validate .build: &build stage: build script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json artifacts: @@ -48,7 +48,7 @@ cache: .deploy: &deploy stage: deploy script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform apply when: manual only: @@ -58,6 +58,6 @@ cache: .destroy: &destroy stage: cleanup script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform destroy when: manual diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 8f4a836441d..5ea2bc07ffa 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -14,7 +14,8 @@ stages: a11y: stage: accessibility image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1 - script: /gitlab-accessibility.sh $a11y_urls + script: + - /gitlab-accessibility.sh "$a11y_urls" allow_failure: true artifacts: when: always diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index e0df9799917..2349c37c130 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -25,7 +25,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index ad24ebae8d4..73ab5fcbe44 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -25,7 +25,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb index 7f43d91e6d7..eaa9be9dd15 100644 --- a/lib/gitlab/ci/trace/remote_checksum.rb +++ b/lib/gitlab/ci/trace/remote_checksum.rb @@ -23,6 +23,7 @@ module Gitlab private attr_reader :trace_artifact + delegate :aws?, :google?, to: :object_store_config, prefix: :provider def fetch_md5_checksum diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 9ef6e7f5fa9..bfcf67693e7 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -10,6 +10,7 @@ module Gitlab @pipeline = pipeline @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) + @group_variables_builder = Builder::Group.new(project.group) end def scoped_variables(job, environment:, dependencies:) @@ -18,8 +19,7 @@ module Gitlab variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner - variables.concat(kubernetes_variables(job)) - variables.concat(deployment_variables(environment: environment, job: job)) + variables.concat(kubernetes_variables(environment: environment, job: job)) variables.concat(job.yaml_variables) variables.concat(user_variables(job.user)) variables.concat(job.dependency_variables) if dependencies @@ -32,11 +32,15 @@ module Gitlab end end - def kubernetes_variables(job) + def kubernetes_variables(environment:, job:) ::Gitlab::Ci::Variables::Collection.new.tap do |collection| - # Should get merged with the cluster kubeconfig in deployment_variables, see - # https://gitlab.com/gitlab-org/gitlab/-/issues/335089 + # NOTE: deployment_variables will be removed as part of cleanup for + # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 + # Until then, we need to make both the old and the new KUBECONFIG contexts available + collection.concat(deployment_variables(environment: environment, job: job)) template = ::Ci::GenerateKubeconfigService.new(job).execute + kubeconfig_yaml = collection['KUBECONFIG']&.value + template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? if template.valid? collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true) @@ -72,9 +76,13 @@ module Gitlab end def secret_group_variables(environment:, ref:) - return [] unless project.group + if memoize_secret_variables? + memoized_secret_group_variables(environment: environment) + else + return [] unless project.group - project.group.ci_variables_for(ref, project, environment: environment) + project.group.ci_variables_for(ref, project, environment: environment) + end end def secret_project_variables(environment:, ref:) @@ -90,6 +98,8 @@ module Gitlab attr_reader :pipeline attr_reader :instance_variables_builder attr_reader :project_variables_builder + attr_reader :group_variables_builder + delegate :project, to: :pipeline def predefined_variables(job) @@ -119,6 +129,15 @@ module Gitlab end end + def memoized_secret_group_variables(environment:) + strong_memoize_with(:secret_group_variables, environment) do + group_variables_builder + .secret_variables( + environment: environment, + protected_ref: protected_ref?) + end + end + def ci_node_total_value(job) parallel = job.options&.dig(:parallel) parallel = parallel.dig(:total) if parallel.is_a?(Hash) diff --git a/lib/gitlab/ci/variables/builder/group.rb b/lib/gitlab/ci/variables/builder/group.rb new file mode 100644 index 00000000000..3f3e04038df --- /dev/null +++ b/lib/gitlab/ci/variables/builder/group.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + class Group + include Gitlab::Utils::StrongMemoize + + def initialize(group) + @group = group + end + + def secret_variables(environment:, protected_ref: false) + return [] unless group + + variables = base_scope + variables = variables.unprotected unless protected_ref + variables = variables.for_environment(environment) + variables = variables.group_by(&:group_id) + variables = list_of_ids.reverse.flat_map { |group| variables[group.id] }.compact + Gitlab::Ci::Variables::Collection.new(variables) + end + + private + + attr_reader :group + + def base_scope + strong_memoize(:base_scope) do + ::Ci::GroupVariable.for_groups(list_of_ids) + end + end + + def list_of_ids + strong_memoize(:list_of_ids) do + if group.root_ancestor.use_traversal_ids? + [group] + group.ancestors(hierarchy_order: :asc) + else + [group] + group.ancestors + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 553508c8638..15ebd506055 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -45,7 +45,7 @@ module Gitlab validate_job!(name, job) end - YamlProcessor::Dag.check_circular_dependencies!(@jobs) + check_circular_dependencies end def validate_job!(name, job) @@ -146,6 +146,17 @@ module Gitlab end end + def check_circular_dependencies + jobs = @jobs.values.to_h do |job| + name = job[:name].to_s + needs = job.dig(:needs, :job).to_a + + [name, needs.map { |need| need[:name].to_s }] + end + + Dag.check_circular_dependencies!(jobs) + end + def error!(message) raise ValidationError, message end diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb index 8ab9573dd20..4a122c73e80 100644 --- a/lib/gitlab/ci/yaml_processor/dag.rb +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -7,28 +7,22 @@ module Gitlab class Dag include TSort - MissingNodeError = Class.new(StandardError) - def initialize(nodes) @nodes = nodes end - def self.check_circular_dependencies!(jobs) - nodes = jobs.values.to_h do |job| - name = job[:name].to_s - needs = job.dig(:needs, :job).to_a - - [name, needs.map { |need| need[:name].to_s }] - end + def self.order(jobs) + new(jobs).tsort + end - new(nodes).tsort + def self.check_circular_dependencies!(jobs) + new(jobs).tsort rescue TSort::Cyclic raise ValidationError, 'The pipeline has circular dependencies' - rescue MissingNodeError end def tsort_each_child(node, &block) - raise MissingNodeError, "node #{node} is missing" unless @nodes[node] + return unless @nodes[node] @nodes[node].each(&block) end diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb new file mode 100644 index 00000000000..e0caabb0ec6 --- /dev/null +++ b/lib/gitlab/color.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +module Gitlab + class Color + PATTERN = /\A\#(?:[0-9A-Fa-f]{3}){1,2}\Z/.freeze + + def initialize(value) + @value = value&.strip&.freeze + end + + module Constants + DARK = Color.new('#333333') + LIGHT = Color.new('#FFFFFF') + + COLOR_NAME_TO_HEX = { + black: '#000000', + silver: '#C0C0C0', + gray: '#808080', + white: '#FFFFFF', + maroon: '#800000', + red: '#FF0000', + purple: '#800080', + fuchsia: '#FF00FF', + green: '#008000', + lime: '#00FF00', + olive: '#808000', + yellow: '#FFFF00', + navy: '#000080', + blue: '#0000FF', + teal: '#008080', + aqua: '#00FFFF', + orange: '#FFA500', + aliceblue: '#F0F8FF', + antiquewhite: '#FAEBD7', + aquamarine: '#7FFFD4', + azure: '#F0FFFF', + beige: '#F5F5DC', + bisque: '#FFE4C4', + blanchedalmond: '#FFEBCD', + blueviolet: '#8A2BE2', + brown: '#A52A2A', + burlywood: '#DEB887', + cadetblue: '#5F9EA0', + chartreuse: '#7FFF00', + chocolate: '#D2691E', + coral: '#FF7F50', + cornflowerblue: '#6495ED', + cornsilk: '#FFF8DC', + crimson: '#DC143C', + darkblue: '#00008B', + darkcyan: '#008B8B', + darkgoldenrod: '#B8860B', + darkgray: '#A9A9A9', + darkgreen: '#006400', + darkgrey: '#A9A9A9', + darkkhaki: '#BDB76B', + darkmagenta: '#8B008B', + darkolivegreen: '#556B2F', + darkorange: '#FF8C00', + darkorchid: '#9932CC', + darkred: '#8B0000', + darksalmon: '#E9967A', + darkseagreen: '#8FBC8F', + darkslateblue: '#483D8B', + darkslategray: '#2F4F4F', + darkslategrey: '#2F4F4F', + darkturquoise: '#00CED1', + darkviolet: '#9400D3', + deeppink: '#FF1493', + deepskyblue: '#00BFFF', + dimgray: '#696969', + dimgrey: '#696969', + dodgerblue: '#1E90FF', + firebrick: '#B22222', + floralwhite: '#FFFAF0', + forestgreen: '#228B22', + gainsboro: '#DCDCDC', + ghostwhite: '#F8F8FF', + gold: '#FFD700', + goldenrod: '#DAA520', + greenyellow: '#ADFF2F', + grey: '#808080', + honeydew: '#F0FFF0', + hotpink: '#FF69B4', + indianred: '#CD5C5C', + indigo: '#4B0082', + ivory: '#FFFFF0', + khaki: '#F0E68C', + lavender: '#E6E6FA', + lavenderblush: '#FFF0F5', + lawngreen: '#7CFC00', + lemonchiffon: '#FFFACD', + lightblue: '#ADD8E6', + lightcoral: '#F08080', + lightcyan: '#E0FFFF', + lightgoldenrodyellow: '#FAFAD2', + lightgray: '#D3D3D3', + lightgreen: '#90EE90', + lightgrey: '#D3D3D3', + lightpink: '#FFB6C1', + lightsalmon: '#FFA07A', + lightseagreen: '#20B2AA', + lightskyblue: '#87CEFA', + lightslategray: '#778899', + lightslategrey: '#778899', + lightsteelblue: '#B0C4DE', + lightyellow: '#FFFFE0', + limegreen: '#32CD32', + linen: '#FAF0E6', + mediumaquamarine: '#66CDAA', + mediumblue: '#0000CD', + mediumorchid: '#BA55D3', + mediumpurple: '#9370DB', + mediumseagreen: '#3CB371', + mediumslateblue: '#7B68EE', + mediumspringgreen: '#00FA9A', + mediumturquoise: '#48D1CC', + mediumvioletred: '#C71585', + midnightblue: '#191970', + mintcream: '#F5FFFA', + mistyrose: '#FFE4E1', + moccasin: '#FFE4B5', + navajowhite: '#FFDEAD', + oldlace: '#FDF5E6', + olivedrab: '#6B8E23', + orangered: '#FF4500', + orchid: '#DA70D6', + palegoldenrod: '#EEE8AA', + palegreen: '#98FB98', + paleturquoise: '#AFEEEE', + palevioletred: '#DB7093', + papayawhip: '#FFEFD5', + peachpuff: '#FFDAB9', + peru: '#CD853F', + pink: '#FFC0CB', + plum: '#DDA0DD', + powderblue: '#B0E0E6', + rosybrown: '#BC8F8F', + royalblue: '#4169E1', + saddlebrown: '#8B4513', + salmon: '#FA8072', + sandybrown: '#F4A460', + seagreen: '#2E8B57', + seashell: '#FFF5EE', + sienna: '#A0522D', + skyblue: '#87CEEB', + slateblue: '#6A5ACD', + slategray: '#708090', + slategrey: '#708090', + snow: '#FFFAFA', + springgreen: '#00FF7F', + steelblue: '#4682B4', + tan: '#D2B48C', + thistle: '#D8BFD8', + tomato: '#FF6347', + turquoise: '#40E0D0', + violet: '#EE82EE', + wheat: '#F5DEB3', + whitesmoke: '#F5F5F5', + yellowgreen: '#9ACD32', + rebeccapurple: '#663399' + }.stringify_keys.transform_values { Color.new(_1) }.freeze + end + + def self.of(color) + raise ArgumentError, 'No color spec' unless color + return color if color.is_a?(self) + + color = color.to_s.strip + Constants::COLOR_NAME_TO_HEX[color.downcase] || new(color) + end + + def to_s + @value.to_s + end + + def as_json(_options = nil) + to_s + end + + def eql(other) + return false unless other.is_a?(self.class) + + to_s == other.to_s + end + alias_method :==, :eql + + def valid? + PATTERN.match?(@value) + end + + def light? + valid? && rgb.sum > 500 + end + + def luminosity + return :light if light? + + :dark + end + + def contrast + return Constants::DARK if light? + + Constants::LIGHT + end + + private + + def rgb + return [] unless valid? + + @rgb ||= begin + if @value.length == 4 + @value[1, 4].scan(/./).map { |v| (v * 2).hex } + else + @value[1, 7].scan(/.{2}/).map(&:hex) + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index b2bc56f46ee..cc24ae837f3 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -39,6 +39,17 @@ module Gitlab end end + class MutuallyExclusiveKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + mutually_exclusive_keys = value.try(:keys).to_a & options[:in] + + if mutually_exclusive_keys.length > 1 + record.errors.add(attribute, "please use only one the following keys: " + + mutually_exclusive_keys.join(', ')) + end + end + end + class AllowedValuesValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless options[:in].include?(value.to_s) @@ -217,12 +228,6 @@ module Gitlab end end - protected - - def fallback - false - end - private def matches_syntax?(value) @@ -231,7 +236,7 @@ module Gitlab def validate_regexp(value) matches_syntax?(value) && - Gitlab::UntrustedRegexp::RubySyntax.valid?(value, fallback: fallback) + Gitlab::UntrustedRegexp::RubySyntax.valid?(value) end end @@ -260,27 +265,6 @@ module Gitlab end end - class ArrayOfStringsOrRegexpsWithFallbackValidator < ArrayOfStringsOrRegexpsValidator - protected - - # TODO - # - # Remove ArrayOfStringsOrRegexpsWithFallbackValidator class too when - # you are removing the `:allow_unsafe_ruby_regexp` feature flag. - # - def validation_message - if ::Feature.enabled?(:allow_unsafe_ruby_regexp, default_enabled: :yaml) - 'should be an array of strings or regular expressions' - else - super - end - end - - def fallback - true - end - end - class ArrayOfStringsOrStringValidator < RegexpValidator def validate_each(record, attribute, value) unless validate_array_of_strings_or_string(value) diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 78ba0916808..0d4b913b7a0 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -15,7 +15,7 @@ module Gitlab directives = { 'default_src' => "'self'", 'base_uri' => "'self'", - 'connect_src' => "'self'", + 'connect_src' => ContentSecurityPolicy::Directives.connect_src, 'font_src' => "'self'", 'form_action' => "'self' https: http:", 'frame_ancestors' => "'self'", diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb index 3b958f8c92e..4ad420f9e2f 100644 --- a/lib/gitlab/content_security_policy/directives.rb +++ b/lib/gitlab/content_security_policy/directives.rb @@ -7,6 +7,10 @@ module Gitlab module ContentSecurityPolicy module Directives + def self.connect_src + "'self'" + end + def self.frame_src "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com https://www.googletagmanager.com/ns.html" end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 0d6767ad564..8ef4977177a 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -70,6 +70,8 @@ module Gitlab else ::ApplicationSetting.create_from_defaults end + rescue ::ApplicationSetting::Recursion + in_memory_application_settings end def fake_application_settings(attributes = {}) diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index f867dbd4d68..5605d48c543 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -4,27 +4,13 @@ module Gitlab module CycleAnalytics module Summary class Base + include Gitlab::CycleAnalytics::Summary::Defaults + def initialize(project:, options:) @project = project @options = options end - def identifier - self.class.name.demodulize.underscore.to_sym - end - - def title - raise NotImplementedError, "Expected #{self.name} to implement title" - end - - def value - raise NotImplementedError, "Expected #{self.name} to implement value" - end - - def links - [] - end - private attr_reader :project, :options diff --git a/lib/gitlab/cycle_analytics/summary/defaults.rb b/lib/gitlab/cycle_analytics/summary/defaults.rb new file mode 100644 index 00000000000..468494a8ab8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/defaults.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Defaults + def identifier + self.class.name.demodulize.underscore.to_sym + end + + # :nocov: the class including this concern is expected to test this method. + def title + raise NotImplementedError, "Expected #{self.name} to implement title" + end + # :nocov: + + # :nocov: the class including this concern is expected to test this method. + def value + raise NotImplementedError, "Expected #{self.name} to implement value" + end + # :nocov: + + def links + [] + end + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 9b32d285ec0..1b16873f737 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -195,6 +195,16 @@ module Gitlab MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup end + def self.all_uncached(&block) + # Calls to #uncached only disable caching for the current connection. Since the load balancer + # can potentially upgrade from read to read-write mode (using a different connection), we specify + # up-front that we'll explicitly use the primary for the duration of the operation. + Gitlab::Database::LoadBalancing::Session.current.use_primary do + base_models = database_base_models.values + base_models.reduce(block) { |blk, model| -> { model.uncached(&blk) } }.call + end + end + def self.allow_cross_joins_across_databases(url:) # this method is implemented in: # spec/support/database/prevent_cross_joins.rb @@ -221,12 +231,26 @@ module Gitlab ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) end + # This returns all matching schemas that a given connection can use + # Since the `ActiveRecord::Base` might change the connection (from main to ci) + # This does not look at literal connection names, but rather compares + # models that are holders for a given db_config_name + def self.gitlab_schemas_for_connection(connection) + connection_name = self.db_config_name(connection) + primary_model = self.database_base_models.fetch(connection_name) + + self.schemas_to_base_models + .select { |_, models| models.include?(primary_model) } + .keys + .map!(&:to_sym) + end + def self.db_config_for_connection(connection) return unless connection - # The LB connection proxy does not have a direct db_config - # that can be referenced - return if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) + if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) + return connection.load_balancer.configuration.primary_db_config + end # During application init we might receive `NullPool` return unless connection.respond_to?(:pool) && diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index 2f990aba2fb..e9846dd4e85 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -5,6 +5,8 @@ module Gitlab module AsyncIndexes module MigrationHelpers def unprepare_async_index(table_name, column_name, **options) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + return unless async_index_creation_available? index_name = options[:name] || index_name(table_name, column_name) @@ -15,6 +17,8 @@ module Gitlab end def unprepare_async_index_by_name(table_name, index_name, **options) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + return unless async_index_creation_available? PostgresAsyncIndex.find_by(name: index_name).try do |async_index| @@ -32,6 +36,8 @@ module Gitlab # If the requested index has already been created, it is not stored in the table for # asynchronous creation. def prepare_async_index(table_name, column_name, **options) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + return unless async_index_creation_available? index_name = options[:name] || index_name(table_name, column_name) @@ -72,8 +78,7 @@ module Gitlab end def async_index_creation_available? - ApplicationRecord.connection.table_exists?(:postgres_async_indexes) && - Feature.enabled?(:database_async_index_creation, type: :ops) + connection.table_exists?(:postgres_async_indexes) end end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 185b6d9629f..f3160679d64 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -3,7 +3,9 @@ module Gitlab module Database module BackgroundMigration - class BatchedJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + SplitAndRetryError = Class.new(StandardError) + + class BatchedJob < SharedModel include EachBatch include FromUnion @@ -11,6 +13,8 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze + TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError, + ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout].freeze belongs_to :batched_migration, foreign_key: :batched_background_migration_id has_many :batched_job_transition_logs, foreign_key: :batched_background_migration_job_id @@ -51,6 +55,16 @@ module Gitlab job.metrics = {} end + after_transition any => :failed do |job, transition| + error_hash = transition.args.find { |arg| arg[:error].present? } + + exception = error_hash&.fetch(:error) + + job.split_and_retry! if job.can_split?(exception) + rescue SplitAndRetryError => error + Gitlab::AppLogger.error(message: error.message, batched_job_id: job.id) + end + after_transition do |job, transition| error_hash = transition.args.find { |arg| arg[:error].present? } @@ -79,20 +93,25 @@ module Gitlab duration.to_f / batched_migration.interval end + def can_split?(exception) + attempts >= MAX_ATTEMPTS && TIMEOUT_EXCEPTIONS.include?(exception&.class) && batch_size > sub_batch_size + end + def split_and_retry! with_lock do - raise 'Only failed jobs can be split' unless failed? + raise SplitAndRetryError, 'Only failed jobs can be split' unless failed? new_batch_size = batch_size / 2 - raise 'Job cannot be split further' if new_batch_size < 1 + raise SplitAndRetryError, 'Job cannot be split further' if new_batch_size < 1 - batching_strategy = batched_migration.batch_class.new + batching_strategy = batched_migration.batch_class.new(connection: self.class.connection) next_batch_bounds = batching_strategy.next_batch( batched_migration.table_name, batched_migration.column_name, batch_min_value: min_value, - batch_size: new_batch_size + batch_size: new_batch_size, + job_arguments: batched_migration.job_arguments ) midpoint = next_batch_bounds.last diff --git a/lib/gitlab/database/background_migration/batched_job_transition_log.rb b/lib/gitlab/database/background_migration/batched_job_transition_log.rb index 418bf1a101f..55a391005a2 100644 --- a/lib/gitlab/database/background_migration/batched_job_transition_log.rb +++ b/lib/gitlab/database/background_migration/batched_job_transition_log.rb @@ -3,7 +3,7 @@ module Gitlab module Database module BackgroundMigration - class BatchedJobTransitionLog < ApplicationRecord + class BatchedJobTransitionLog < SharedModel include PartitionedTable self.table_name = :batched_background_migration_job_transition_logs diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 1f8ca982ed5..65c15795de6 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -3,7 +3,7 @@ module Gitlab module Database module BackgroundMigration - class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + class BatchedMigration < SharedModel JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 9308bae20cf..06cd40f1e06 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -6,12 +6,13 @@ module Gitlab class BatchedMigrationRunner FailedToFinalize = Class.new(RuntimeError) - def self.finalize(job_class_name, table_name, column_name, job_arguments) - new.finalize(job_class_name, table_name, column_name, job_arguments) + def self.finalize(job_class_name, table_name, column_name, job_arguments, connection: ApplicationRecord.connection) + new(connection: connection).finalize(job_class_name, table_name, column_name, job_arguments) end - def initialize(migration_wrapper = BatchedMigrationWrapper.new) + def initialize(migration_wrapper = BatchedMigrationWrapper.new, connection: ApplicationRecord.connection) @migration_wrapper = migration_wrapper + @connection = connection end # Runs the next batched_job for a batched_background_migration. @@ -77,7 +78,7 @@ module Gitlab private - attr_reader :migration_wrapper + attr_reader :migration_wrapper, :connection def find_or_create_next_batched_job(active_migration) if next_batch_range = find_next_batch_range(active_migration) @@ -88,7 +89,7 @@ module Gitlab end def find_next_batch_range(active_migration) - batching_strategy = active_migration.batch_class.new + batching_strategy = active_migration.batch_class.new(connection: connection) batch_min_value = active_migration.next_min_value next_batch_bounds = batching_strategy.next_batch( diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index 0b8fe640bf8..345c7e44b05 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -12,6 +12,7 @@ module Gitlab # Note that for very large tables, this may even timeout. class ExactCountStrategy attr_reader :models + def initialize(models) @models = models end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 68a0c15480a..0f89c500688 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -14,6 +14,7 @@ module Gitlab # however is guaranteed to be "fast", because it only looks up statistics. class ReltuplesCountStrategy attr_reader :models + def initialize(models) @models = models end @@ -46,7 +47,7 @@ module Gitlab end def table_to_model_mapping - @table_to_model_mapping ||= models.each_with_object({}) { |model, h| h[model.table_name] = model } + @table_to_model_mapping ||= models.index_by(&:table_name) end def table_to_model(table_name) diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index c3eea0515d4..cccd4b48723 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -4,8 +4,11 @@ module Gitlab module Database module EachDatabase class << self - def each_database_connection - Gitlab::Database.database_base_models.each_pair do |connection_name, model| + def each_database_connection(only: nil) + selected_names = Array.wrap(only) + base_models = select_base_models(selected_names) + + base_models.each_pair do |connection_name, model| connection = model.connection with_shared_connection(connection, connection_name) do @@ -14,34 +17,52 @@ module Gitlab end end - def each_model_connection(models, &blk) + def each_model_connection(models, only_on: nil, &blk) + selected_databases = Array.wrap(only_on).map(&:to_sym) + models.each do |model| # If model is shared, iterate all available base connections # Example: `LooseForeignKeys::DeletedRecord` if model < ::Gitlab::Database::SharedModel - with_shared_model_connections(model, &blk) + with_shared_model_connections(model, selected_databases, &blk) else - with_model_connection(model, &blk) + with_model_connection(model, selected_databases, &blk) end end end private - def with_shared_model_connections(shared_model, &blk) + def select_base_models(names) + base_models = Gitlab::Database.database_base_models + + return base_models if names.empty? + + names.each_with_object(HashWithIndifferentAccess.new) do |name, hash| + raise ArgumentError, "#{name} is not a valid database name" unless base_models.key?(name) + + hash[name] = base_models[name] + end + end + + def with_shared_model_connections(shared_model, selected_databases, &blk) Gitlab::Database.database_base_models.each_pair do |connection_name, connection_model| if shared_model.limit_connection_names next unless shared_model.limit_connection_names.include?(connection_name.to_sym) end + next if selected_databases.present? && !selected_databases.include?(connection_name.to_sym) + with_shared_connection(connection_model.connection, connection_name) do yield shared_model, connection_name end end end - def with_model_connection(model, &blk) - connection_name = model.connection.pool.db_config.name + def with_model_connection(model, selected_databases, &blk) + connection_name = model.connection_db_config.name + + return if selected_databases.present? && !selected_databases.include?(connection_name.to_sym) with_shared_connection(model.connection, connection_name) do yield model, connection_name diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 7adf6ba6afb..d7ea614e2af 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -95,6 +95,10 @@ module Gitlab def self.tables_to_schema @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml')) end + + def self.schema_names + @schema_names ||= self.tables_to_schema.values.to_set + end end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 93cd75ce5a7..dcd78bfd84f 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -8,6 +8,7 @@ alert_management_alert_metric_images: :gitlab_main alert_management_alert_user_mentions: :gitlab_main alert_management_http_integrations: :gitlab_main allowed_email_domains: :gitlab_main +analytics_cycle_analytics_aggregations: :gitlab_main analytics_cycle_analytics_group_stages: :gitlab_main analytics_cycle_analytics_group_value_streams: :gitlab_main analytics_cycle_analytics_issue_stage_events: :gitlab_main @@ -273,6 +274,7 @@ issue_emails: :gitlab_main issue_email_participants: :gitlab_main issue_links: :gitlab_main issue_metrics: :gitlab_main +issue_search_data: :gitlab_main issues: :gitlab_main issues_prometheus_alert_events: :gitlab_main issues_self_managed_prometheus_alert_events: :gitlab_main @@ -379,7 +381,6 @@ pages_deployments: :gitlab_main pages_deployment_states: :gitlab_main pages_domain_acme_orders: :gitlab_main pages_domains: :gitlab_main -partitioned_foreign_keys: :gitlab_main path_locks: :gitlab_main personal_access_tokens: :gitlab_main plan_limits: :gitlab_main @@ -400,6 +401,7 @@ project_alerting_settings: :gitlab_main project_aliases: :gitlab_main project_authorizations: :gitlab_main project_auto_devops: :gitlab_main +project_build_artifacts_size_refreshes: :gitlab_main project_ci_cd_settings: :gitlab_main project_ci_feature_usages: :gitlab_main project_compliance_framework_settings: :gitlab_main @@ -441,6 +443,7 @@ push_event_payloads: :gitlab_main push_rules: :gitlab_main raw_usage_data: :gitlab_main redirect_routes: :gitlab_main +related_epic_links: :gitlab_main release_links: :gitlab_main releases: :gitlab_main remote_mirrors: :gitlab_main @@ -457,6 +460,7 @@ reviews: :gitlab_main routes: :gitlab_main saml_group_links: :gitlab_main saml_providers: :gitlab_main +saved_replies: :gitlab_main schema_migrations: :gitlab_shared scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index e16db5af8ce..6517923d23e 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -47,6 +47,8 @@ module Gitlab # Returns the role (primary/replica) of the database the connection is # connecting to. def self.db_role_for_connection(connection) + return ROLE_UNKNOWN if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) + db_config = Database.db_config_for_connection(connection) return ROLE_UNKNOWN unless db_config diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 63444ebe169..86b3afaa47b 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -94,6 +94,10 @@ module Gitlab end end + def primary_db_config + primary_model_or_model_if_enabled.connection_db_config + end + def replica_db_config @model.connection_db_config end diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index 126c8bb2aa6..6d667e8ecf0 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -90,7 +90,8 @@ module Gitlab def use_model_load_balancing? # Cache environment variable and return env variable first if defined - use_model_load_balancing_env = Gitlab::Utils.to_boolean(ENV["GITLAB_USE_MODEL_LOAD_BALANCING"]) + default_use_model_load_balancing_env = Gitlab.dev_or_test_env? || nil + use_model_load_balancing_env = Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_USE_MODEL_LOAD_BALANCING', default_use_model_load_balancing_env)) unless use_model_load_balancing_env.nil? return use_model_load_balancing_env diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb new file mode 100644 index 00000000000..b4e31565c60 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module RestrictGitlabSchema + extend ActiveSupport::Concern + + MigrationSkippedError = Class.new(StandardError) + + included do + class_attribute :allowed_gitlab_schemas + end + + class_methods do + def restrict_gitlab_migration(gitlab_schema:) + unless Gitlab::Database::GitlabSchema.schema_names.include?(gitlab_schema) + raise "Unknown 'gitlab_schema: #{gitlab_schema}' specified. It needs to be one of: " \ + "#{Gitlab::Database::GitlabSchema.schema_names.to_a}" + end + + self.allowed_gitlab_schemas = [gitlab_schema] + end + end + + def migrate(direction) + if unmatched_schemas.any? + # TODO: Today skipping migration would raise an exception. + # Ideally, skipped migration should be ignored (not loaded), or softly ignored. + # Read more in: https://gitlab.com/gitlab-org/gitlab/-/issues/355014 + raise MigrationSkippedError, "Current migration is skipped since it modifies "\ + "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'" + end + + Gitlab::Database::QueryAnalyzer.instance.within([validator_class]) do + validator_class.allowed_gitlab_schemas = self.allowed_gitlab_schemas + + super + end + end + + private + + def validator_class + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas + end + + def unmatched_schemas + (self.allowed_gitlab_schemas || []) - allowed_schemas_for_connection + end + + def allowed_schemas_for_connection + Gitlab::Database.gitlab_schemas_for_connection(connection) + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 4f1b490cc8f..7e5c002d072 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -42,7 +42,7 @@ module Gitlab # end def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) - raise "#{primary_column_name} is not an integer column" unless model_class.columns_hash[primary_column_name.to_s].type == :integer + raise "#{primary_column_name} is not an integer or string column" unless [:integer, :string].include?(model_class.columns_hash[primary_column_name.to_s].type) job_coordinator = coordinator_for_tracking_database diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 7f34768350b..9d28db6b886 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -17,7 +17,11 @@ module Gitlab def observe(version:, name:, connection:, &block) observation = Observation.new(version: version, name: name, success: false) - observers = observer_classes.map { |c| c.new(observation, @result_dir, connection) } + per_migration_result_dir = File.join(@result_dir, name) + + FileUtils.mkdir_p(per_migration_result_dir) + + observers = observer_classes.map { |c| c.new(observation, per_migration_result_dir, connection) } on_each_observer(observers) { |observer| observer.before } diff --git a/lib/gitlab/database/migrations/observers/query_details.rb b/lib/gitlab/database/migrations/observers/query_details.rb index 8f4406e79a5..1549f5bf626 100644 --- a/lib/gitlab/database/migrations/observers/query_details.rb +++ b/lib/gitlab/database/migrations/observers/query_details.rb @@ -6,7 +6,7 @@ module Gitlab module Observers class QueryDetails < MigrationObserver def before - file_path = File.join(output_dir, "#{observation.version}_#{observation.name}-query-details.json") + file_path = File.join(output_dir, "query-details.json") @file = File.open(file_path, 'wb') @writer = Oj::StreamWriter.new(@file, {}) @writer.push_array diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb index c42fd8bd23d..8ca57bb7df9 100644 --- a/lib/gitlab/database/migrations/observers/query_log.rb +++ b/lib/gitlab/database/migrations/observers/query_log.rb @@ -7,7 +7,7 @@ module Gitlab class QueryLog < MigrationObserver def before @logger_was = ActiveRecord::Base.logger - file_path = File.join(output_dir, "#{observation.version}_#{observation.name}.log") + file_path = File.join(output_dir, "migration.log") @logger = Logger.new(file_path) ActiveRecord::Base.logger = @logger end diff --git a/lib/gitlab/database/migrations/observers/transaction_duration.rb b/lib/gitlab/database/migrations/observers/transaction_duration.rb index a96b94334cf..182aeb10fda 100644 --- a/lib/gitlab/database/migrations/observers/transaction_duration.rb +++ b/lib/gitlab/database/migrations/observers/transaction_duration.rb @@ -6,7 +6,7 @@ module Gitlab module Observers class TransactionDuration < MigrationObserver def before - file_path = File.join(output_dir, "#{observation.version}_#{observation.name}-transaction-duration.json") + file_path = File.join(output_dir, "transaction-duration.json") @file = File.open(file_path, 'wb') @writer = Oj::StreamWriter.new(@file, {}) @writer.push_array diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index f0bac594119..02645a0d452 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -5,6 +5,8 @@ module Gitlab module Migrations class Runner BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze + METADATA_FILENAME = 'metadata.json' + SCHEMA_VERSION = 2 # Version of the output format produced by the runner class << self def up @@ -75,9 +77,11 @@ module Gitlab end ensure if instrumentation - File.open(File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME), 'wb+') do |io| - io << instrumentation.observations.to_json - end + stats_filename = File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME) + File.write(stats_filename, instrumentation.observations.to_json) + + metadata_filename = File.join(result_dir, METADATA_FILENAME) + File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) end # We clear the cache here to mirror the cache clearing that happens at the end of `db:migrate` tasks diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb new file mode 100644 index 00000000000..821d68c06c9 --- /dev/null +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class TestBackgroundRunner + # TODO - build a rake task to call this method, and support it in the gitlab-com-database-testing project. + # Until then, we will inject a migration with a very high timestamp during database testing + # that calls this class to run jobs + # See https://gitlab.com/gitlab-org/database-team/gitlab-com-database-testing/-/issues/41 for details + + def initialize + @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) + end + + def traditional_background_migrations + @job_coordinator.pending_jobs + end + + def run_jobs(for_duration:) + jobs_to_run = traditional_background_migrations.group_by { |j| class_name_for_job(j) } + return if jobs_to_run.empty? + + # without .to_f, we do integer division + # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) + duration_per_migration_type = for_duration / jobs_to_run.count.to_f + jobs_to_run.each do |_migration_name, jobs| + run_until = duration_per_migration_type.from_now + jobs.shuffle.each do |j| + break if run_until <= Time.current + + run_job(j) + end + end + end + + private + + def run_job(job) + Gitlab::BackgroundMigration.perform(job.args[0], job.args[1]) + end + + def class_name_for_job(job) + job.args[0] + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index c7d8bdf30bc..92825d41599 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -26,10 +26,10 @@ module Gitlab # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end - def sync_partitions(models_to_sync = registered_for_sync) + def sync_partitions(models_to_sync = registered_for_sync, only_on: nil) Gitlab::AppLogger.info(message: 'Syncing dynamic postgres partitions') - Gitlab::Database::EachDatabase.each_model_connection(models_to_sync) do |model| + Gitlab::Database::EachDatabase.each_model_connection(models_to_sync, only_on: only_on) do |model| PartitionManager.new(model).sync_partitions end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index ab414f91169..3ee9a193b45 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -46,6 +46,7 @@ module Gitlab private attr_reader :model + delegate :connection, to: :model def missing_partitions diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb index a7686e97553..21a175a660d 100644 --- a/lib/gitlab/database/partitioning/replace_table.rb +++ b/lib/gitlab/database/partitioning/replace_table.rb @@ -31,6 +31,7 @@ module Gitlab private attr_reader :connection + delegate :execute, :quote_table_name, :quote_column_name, to: :connection def default_sequence(table, column) diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 2736f9d18dc..0c78dda734c 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -30,13 +30,17 @@ module Gitlab end end - def within + def within(user_analyzers = nil) # Due to singleton nature of analyzers # only an outer invocation of the `.within` # is allowed to initialize them - return yield if already_within? + if already_within? + raise 'Query analyzers are already defined, cannot re-define them.' if user_analyzers - begin! + return yield + end + + begin!(user_analyzers || all_analyzers) begin yield @@ -61,21 +65,21 @@ module Gitlab next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) analyzer.analyze(parsed) - rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e + rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end end # Enable query analyzers - def begin! - analyzers = all_analyzers.select do |analyzer| + def begin!(analyzers = all_analyzers) + analyzers = analyzers.select do |analyzer| if analyzer.enabled? analyzer.begin! true end - rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e + rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) false @@ -88,7 +92,7 @@ module Gitlab def end! enabled_analyzers.select do |analyzer| analyzer.end! - rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e + rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index a604f79dc41..a53da514df2 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -94,7 +94,7 @@ module Gitlab if schemas.many? message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ - "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables." \ + "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." if schemas.any? { |s| s.to_s.start_with?("undefined") } diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb new file mode 100644 index 00000000000..ab40ba5d59b --- /dev/null +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class RestrictAllowedSchemas < Base + UnsupportedSchemaError = Class.new(QueryAnalyzerError) + DDLNotAllowedError = Class.new(UnsupportedSchemaError) + DMLNotAllowedError = Class.new(UnsupportedSchemaError) + DMLAccessDeniedError = Class.new(UnsupportedSchemaError) + + IGNORED_SCHEMAS = %i[gitlab_shared].freeze + + class << self + def enabled? + true + end + + def allowed_gitlab_schemas + self.context[:allowed_gitlab_schemas] + end + + def allowed_gitlab_schemas=(value) + self.context[:allowed_gitlab_schemas] = value + end + + def analyze(parsed) + # If list of schemas is empty, we allow only DDL changes + if self.dml_mode? + self.restrict_to_dml_only(parsed) + else + self.restrict_to_ddl_only(parsed) + end + end + + def require_ddl_mode!(message = "") + return unless self.context + + self.raise_dml_not_allowed_error(message) if self.dml_mode? + end + + def require_dml_mode!(message = "") + return unless self.context + + self.raise_ddl_not_allowed_error(message) if self.ddl_mode? + end + + private + + def restrict_to_ddl_only(parsed) + tables = self.dml_tables(parsed) + schemas = self.dml_schemas(tables) + + if schemas.any? + self.raise_dml_not_allowed_error("Modifying of '#{tables}' (#{schemas.to_a}) with '#{parsed.sql}'") + end + end + + def restrict_to_dml_only(parsed) + if parsed.pg.ddl_tables.any? + self.raise_ddl_not_allowed_error("Modifying of '#{parsed.pg.ddl_tables}' with '#{parsed.sql}'") + end + + if parsed.pg.ddl_functions.any? + self.raise_ddl_not_allowed_error("Modifying of '#{parsed.pg.ddl_functions}' with '#{parsed.sql}'") + end + + tables = self.dml_tables(parsed) + schemas = self.dml_schemas(tables) + + if (schemas - self.allowed_gitlab_schemas).any? + raise DMLAccessDeniedError, "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ + "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'." + end + end + + def dml_mode? + self.allowed_gitlab_schemas&.any? + end + + def ddl_mode? + !self.dml_mode? + end + + def dml_tables(parsed) + parsed.pg.select_tables + parsed.pg.dml_tables + end + + def dml_schemas(tables) + extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) + extra_schemas.subtract(IGNORED_SCHEMAS) + extra_schemas + end + + def raise_dml_not_allowed_error(message) + raise DMLNotAllowedError, "Select/DML queries (SELECT/UPDATE/DELETE) are disallowed in the DDL (structure) mode. #{message}" + end + + def raise_ddl_not_allowed_error(message) + raise DDLNotAllowedError, "DDL queries (structure) are disallowed in the Select/DML (SELECT/UPDATE/DELETE) mode. #{message}" + end + end + end + end + end +end diff --git a/lib/gitlab/database/transaction/context.rb b/lib/gitlab/database/transaction/context.rb index a902537f02e..6defe9ae443 100644 --- a/lib/gitlab/database/transaction/context.rb +++ b/lib/gitlab/database/transaction/context.rb @@ -6,8 +6,10 @@ module Gitlab class Context attr_reader :context - LOG_SAVEPOINTS_THRESHOLD = 1 # 1 `SAVEPOINT` created in a transaction - LOG_DURATION_S_THRESHOLD = 120 # transaction that is running for 2 minutes or longer + LOG_SAVEPOINTS_THRESHOLD = 1 # 1 `SAVEPOINT` created in a transaction + LOG_DURATION_S_THRESHOLD = 120 # transaction that is running for 2 minutes or longer + LOG_EXTERNAL_HTTP_COUNT_THRESHOLD = 50 # 50 external HTTP requests executed within transaction + LOG_EXTERNAL_HTTP_DURATION_S_THRESHOLD = 1 # 1 second spent in HTTP requests in total within transaction LOG_THROTTLE_DURATION = 1 def initialize @@ -43,6 +45,11 @@ module Gitlab (@context[:backtraces] ||= []).push(cleaned_backtrace) end + def initialize_external_http_tracking + @context[:external_http_count_start] = external_http_requests_count_total + @context[:external_http_duration_start] = external_http_requests_duration_total + end + def duration return unless @context[:start_time].present? @@ -57,10 +64,16 @@ module Gitlab duration.to_i >= LOG_DURATION_S_THRESHOLD end + def external_http_requests_threshold_exceeded? + external_http_requests_count >= LOG_EXTERNAL_HTTP_COUNT_THRESHOLD || + external_http_requests_duration >= LOG_EXTERNAL_HTTP_DURATION_S_THRESHOLD + end + def should_log? return false if logged_already? - savepoints_threshold_exceeded? || duration_threshold_exceeded? + savepoints_threshold_exceeded? || duration_threshold_exceeded? || + external_http_requests_threshold_exceeded? end def commit @@ -75,6 +88,14 @@ module Gitlab @context[:backtraces].to_a end + def external_http_requests_count + @requests_count ||= external_http_requests_count_total - @context[:external_http_count_start].to_i + end + + def external_http_requests_duration + @requests_duration ||= external_http_requests_duration_total - @context[:external_http_duration_start].to_f + end + private def queries @@ -108,6 +129,8 @@ module Gitlab savepoints_count: @context[:savepoints].to_i, rollbacks_count: @context[:rollbacks].to_i, releases_count: @context[:releases].to_i, + external_http_requests_count: external_http_requests_count, + external_http_requests_duration: external_http_requests_duration, sql: queries, savepoint_backtraces: backtraces } @@ -118,6 +141,14 @@ module Gitlab def application_info(attributes) Gitlab::AppJsonLogger.info(attributes) end + + def external_http_requests_count_total + ::Gitlab::Metrics::Subscribers::ExternalHttp.request_count.to_i + end + + def external_http_requests_duration_total + ::Gitlab::Metrics::Subscribers::ExternalHttp.duration.to_f + end end end end diff --git a/lib/gitlab/database/transaction/observer.rb b/lib/gitlab/database/transaction/observer.rb index ad6886a3d52..87e76257c24 100644 --- a/lib/gitlab/database/transaction/observer.rb +++ b/lib/gitlab/database/transaction/observer.rb @@ -21,6 +21,7 @@ module Gitlab context.set_start_time context.set_depth(0) context.track_sql(event.payload[:sql]) + context.initialize_external_http_tracking elsif cmd.start_with?('SAVEPOINT', 'EXCEPTION') context.set_depth(manager.open_transactions) context.increment_savepoints diff --git a/lib/gitlab/database/type/color.rb b/lib/gitlab/database/type/color.rb new file mode 100644 index 00000000000..32ea33a42f3 --- /dev/null +++ b/lib/gitlab/database/type/color.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + class Color < ActiveModel::Type::Value + def serialize(value) + value.to_s if value + end + + def serializable?(value) + value.nil? || value.is_a?(::String) || value.is_a?(::Gitlab::Color) + end + + def cast_value(value) + ::Gitlab::Color.new(value.to_s) + end + end + end + end +end diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb index 3928ece9281..af1fd8fb03e 100644 --- a/lib/gitlab/diff/custom_diff.rb +++ b/lib/gitlab/diff/custom_diff.rb @@ -10,16 +10,16 @@ module Gitlab transformed_for_diff(new_blob, old_blob) Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) end - rescue IpynbDiff::InvalidNotebookError => e + rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e Gitlab::ErrorTracking.log_exception(e) nil end def transformed_diff(before, after) transformed_diff = IpynbDiff.diff(before, after, - diff_opts: { context: 5, include_diff_info: true }, - transform_options: { cell_decorator: :percent }, - raise_if_invalid_notebook: true) + raise_if_invalid_nb: true, + diffy_opts: { include_diff_info: true }).to_s(:text) + strip_diff_frontmatter(transformed_diff) end @@ -29,9 +29,7 @@ module Gitlab def transformed_blob_data(blob) if transformed_for_diff?(blob) - IpynbDiff.transform(blob.data, - raise_errors: true, - options: { include_metadata: false, cell_decorator: :percent }) + IpynbDiff.transform(blob.data, raise_errors: true, include_frontmatter: false) end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d9860d9fb86..89822af2455 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,11 +44,15 @@ module Gitlab new_blob_lazy old_blob_lazy - diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff if use_custom_diff? + diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff unless use_renderable_diff? end - def use_custom_diff? - strong_memoize(:_custom_diff_enabled) { Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true) } + def use_renderable_diff? + strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project, default_enabled: :yaml) } + end + + def has_renderable? + rendered&.has_renderable? end def position(position_marker, position_type: :text) @@ -292,13 +296,13 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def size - valid_blobs.map(&:size).sum + valid_blobs.sum(&:size) end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def raw_size - valid_blobs.map(&:raw_size).sum + valid_blobs.sum(&:raw_size) end # rubocop: enable CodeReuse/ActiveRecord @@ -370,10 +374,16 @@ module Gitlab lines.none? { |line| line.type.to_s == 'match' } end + def rendered + return unless use_renderable_diff? && ipynb? + + strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } + end + private def diffable_by_attribute? - repository.attributes(file_path).fetch('diff') { true } + repository.attributes(file_path).fetch('diff', true) end # NOTE: Files with unsupported encodings (e.g. UTF-16) are treated as binary by git, but they are recognized as text files during encoding detection. These files have `Binary files a/filename and b/filename differ' as their raw diff content which cannot be used. We need to handle this special case and avoid displaying incorrect diff. @@ -399,6 +409,10 @@ module Gitlab new_file? || deleted_file? || content_changed? end + def ipynb? + modified_file? && file_path.ends_with?('.ipynb') + end + # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index f73e060be7f..7fa1bd6b5ec 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -11,7 +11,7 @@ module Gitlab delegate :count, :size, :real_size, to: :raw_diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true, use_extra_viewer_as_main: false) end def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) @@ -25,6 +25,7 @@ module Gitlab @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs @repository = project.repository + @use_extra_viewer_as_main = diff_options.delete(:use_extra_viewer_as_main) end def diffs @@ -120,11 +121,17 @@ module Gitlab stats = diff_stats_collection&.find_by_path(diff.new_path) - Gitlab::Diff::File.new(diff, + diff_file = Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs, stats: stats) + + if @use_extra_viewer_as_main && diff_file.has_renderable? + diff_file.rendered + else + diff_file + end end def sort_diffs(diffs) diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index b459e3f6619..c6ab56e783a 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -12,10 +12,11 @@ module Gitlab @merge_request_diff = merge_request_diff super(merge_request_diff, - project: merge_request_diff.project, - diff_options: diff_options, - diff_refs: merge_request_diff.diff_refs, - fallback_diff_refs: merge_request_diff.fallback_diff_refs) + project: merge_request_diff.project, + diff_options: diff_options, + diff_refs: merge_request_diff.diff_refs, + fallback_diff_refs: merge_request_diff.fallback_diff_refs + ) end def diff_files(sorted: false) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 6cf414e29cc..c2b834c71b5 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -8,9 +8,9 @@ module Gitlab # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code, :marker_ranges - attr_writer :text, :rich_text - attr_accessor :index, :type, :old_pos, :new_pos + attr_reader :marker_ranges + attr_writer :text, :rich_text, :discussable + attr_accessor :index, :type, :old_pos, :new_pos, :line_code def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text = text @@ -26,6 +26,7 @@ module Gitlab @line_code = line_code || calculate_line_code @marker_ranges = [] + @discussable = true end def self.init_from_hash(hash) @@ -96,7 +97,7 @@ module Gitlab end def discussable? - !meta? + @discussable && !meta? end def suggestible? diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb new file mode 100644 index 00000000000..e700e730f20 --- /dev/null +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true +module Gitlab + module Diff + module Rendered + module Notebook + include Gitlab::Utils::StrongMemoize + + class DiffFile < Gitlab::Diff::File + attr_reader :source_diff + + delegate :repository, :diff_refs, :fallback_diff_refs, :unfolded, :unique_identifier, + to: :source_diff + + def initialize(diff_file) + @source_diff = diff_file + end + + def old_blob + return unless notebook_diff + + strong_memoize(:old_blob) { ::Blobs::Notebook.decorate(source_diff.old_blob, notebook_diff.from.as_text) } + end + + def new_blob + return unless notebook_diff + + strong_memoize(:new_blob) { ::Blobs::Notebook.decorate(source_diff.new_blob, notebook_diff.to.as_text) } + end + + def diff + strong_memoize(:diff) { transformed_diff } + end + + def has_renderable? + !notebook_diff.nil? && diff.diff.present? + end + + def rendered + self + end + + def highlighted_diff_lines + @highlighted_diff_lines ||= begin + removal_line_maps, addition_line_maps = compute_end_start_map + Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight.map do |line| + mutate_line(line, addition_line_maps, removal_line_maps) + end + end + end + + private + + def notebook_diff + strong_memoize(:notebook_diff) do + Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) + + IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, + raise_if_invalid_nb: true, + diffy_opts: { include_diff_info: true }) + rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e + Gitlab::ErrorTracking.log_exception(e) + nil + end + end + + def transformed_diff + return unless notebook_diff + + diff = source_diff.diff.clone + diff.diff = strip_diff_frontmatter(notebook_diff.to_s(:text)) + diff + end + + def strip_diff_frontmatter(diff_content) + diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present? + end + + def transformed_line_to_source(transformed_line, transformed_blocks) + transformed_blocks.empty? ? 0 : ( transformed_blocks[transformed_line - 1][:source_line] || -1 ) + 1 + end + + def mutate_line(line, addition_line_maps, removal_line_maps) + line.new_pos = transformed_line_to_source(line.new_pos, notebook_diff.to.blocks) + line.old_pos = transformed_line_to_source(line.old_pos, notebook_diff.from.blocks) + + line.old_pos = addition_line_maps[line.new_pos] if line.old_pos == 0 && line.new_pos != 0 + line.new_pos = removal_line_maps[line.old_pos] if line.new_pos == 0 && line.old_pos != 0 + + # Lines that do not appear on the original diff should not be commentable + + unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos] + line.discussable = false + end + + line.line_code = line_code(line) + line + end + + def compute_end_start_map + # line_codes are used for assigning notes to diffs, and these depend on the line on the new version and the + # line that would have been that one in the previous version. However, since we do a transformation on the + # file, that map gets lost. To overcome this, we look at the original source lines and build two maps: + # - For additions, we look at the latest line change for that line and pick the old line for that id + # - For removals, we look at the first line in the old version, and pick the first line on the new version + # + # + # The caveat here is that we can't have notes on lines that are not a translation of a line in the source + # diff + # + # (gitlab/diff/file.rb:75) + + removals = {} + additions = {} + + source_diff.highlighted_diff_lines.each do |line| + removals[line.old_pos] = line.new_pos + additions[line.new_pos] = line.old_pos + end + + [removals, additions] + end + end + end + end + end +end diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index e213adbfcfd..b67ca8d8a7d 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -15,7 +15,9 @@ module Gitlab filter_signature_attachments(message).each do |attachment| tmp = Tempfile.new("gitlab-email-attachment") begin - File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded } + content = attachment.body.decoded + File.open(tmp.path, "w+b") { |f| f.write content } + sanitize_exif_if_needed(content, tmp.path) file = { tempfile: tmp, @@ -55,6 +57,12 @@ module Gitlab def normalize_mime(content_type) MIME::Type.simplified(content_type, remove_x_prefix: true) end + + # https://gitlab.com/gitlab-org/gitlab/-/issues/239343 + def sanitize_exif_if_needed(content, path) + exif_sanitizer = Gitlab::Sanitizers::Exif.new + exif_sanitizer.clean_existing_path(path, content: content, skip_unallowed_types: true) + end end end end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 71b1d4ed8f9..bb57494c729 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -34,7 +34,7 @@ module Gitlab create_issue_or_note - if from_address + if issue_creator_address add_email_participant send_thank_you_email unless reply_email? end @@ -98,7 +98,7 @@ module Gitlab title: mail.subject, description: message_including_template, confidential: true, - external_author: from_address + external_author: external_author }, spam_params: nil ).execute @@ -176,8 +176,22 @@ module Gitlab ).execute end + def issue_creator_address + reply_to_address || from_address + end + def from_address - (mail.reply_to || []).first || mail.from.first || mail.sender + mail.from.first || mail.sender + end + + def reply_to_address + (mail.reply_to || []).first + end + + def external_author + return issue_creator_address unless reply_to_address && from_address + + _("%{from_address} (reply to: %{reply_to_address})") % { from_address: from_address, reply_to_address: reply_to_address } end def can_handle_legacy_format? @@ -191,7 +205,7 @@ module Gitlab def add_email_participant return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) - @issue.issue_email_participants.create(email: from_address) + @issue.issue_email_participants.create(email: issue_creator_address) end end end diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb index 77f299bcade..27ba5d2a314 100644 --- a/lib/gitlab/email/html_parser.rb +++ b/lib/gitlab/email/html_parser.rb @@ -8,6 +8,7 @@ module Gitlab end attr_reader :raw_body + def initialize(raw_body) @raw_body = raw_body end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 5b2bbfbe66b..58e7b2f1b44 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -8,6 +8,8 @@ module Gitlab class Receiver include Gitlab::Utils::StrongMemoize + RECEIVED_HEADER_REGEX = /for\s+\<(.+)\>/.freeze + def initialize(raw) @raw = raw end @@ -37,6 +39,8 @@ module Gitlab delivered_to: delivered_to.map(&:value), envelope_to: envelope_to.map(&:value), x_envelope_to: x_envelope_to.map(&:value), + # reduced down to what looks like an email in the received headers + received_recipients: recipients_from_received_headers, meta: { client_id: "email/#{mail.from.first}", project: handler&.project&.full_path @@ -82,7 +86,8 @@ module Gitlab find_key_from_references || find_key_from_delivered_to_header || find_key_from_envelope_to_header || - find_key_from_x_envelope_to_header + find_key_from_x_envelope_to_header || + find_first_key_from_received_headers end def ensure_references_array(references) @@ -117,6 +122,10 @@ module Gitlab Array(mail[:x_envelope_to]) end + def received + Array(mail[:received]) + end + def find_key_from_delivered_to_header delivered_to.find do |header| key = email_class.key_from_address(header.value) @@ -138,6 +147,21 @@ module Gitlab end end + def find_first_key_from_received_headers + return unless ::Feature.enabled?(:use_received_header_for_incoming_emails, default_enabled: :yaml) + + recipients_from_received_headers.find do |email| + key = email_class.key_from_address(email) + break key if key + end + end + + def recipients_from_received_headers + strong_memoize :emails_from_received_headers do + received.map { |header| header.value[RECEIVED_HEADER_REGEX, 1] }.compact + end + end + def ignore_auto_reply! if auto_submitted? || auto_replied? raise AutoGeneratedEmailError diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 6a637306225..259b430a73c 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -23,7 +23,12 @@ module Gitlab ].freeze class << self - def configure + def configure(&block) + configure_raven(&block) + configure_sentry(&block) + end + + def configure_raven Raven.configure do |config| config.dsn = sentry_dsn config.release = Gitlab.revision @@ -34,7 +39,20 @@ module Gitlab # Sanitize authentication headers config.sanitize_http_headers = %w[Authorization Private-Token] - config.before_send = method(:before_send) + config.before_send = method(:before_send_raven) + + yield config if block_given? + end + end + + def configure_sentry + Sentry.init do |config| + config.dsn = new_sentry_dsn + config.release = Gitlab.revision + config.environment = new_sentry_environment + config.before_send = method(:before_send_sentry) + config.background_worker_threads = 0 + config.send_default_pii = true yield config if block_given? end @@ -96,6 +114,18 @@ module Gitlab private + def before_send_raven(event, hint) + return unless Feature.enabled?(:enable_old_sentry_integration, default_enabled: :yaml) + + before_send(event, hint) + end + + def before_send_sentry(event, hint) + return unless Feature.enabled?(:enable_new_sentry_integration, default_enabled: :yaml) + + before_send(event, hint) + end + def before_send(event, hint) inject_context_for_exception(event, hint[:exception]) custom_fingerprinting(event, hint[:exception]) @@ -112,6 +142,13 @@ module Gitlab Raven.capture_exception(exception, **context_payload) end + # There is a possibility that this method is called before Sentry is + # configured. Since Sentry 4.0, some methods of Sentry are forwarded to + # to `nil`, hence we have to check the client as well. + if sentry && ::Sentry.get_current_client && ::Sentry.configuration.dsn + ::Sentry.capture_exception(exception, **context_payload) + end + if logging formatter = Gitlab::ErrorTracking::LogFormatter.new log_hash = formatter.generate_log(exception, context_payload) @@ -121,12 +158,30 @@ module Gitlab end def sentry_dsn - return unless Rails.env.production? || Rails.env.development? + return unless sentry_configurable? return unless Gitlab.config.sentry.enabled Gitlab.config.sentry.dsn end + def new_sentry_dsn + return unless sentry_configurable? + return unless Gitlab::CurrentSettings.respond_to?(:sentry_enabled?) + return unless Gitlab::CurrentSettings.sentry_enabled? + + Gitlab::CurrentSettings.sentry_dsn + end + + def new_sentry_environment + return unless Gitlab::CurrentSettings.respond_to?(:sentry_environment) + + Gitlab::CurrentSettings.sentry_environment + end + + def sentry_configurable? + Rails.env.production? || Rails.env.development? + end + def should_raise_for_dev? Rails.env.development? || Rails.env.test? end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index e835deeea2c..045a18f4110 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -18,7 +18,7 @@ module Gitlab # only the first one since that's what is used for grouping. def process_first_exception_value(event) # Better in new version, will be event.exception.values - exceptions = event.instance_variable_get(:@interfaces)[:exception]&.values + exceptions = extract_exceptions_from(event) return unless exceptions.is_a?(Array) @@ -37,7 +37,13 @@ module Gitlab # instance variable if message.present? exceptions.each do |exception| - exception.value = message if valid_exception?(exception) + next unless valid_exception?(exception) + + if exception.respond_to?(:value=) + exception.value = message + else + exception.instance_variable_set(:@value, message) + end end end @@ -55,6 +61,14 @@ module Gitlab private + def extract_exceptions_from(event) + if event.is_a?(Raven::Event) + event.instance_variable_get(:@interfaces)[:exception]&.values + else + event.exception&.instance_variable_get(:@values) + end + end + def custom_grpc_fingerprint?(fingerprint) fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::') end @@ -71,7 +85,7 @@ module Gitlab def valid_exception?(exception) case exception - when Raven::SingleExceptionInterface + when Raven::SingleExceptionInterface, Sentry::SingleExceptionInterface exception&.value else false diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index d5bf0cffb1e..a1918ee6ad5 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -67,7 +67,10 @@ module Gitlab add_instrument_for_cache_hit(status_code, route, request) - Gitlab::ApplicationContext.push(feature_category: route.feature_category) + Gitlab::ApplicationContext.push( + feature_category: route.feature_category, + caller_id: route.caller_id + ) new_headers = { 'ETag' => etag, diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 742b72ecde9..684afc6762a 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -3,22 +3,33 @@ module Gitlab module EtagCaching module Router - Route = Struct.new(:regexp, :name, :feature_category, :router) do + Route = Struct.new(:router, :regexp, :name, :feature_category, :caller_id) do delegate :match, to: :regexp delegate :cache_key, to: :router end module Helpers def build_route(attrs) - EtagCaching::Router::Route.new(*attrs, self) + EtagCaching::Router::Route.new(self, *attrs) + end + + def build_rails_route(attrs) + regexp, name, controller, action_name = *attrs + EtagCaching::Router::Route.new( + self, + regexp, + name, + controller.feature_category_for_action(action_name).to_s, + controller.endpoint_id_for_action(action_name).to_s + ) end end - # Performing RESTful routing match before GraphQL would be more expensive + # Performing Rails routing match before GraphQL would be more expensive # for the GraphQL requests because we need to traverse all of the RESTful # route definitions before falling back to GraphQL. def self.match(request) - Router::Graphql.match(request) || Router::Restful.match(request) + Router::Graphql.match(request) || Router::Rails.match(request) end end end diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/rails.rb index 176676bd6ba..d80c003fe53 100644 --- a/lib/gitlab/etag_caching/router/restful.rb +++ b/lib/gitlab/etag_caching/router/rails.rb @@ -3,7 +3,7 @@ module Gitlab module EtagCaching module Router - class Restful + class Rails extend EtagCaching::Router::Helpers # We enable an ETag for every request matching the regex. @@ -23,74 +23,88 @@ module Gitlab [ %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), 'issue_notes', - 'team_planning' + ::Projects::NotesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), 'merge_request_notes', - 'code_review' + ::Projects::NotesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), 'issue_title', - 'team_planning' + ::Projects::IssuesController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), 'commit_pipelines', - 'continuous_integration' + ::Projects::CommitController, + :pipelines ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), 'new_merge_request_pipelines', - 'continuous_integration' + ::Projects::MergeRequests::CreationsController, + :new ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), 'merge_request_pipelines', - 'continuous_integration' + ::Projects::MergeRequestsController, + :pipelines ], [ %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), 'project_pipelines', - 'continuous_integration' + ::Projects::PipelinesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), 'project_pipeline', - 'continuous_integration' + ::Projects::PipelinesController, + :show ], [ %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), 'project_build', - 'continuous_integration' + ::Projects::BuildsController, + :show ], [ %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), 'cluster_environments', - 'continuous_delivery' + ::Groups::ClustersController, + :environments ], [ %r(#{RESERVED_WORDS_PREFIX}/-/environments\.json\z), 'environments', - 'continuous_delivery' + ::Projects::EnvironmentsController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), 'realtime_changes_import_github', - 'importers' + ::Import::GithubController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), 'realtime_changes_import_gitea', - 'importers' + ::Import::GiteaController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), 'merge_request_widget', - 'code_review' + ::Projects::MergeRequests::ContentController, + :cached_widget ] - ].map(&method(:build_route)).freeze + ].map(&method(:build_rails_route)).freeze # Overridden in EE to add more routes def self.all_routes @@ -109,4 +123,4 @@ module Gitlab end end -Gitlab::EtagCaching::Router::Restful.prepend_mod_with('Gitlab::EtagCaching::Router::Restful') +Gitlab::EtagCaching::Router::Rails.prepend_mod_with('Gitlab::EtagCaching::Router::Rails') diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index 5a14e3c272e..70c363877b1 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -12,10 +12,11 @@ module Gitlab # - not have rolled out the feature flag at all (no percent of actors, # no inclusions, etc.) def enabled? - return false if ::Feature::Definition.get(feature_flag_name).nil? - return false unless Gitlab.dev_env_or_com? + return false unless feature_flag_defined? + return false unless Gitlab.com? + return false unless ::Feature.enabled?(:gitlab_experiment, type: :ops, default_enabled: :yaml) - ::Feature.get(feature_flag_name).state != :off # rubocop:disable Gitlab/AvoidFeatureGet + feature_flag_instance.state != :off end # For assignment we first check to see if our feature flag is enabled @@ -58,6 +59,14 @@ module Gitlab private + def feature_flag_instance + ::Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet + end + + def feature_flag_defined? + ::Feature::Definition.get(feature_flag_name).present? + end + def feature_flag_name experiment.name.tr('/', '_') end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 7edda290204..8a5432025d8 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -10,9 +10,9 @@ # The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), # which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. # -# To enable the experiment for 10% of the users: +# To enable the experiment for 10% of the time: # -# chatops: `/chatops run feature set experiment_key_experiment_percentage 10` +# chatops: `/chatops run feature set experiment_key_experiment_percentage 10 --random` # console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)` # # To disable the experiment: diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 303d952381f..a68e2db4dac 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -20,7 +20,7 @@ module Gitlab end def set_experimentation_subject_id_cookie - if Gitlab.dev_env_or_com? + if Gitlab.com? return if cookies[:experimentation_subject_id].present? cookies.permanent.signed[:experimentation_subject_id] = { diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index 8ba95520638..b13f55e7969 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -18,7 +18,7 @@ module Gitlab # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? Feature.enabled?(feature_flag_name, type: :experiment, default_enabled: :yaml) - ::Gitlab.dev_env_or_com? && experiment_percentage > 0 + ::Gitlab.com? && experiment_percentage > 0 end def enabled_for_index?(index) diff --git a/lib/gitlab/fips.rb b/lib/gitlab/fips.rb new file mode 100644 index 00000000000..1dd363ceb17 --- /dev/null +++ b/lib/gitlab/fips.rb @@ -0,0 +1,25 @@ +# rubocop: disable Naming/FileName +# frozen_string_literal: true + +module Gitlab + class FIPS + # A simple utility class for FIPS-related helpers + + class << self + # Returns whether we should be running in FIPS mode or not + # + # @return [Boolean] + def enabled? + # Attempt to auto-detect FIPS mode from OpenSSL + return true if OpenSSL.fips_mode + + # Otherwise allow it to be set manually via the env vars + return true if ENV["FIPS_MODE"] == "true" + + false + end + end + end +end + +# rubocop: enable Naming/FileName diff --git a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb index 3f9053d4e0c..e8e87a864cc 100644 --- a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb +++ b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb @@ -16,13 +16,15 @@ module Gitlab :div, class: 'gl-form-checkbox custom-control custom-checkbox' ) do + value = checkbox_options[:multiple] ? checked_value : nil + @template.check_box( @object_name, method, format_options(checkbox_options, ['custom-control-input']), checked_value, unchecked_value - ) + generic_label(method, label, label_options, help_text: help_text) + ) + generic_label(method, label, label_options, help_text: help_text, value: value) end end diff --git a/lib/gitlab/front_matter.rb b/lib/gitlab/front_matter.rb index 5c5c74ca1a0..093501e860b 100644 --- a/lib/gitlab/front_matter.rb +++ b/lib/gitlab/front_matter.rb @@ -11,12 +11,12 @@ module Gitlab DELIM = Regexp.union(DELIM_LANG.keys) PATTERN = %r{ - \A(?:[^\r\n]*coding:[^\r\n]*\R)? # optional encoding line - \s* + \A(?<encoding>[^\r\n]*coding:[^\r\n]*\R)? # optional encoding line + (?<before>\s*) ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*)\R # opening front matter marker (optional language specifier) (?<front_matter>.*?) # front matter block content (not greedy) ^(\k<delim> | \.{3}) # closing front matter marker - \s* + [^\S\r\n]*(\R|\z) }mx.freeze end end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index a5b1b7d914b..5669a65cbd9 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -63,6 +63,7 @@ module Gitlab class BlameLine attr_accessor :lineno, :oldlineno, :commit, :line + def initialize(lineno, oldlineno, commit, line) @lineno = lineno @oldlineno = oldlineno diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index c3ee5b97379..1492ea1ce76 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -99,9 +99,9 @@ module Gitlab gitaly_repository_client.exists? end - def create_repository + def create_repository(default_branch = nil) wrapped_gitaly_errors do - gitaly_repository_client.create_repository + gitaly_repository_client.create_repository(default_branch) rescue GRPC::AlreadyExists => e raise RepositoryExists, e.message end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 194f5da0a5c..4bab94968d7 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -93,9 +93,9 @@ module Gitlab end end - def page(title:, version: nil, dir: nil) + def page(title:, version: nil, dir: nil, load_content: true) wrapped_gitaly_errors do - gitaly_find_page(title: title, version: version, dir: dir) + gitaly_find_page(title: title, version: version, dir: dir, load_content: load_content) end end @@ -121,10 +121,10 @@ module Gitlab gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) end - def gitaly_find_page(title:, version: nil, dir: nil) + def gitaly_find_page(title:, version: nil, dir: nil, load_content: true) return unless title.present? - wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir) + wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir, load_content: load_content) return unless wiki_page Gitlab::Git::WikiPage.new(wiki_page, version) diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 4d87b91764a..5ae17dbbb91 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -30,10 +30,7 @@ module Gitlab def check(cmd, changes) check_snippet_accessibility! - super.tap do |_| - # Ensure HEAD points to the default branch in case it is not master - snippet.change_head_to_default_branch - end + super end override :download_ability diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index a824f97e197..f376dbce177 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -390,7 +390,7 @@ module Gitlab end def self.long_timeout - if Gitlab::Runtime.web_server? + if Gitlab::Runtime.puma? default_timeout else 6.hours diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index c2b4182f609..0e3f9c2598d 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -212,7 +212,7 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) - response.flat_map(&:stats) + response.flat_map { |rsp| rsp.stats.to_a } end def find_changed_paths(commits) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index adbf07de1b9..4637bf2e3ff 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -119,10 +119,6 @@ module Gitlab response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request, timeout: GitalyClient.long_timeout) - if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::PreReceiveError, pre_receive_error - end - response.commit_id end @@ -153,10 +149,6 @@ module Gitlab second_response = response_enum.next - if second_response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError, second_response.pre_receive_error - end - branch_update = second_response.branch_update return if branch_update.nil? raise Gitlab::Git::CommitError, 'failed to apply merge to branch' unless branch_update.commit_id.present? @@ -164,16 +156,20 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) rescue GRPC::BadStatus => e - decoded_error = decode_detailed_error(e) - - raise unless decoded_error.present? - - # We simply ignore any reference update errors which are typically an - # indicator of multiple RPC calls trying to update the same reference - # at the same point in time. - return if decoded_error.is_a?(Gitlab::Git::ReferenceUpdateError) + detailed_error = decode_detailed_error(e) - raise decoded_error + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :reference_update + # We simply ignore any reference update errors which are typically an + # indicator of multiple RPC calls trying to update the same reference + # at the same point in time. + else + raise + end ensure request_enum.close end @@ -267,6 +263,19 @@ module Gitlab perform_next_gitaly_rebase_request(response_enum) rebase_sha + rescue GRPC::BadStatus => e + detailed_error = decode_detailed_error(e) + + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :rebase_conflict + raise Gitlab::Git::Repository::GitError, e.details + else + raise e + end ensure request_enum.close end @@ -295,6 +304,26 @@ module Gitlab end response.squash_sha + rescue GRPC::BadStatus => e + detailed_error = decode_detailed_error(e) + + case detailed_error&.error + when :resolve_revision, :rebase_conflict + # Theoretically, we could now raise specific errors based on the type + # of the detailed error. Most importantly, we get error details when + # Gitaly was not able to resolve the `start_sha` or `end_sha` via a + # ResolveRevisionError, and we get information about which files are + # conflicting via a MergeConflictError. + # + # We don't do this now though such that we can maintain backwards + # compatibility with the minimum required set of changes during the + # transitory period where we're migrating UserSquash to use + # structured errors. We thus continue to just return a GitError, like + # we previously did. + raise Gitlab::Git::Repository::GitError, e.details + else + raise + end end def user_update_submodule(user:, submodule:, commit_sha:, branch:, message:) @@ -492,23 +521,7 @@ module Gitlab prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)} error_type = prefix.match(detailed_error.type_url)[:error_type] - detailed_error = Gitaly.const_get(error_type, false).decode(detailed_error.value) - - case detailed_error.error - when :access_check - access_check_error = detailed_error.access_check - # These messages were returned from internal/allowed API calls - Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) - when :reference_update - reference_update_error = detailed_error.reference_update - Gitlab::Git::ReferenceUpdateError.new(err.details, - reference_update_error.reference_name, - reference_update_error.old_oid, - reference_update_error.new_oid) - else - # We're handling access_check only for now, but we'll add more detailed error types - nil - end + Gitaly.const_get(error_type, false).decode(detailed_error.value) rescue NameError, NoMethodError # Error Class might not be known to ruby yet nil diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 73848dfff5d..5c447dfd417 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -21,6 +21,16 @@ module Gitlab response.exists end + def optimize_repository + request = Gitaly::OptimizeRepositoryRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :optimize_repository, request, timeout: GitalyClient.long_timeout) + end + + def prune_unreachable_objects + request = Gitaly::PruneUnreachableObjectsRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :prune_unreachable_objects, request, timeout: GitalyClient.long_timeout) + end + def garbage_collect(create_bitmap, prune:) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune) GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) @@ -97,8 +107,8 @@ module Gitlab end # rubocop: enable Metrics/ParameterLists - def create_repository - request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) + def create_repository(default_branch = nil) + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: default_branch) GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 3613cd01122..ca839b232cf 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -64,12 +64,13 @@ module Gitlab GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout) end - def find_page(title:, version: nil, dir: nil) + def find_page(title:, version: nil, dir: nil, load_content: true) request = Gitaly::WikiFindPageRequest.new( repository: @gitaly_repo, title: encode_binary(title), revision: encode_binary(version), - directory: encode_binary(dir) + directory: encode_binary(dir), + skip_content: !load_content ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request, timeout: GitalyClient.fast_timeout) diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 02b582190b6..a9f8483d8c3 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport module Importer class DiffNoteImporter + DiffNoteCreationError = Class.new(ActiveRecord::RecordInvalid) + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote` # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` @@ -31,7 +33,7 @@ module Gitlab else import_with_legacy_diff_note end - rescue ::DiffNote::NoteDiffFileCreationError => e + rescue ::DiffNote::NoteDiffFileCreationError, DiffNoteCreationError => e Logger.warn(message: e.message, 'error.class': e.class.name) import_with_legacy_diff_note @@ -84,7 +86,7 @@ module Gitlab def import_with_diff_note log_diff_note_creation('DiffNote') - ::Import::Github::Notes::CreateService.new(project, author, { + record = ::Import::Github::Notes::CreateService.new(project, author, { noteable_type: note.noteable_type, system: false, type: 'DiffNote', @@ -97,6 +99,8 @@ module Gitlab updated_at: note.updated_at, position: note.diff_position }).execute + + raise DiffNoteCreationError, record unless record.persisted? end def note_body diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index fc0c099b71c..5d291d9d723 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -74,6 +74,10 @@ module Gitlab { state: 'all', sort: 'created', direction: 'asc' } end + def parallel_import_batch + { size: 200, delay: 1.minute } + end + def repository_updates_counter @repository_updates_counter ||= Gitlab::Metrics.counter( :github_importer_repository_updates, diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index a8e006ea082..4dec9543a13 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -72,6 +72,14 @@ module Gitlab # Imports all objects in parallel by scheduling a Sidekiq job for every # individual object. def parallel_import + if Feature.enabled?(:spread_parallel_import, default_enabled: :yaml) && parallel_import_batch.present? + spread_parallel_import + else + parallel_import_deprecated + end + end + + def parallel_import_deprecated waiter = JobWaiter.new each_object_to_import do |object| @@ -86,6 +94,33 @@ module Gitlab waiter end + def spread_parallel_import + waiter = JobWaiter.new + + import_arguments = [] + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + import_arguments << [project.id, repr.to_hash, waiter.key] + + waiter.jobs_remaining += 1 + end + + # rubocop:disable Scalability/BulkPerformWithContext + Gitlab::ApplicationContext.with_context(project: project) do + sidekiq_worker_class.bulk_perform_in( + 1.second, + import_arguments, + batch_size: parallel_import_batch[:size], + batch_delay: parallel_import_batch[:delay] + ) + end + # rubocop:enable Scalability/BulkPerformWithContext + + waiter + end + # The method that will be called for traversing through all the objects to # import, yielding them to the supplied block. def each_object_to_import @@ -171,6 +206,12 @@ module Gitlab raise NotImplementedError end + # Default batch settings for parallel import (can be redefined in Importer classes) + # Example: { size: 100, delay: 1.minute } + def parallel_import_batch + {} + end + def abort_on_failure false end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2bd59415771..9f18513f066 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -40,7 +40,6 @@ module Gitlab gon.ee = Gitlab.ee? gon.jh = Gitlab.jh? gon.dot_com = Gitlab.com? - gon.dev_env_or_com = Gitlab.dev_env_or_com? if current_user gon.current_user_id = current_user.id @@ -52,13 +51,15 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) push_frontend_feature_flag(:sandboxed_mermaid, default_enabled: :yaml) + push_frontend_feature_flag(:source_editor_toolbar, default_enabled: :yaml) + push_frontend_feature_flag(:gl_avatar_for_all_user_avatars, default_enabled: :yaml) + push_frontend_feature_flag(:mr_attention_requests, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/batch_key.rb b/lib/gitlab/graphql/batch_key.rb index 51203af5a43..553e0573c63 100644 --- a/lib/gitlab/graphql/batch_key.rb +++ b/lib/gitlab/graphql/batch_key.rb @@ -4,6 +4,7 @@ module Gitlab module Graphql class BatchKey attr_reader :object + delegate :hash, to: :object def initialize(object, lookahead = nil, object_name: nil) diff --git a/lib/gitlab/graphql/loaders/batch_commit_loader.rb b/lib/gitlab/graphql/loaders/batch_commit_loader.rb new file mode 100644 index 00000000000..26c1f61c567 --- /dev/null +++ b/lib/gitlab/graphql/loaders/batch_commit_loader.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class BatchCommitLoader + def initialize(container_class:, container_id:, oid:) + @container_class = container_class + @container_id = container_id + @oid = oid + end + + def find + Gitlab::Graphql::Lazy.with_value(find_containers) do |container| + BatchLoader::GraphQL.for(oid).batch(key: container) do |oids, loader, args| + container = args[:key] + + container.repository.commits_by(oids: oids).each do |commit| + loader.call(commit.id, commit) if commit + end + end + end + end + + private + + def find_containers + BatchLoader::GraphQL.for(container_id.to_i).batch(key: container_class) do |ids, loader, args| + model = args[:key] + results = model.includes(:route).id_in(ids) # rubocop: disable CodeReuse/ActiveRecord + + results.each { |record| loader.call(record.id, record) } + end + end + + attr_reader :container_class, :container_id, :oid + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index 15f95edd318..e8335a3c79c 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -17,21 +17,13 @@ module Gitlab strong_memoize(:generic_keyset_pagination_has_next_page) do if before - # If `before` is specified, that points to a specific record, - # even if it's the last one. Since we're asking for `before`, - # then the specific record we're pointing to is in the - # next page true elsif first case sliced_nodes when Array sliced_nodes.size > limit_value else - # If we count the number of requested items plus one (`limit_value + 1`), - # then if we get `limit_value + 1` then we know there is a next page sliced_nodes.limit(1).offset(limit_value).exists? - # replacing relation count - # relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 end else false diff --git a/lib/gitlab/harbor/client.rb b/lib/gitlab/harbor/client.rb new file mode 100644 index 00000000000..06142ae2b40 --- /dev/null +++ b/lib/gitlab/harbor/client.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Harbor + class Client + Error = Class.new(StandardError) + ConfigError = Class.new(Error) + + attr_reader :integration + + def initialize(integration) + raise ConfigError, 'Please check your integration configuration.' unless integration + + @integration = integration + end + + def ping + options = { headers: headers.merge!('Accept': 'text/plain') } + response = Gitlab::HTTP.get(url('ping'), options) + + { success: response.success? } + end + + private + + def url(path) + Gitlab::Utils.append_path(base_url, path) + end + + def base_url + Gitlab::Utils.append_path(integration.url, '/api/v2.0/') + end + + def headers + auth = Base64.strict_encode64("#{integration.username}:#{integration.password}") + { + 'Content-Type': 'application/json', + 'Authorization': "Basic #{auth}" + } + end + end + end +end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index ec4b97eaca4..3df312af1bc 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -13,12 +13,14 @@ module Gitlab end def successful?(result) - result == '1' + result == Gitlab::Database.database_base_models.size end def check catch_timeout 10.seconds do - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s + Gitlab::Database.database_base_models.sum do |_, base| + base.connection.select_value('SELECT 1') + end end end end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 49712548960..758a594036b 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -11,11 +11,7 @@ module Gitlab end def self.too_large?(size) - return false unless size.to_i > self.file_size_limit - - over_highlight_size_limit.increment(source: "file size: #{self.file_size_limit}") if Feature.enabled?(:track_file_size_over_highlight_limit) - - true + size.to_i > self.file_size_limit end attr_reader :blob_name @@ -74,14 +70,10 @@ module Gitlab end def highlight_rich(text, continue: true) - add_highlight_attempt_metric - tag = lexer.tag tokens = lexer.lex(text, continue: continue) Timeout.timeout(timeout_time) { @formatter.format(tokens, **context, tag: tag).html_safe } rescue Timeout::Error => e - add_highlight_timeout_metric - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) highlight_plain(text) rescue StandardError @@ -95,38 +87,5 @@ module Gitlab def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end - - def add_highlight_attempt_metric - return unless Feature.enabled?(:track_highlight_timeouts) - - highlighting_attempt.increment(source: (@language || "undefined")) - end - - def add_highlight_timeout_metric - return unless Feature.enabled?(:track_highlight_timeouts) - - highlight_timeout.increment(source: Gitlab::Runtime.sidekiq? ? "background" : "foreground") - end - - def highlighting_attempt - @highlight_attempt ||= Gitlab::Metrics.counter( - :file_highlighting_attempt, - 'Counts the times highlighting has been attempted on a file' - ) - end - - def highlight_timeout - @highlight_timeout ||= Gitlab::Metrics.counter( - :highlight_timeout, - 'Counts the times highlights have timed out' - ) - end - - def self.over_highlight_size_limit - @over_highlight_size_limit ||= Gitlab::Metrics.counter( - :over_highlight_size_limit, - 'Count the times files have been over the highlight size limit' - ) - end end end diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index b8da6731081..5c8aa5050ed 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -26,7 +26,7 @@ module Gitlab end def safe_keys - issuable_builder.safe_hook_attributes + issuable_builder::SAFE_HOOK_RELATIONS + issuable_builder.safe_hook_attributes + issuable_builder.safe_hook_relations end private diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index 181ce447b52..bd0603c5e5b 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -3,13 +3,16 @@ module Gitlab module HookData class IssueBuilder < BaseBuilder - SAFE_HOOK_RELATIONS = %i[ - assignees - labels - total_time_spent - time_change - severity - ].freeze + def self.safe_hook_relations + %i[ + assignees + labels + total_time_spent + time_change + severity + escalation_status + ].freeze + end def self.safe_hook_attributes %i[ @@ -56,6 +59,10 @@ module Gitlab severity: issue.severity } + if issue.supports_escalation? && issue.escalation_status + attrs[:escalation_status] = issue.escalation_status.status_name + end + issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) .merge!(attrs) end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 0e787a77a25..aaca16d8d7c 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -34,12 +34,14 @@ module Gitlab ].freeze end - SAFE_HOOK_RELATIONS = %i[ - assignees - labels - total_time_spent - time_change - ].freeze + def self.safe_hook_relations + %i[ + assignees + labels + total_time_spent + time_change + ].freeze + end alias_method :merge_request, :object diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index dfecf3a669e..002708beb3c 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -29,7 +29,7 @@ module Gitlab http = super http.hostname_override = hostname if hostname - if Feature.enabled?(:header_read_timeout_buffered_io) + if Feature.enabled?(:header_read_timeout_buffered_io, default_enabled: :yaml) gitlab_http = Gitlab::NetHttpAdapter.new(http.address, http.port) http.instance_variables.each do |variable| @@ -47,6 +47,7 @@ module Gitlab def validate_url!(url) Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?, + allow_object_storage: allow_object_storage?, dns_rebind_protection: dns_rebind_protection?) rescue Gitlab::UrlBlocker::BlockedUrlError => e raise Gitlab::HTTP::BlockedUrlError, "URL '#{url}' is blocked: #{e.message}" @@ -56,6 +57,10 @@ module Gitlab options.fetch(:allow_local_requests, allow_settings_local_requests?) end + def allow_object_storage? + options.fetch(:allow_object_storage, false) + end + def dns_rebind_protection? return false if Gitlab.http_proxy_env? diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 584f7d4aeaf..d01f7d0074f 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 48, + 'da_DK' => 46, 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 39, + 'es' => 40, 'fil_PH' => 0, 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 35, - 'ko' => 13, - 'nb_NO' => 31, + 'ja' => 34, + 'ko' => 12, + 'nb_NO' => 30, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 50, + 'pt_BR' => 49, 'ro_RO' => 22, 'ru' => 32, 'tr_TR' => 14, - 'uk' => 44, - 'zh_CN' => 96, + 'uk' => 48, + 'zh_CN' => 95, 'zh_HK' => 2, 'zh_TW' => 2 }.freeze diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 8a8c74c302d..53dd6f8cd55 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -300,7 +300,7 @@ module Gitlab return cache[table_name] if cache.has_key?(table_name) index_exists = - ActiveRecord::Base.connection.index_exists?( + relation_class.connection.index_exists?( relation_class.table_name, importable_foreign_key, unique: true) diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb new file mode 100644 index 00000000000..d0fae2cbb95 --- /dev/null +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +# RelationObjectSaver allows for an alternative approach to persisting +# objects during Project/Group Import which persists object's +# nested collection subrelations separately, in batches. +# +# Instead of the regular `relation_object.save!` that opens one db +# transaction for the object itself and all of its subrelations we +# separate collection subrelations from the object and save them +# in batches in smaller more frequent db transactions. +module Gitlab + module ImportExport + module Base + class RelationObjectSaver + include Gitlab::Utils::StrongMemoize + + BATCH_SIZE = 100 + MIN_RECORDS_SIZE = 5 + + # @param relation_object [Object] Object of a project/group, e.g. an issue + # @param relation_key [String] Name of the object association to group/project, e.g. :issues + # @param relation_definition [Hash] Object subrelations as defined in import_export.yml + # @param importable [Project|Group] Project or group where relation object is getting saved to + # + # @example + # Gitlab::ImportExport::Base::RelationObjectSaver.new( + # relation_key: 'merge_requests', + # relation_object: #<MergeRequest id: root/mrs!1, notes: [#<Note id: nil, note: 'test', ...>, #<Note id: nil, noteL 'another note'>]>, + # relation_definition: {"metrics"=>{}, "award_emoji"=>{}, "notes"=>{"author"=>{}, ... }} + # importable: @importable + # ).execute + def initialize(relation_object:, relation_key:, relation_definition:, importable:) + @relation_object = relation_object + @relation_key = relation_key + @relation_definition = relation_definition + @importable = importable + @invalid_subrelations = [] + end + + def execute + move_subrelations + + relation_object.save! + + save_subrelations + ensure + log_invalid_subrelations + end + + private + + attr_reader :relation_object, :relation_key, :relation_definition, + :importable, :collection_subrelations, :invalid_subrelations + + # rubocop:disable GitlabSecurity/PublicSend + def save_subrelations + collection_subrelations.each_pair do |relation_name, records| + records.each_slice(BATCH_SIZE) do |batch| + valid_records, invalid_records = batch.partition { |record| record.valid? } + + invalid_subrelations << invalid_records + relation_object.public_send(relation_name) << valid_records + end + end + end + + def move_subrelations + strong_memoize(:collection_subrelations) do + relation_definition.each_key.each_with_object({}) do |definition, collection_subrelations| + subrelation = relation_object.public_send(definition) + association = relation_object.class.reflect_on_association(definition) + + if association&.collection? && subrelation.size > MIN_RECORDS_SIZE + collection_subrelations[definition] = subrelation.records + + subrelation.clear + end + end + end + end + # rubocop:enable GitlabSecurity/PublicSend + + def log_invalid_subrelations + invalid_subrelations.flatten.each do |record| + Gitlab::Import::Logger.info( + message: '[Project/Group Import] Invalid subrelation', + importable_column_name => importable.id, + relation_key: relation_key, + error_messages: record.errors.full_messages.to_sentence + ) + + ImportFailure.create( + source: 'RelationObjectSaver#save!', + relation_key: relation_key, + exception_class: 'RecordInvalid', + exception_message: record.errors.full_messages.to_sentence, + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + importable_column_name => importable.id + ) + end + end + + def importable_column_name + @column_name ||= importable.class.reflect_on_association(:import_failures).foreign_key.to_sym + end + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e520cade517..2b0467d8779 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -6,6 +6,8 @@ module Gitlab UNTAR_MASK = 'u+rwX,go+rX,go-w' DEFAULT_DIR_MODE = 0700 + FileOversizedError = Class.new(StandardError) + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -51,19 +53,34 @@ module Gitlab private - def download_or_copy_upload(uploader, upload_path) + def download_or_copy_upload(uploader, upload_path, size_limit: nil) if uploader.upload.local? copy_files(uploader.path, upload_path) else - download(uploader.url, upload_path) + download(uploader.url, upload_path, size_limit: size_limit) end end - def download(url, upload_path) - File.open(upload_path, 'w') do |file| - # Download (stream) file from the uploader's location - IO.copy_stream(URI.parse(url).open, file) + def download(url, upload_path, size_limit: nil) + File.open(upload_path, 'wb') do |file| + current_size = 0 + + Gitlab::HTTP.get(url, stream_body: true, allow_object_storage: true) do |fragment| + if [301, 302, 307].include?(fragment.code) + Gitlab::Import::Logger.warn(message: "received redirect fragment", fragment_code: fragment.code) + elsif fragment.code == 200 + current_size += fragment.bytesize + + raise FileOversizedError if size_limit.present? && current_size > size_limit + + file.write(fragment) + else + raise Gitlab::ImportExport::Error, "unsupported response downloading fragment #{fragment.code}" + end + end end + rescue FileOversizedError + nil end def tar_with_options(archive:, dir:, options:) diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 5274fcec43e..829b3771518 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -72,9 +72,17 @@ module Gitlab import_export_upload = @importable.import_export_upload if import_export_upload.remote_import_url.present? - download(import_export_upload.remote_import_url, @archive_file) + download( + import_export_upload.remote_import_url, + @archive_file, + size_limit: ::Import::GitlabProjects::RemoteFileValidator::FILE_SIZE_LIMIT + ) else - download_or_copy_upload(import_export_upload.import_file, @archive_file) + download_or_copy_upload( + import_export_upload.import_file, + @archive_file, + size_limit: ::Import::GitlabProjects::RemoteFileValidator::FILE_SIZE_LIMIT + ) end end diff --git a/lib/gitlab/import_export/group/object_builder.rb b/lib/gitlab/import_export/group/object_builder.rb index 43cc7a78a61..e26f37c3347 100644 --- a/lib/gitlab/import_export/group/object_builder.rb +++ b/lib/gitlab/import_export/group/object_builder.rb @@ -13,21 +13,12 @@ module Gitlab super @group = @attributes['group'] - - update_description end private attr_reader :group - # Convert description empty string to nil - # due to existing object being saved with description: nil - # Which makes object lookup to fail since nil != '' - def update_description - attributes['description'] = nil if attributes['description'] == '' - end - def where_clauses [ where_clause_base, diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index c2cbd2fdf47..b44874f598c 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -29,7 +29,7 @@ module Gitlab end def restore - ActiveRecord::Base.uncached do + Gitlab::Database.all_uncached do ActiveRecord::Base.no_touching do update_params! @@ -79,10 +79,7 @@ module Gitlab relation_object.assign_attributes(importable_class_sym => @importable) - import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do - relation_object.save! - log_relation_creation(@importable, relation_key, relation_object) - end + save_relation_object(relation_object, relation_key, relation_definition, relation_index) rescue StandardError => e import_failure_service.log_import_failure( source: 'process_relation_item!', @@ -91,6 +88,23 @@ module Gitlab exception: e) end + def save_relation_object(relation_object, relation_key, relation_definition, relation_index) + if Feature.enabled?(:import_relation_object_persistence, default_enabled: :yaml) && relation_object.new_record? + Gitlab::ImportExport::Base::RelationObjectSaver.new( + relation_object: relation_object, + relation_key: relation_key, + relation_definition: relation_definition, + importable: @importable + ).execute + else + import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do + relation_object.save! + end + end + + log_relation_creation(@importable, relation_key, relation_object) + end + def import_failure_service @import_failure_service ||= ImportFailureService.new(@importable) end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index d893c8dfaa3..55b8c1d4531 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -166,8 +166,6 @@ module Gitlab end def read_from_replica_if_available(&block) - return yield unless ::Feature.enabled?(:load_balancing_for_export_workers, type: :development, default_enabled: :yaml) - ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block) end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 059f6bd42e3..fc05cc1a79c 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -370,6 +370,7 @@ included_attributes: - :name - :email events: + - :project_id - :target_type - :action - :author_id diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb index 7b1cf5e7931..ef342f3819f 100644 --- a/lib/gitlab/insecure_key_fingerprint.rb +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -10,6 +10,7 @@ module Gitlab # class InsecureKeyFingerprint attr_accessor :key + alias_attribute :fingerprint_md5, :fingerprint # diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index 1350d75b216..82c2b3297c1 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -5,7 +5,7 @@ module Gitlab class StiType < ActiveRecord::Type::String NAMESPACED_INTEGRATIONS = Set.new(%w( Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog - Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost + Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao )).freeze diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 368b621bdfb..9824b46554f 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -16,6 +16,9 @@ module Gitlab # @return [Boolean, String, Array, Hash] # @raise [JSON::ParserError] raised if parsing fails def parse(string, opts = {}) + # Parse nil as nil + return if string.nil? + # First we should ensure this really is a string, not some other # type which purports to be a string. This handles some legacy # usage of the JSON class. @@ -30,6 +33,7 @@ module Gitlab end alias_method :parse!, :parse + alias_method :load, :parse # Restricted method for converting a Ruby object to JSON. If you # need to pass options to this, you should use `.generate` instead, @@ -67,6 +71,14 @@ module Gitlab ::JSON.pretty_generate(object, opts) end + # The standard parser error we should be returning. Defined in a method + # so we can potentially override it later. + # + # @return [JSON::ParserError] + def parser_error + ::JSON::ParserError + end + private # Convert JSON string into Ruby through toggleable adapters. @@ -134,14 +146,6 @@ module Gitlab opts end - # The standard parser error we should be returning. Defined in a method - # so we can potentially override it later. - # - # @return [JSON::ParserError] - def parser_error - ::JSON::ParserError - end - # @param [Nil, Boolean] an extracted :legacy_mode key from the opts hash # @return [Boolean] def legacy_mode_enabled?(arg_value) diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 41c18f82a4b..d5c018cfc68 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -2,12 +2,17 @@ module Gitlab class JsonCache - attr_reader :backend, :cache_key_with_version, :namespace + attr_reader :backend, :namespace + + STRATEGY_KEY_COMPONENTS = { + revision: Gitlab.revision, + version: [Gitlab::VERSION, Rails.version] + }.freeze def initialize(options = {}) @backend = options.fetch(:backend, Rails.cache) @namespace = options.fetch(:namespace, nil) - @cache_key_with_version = options.fetch(:cache_key_with_version, true) + @cache_key_strategy = options.fetch(:cache_key_strategy, :revision) end def active? @@ -19,13 +24,12 @@ module Gitlab end def cache_key(key) - expanded_cache_key = [namespace, key].compact - - if cache_key_with_version - expanded_cache_key << [Gitlab::VERSION, Rails.version] - end + expanded_cache_key = [namespace, key, *strategy_key_component].compact + expanded_cache_key.join(':').freeze + end - expanded_cache_key.flatten.join(':').freeze + def strategy_key_component + STRATEGY_KEY_COMPONENTS.fetch(@cache_key_strategy) end def expire(key) @@ -39,7 +43,9 @@ module Gitlab end def write(key, value, options = nil) - backend.write(cache_key(key), value.to_json, options) + # As we use json as the serialization format, return everything from + # ActiveModel objects included encrypted values. + backend.write(cache_key(key), value.to_json(unsafe_serialization_hash: true), options) end def fetch(key, options = {}, &block) diff --git a/lib/gitlab/kubernetes/kubeconfig/template.rb b/lib/gitlab/kubernetes/kubeconfig/template.rb index da0861ee86a..d40b9ce117e 100644 --- a/lib/gitlab/kubernetes/kubeconfig/template.rb +++ b/lib/gitlab/kubernetes/kubeconfig/template.rb @@ -14,6 +14,7 @@ module Gitlab @clusters = [] @users = [] @contexts = [] + @current_context = nil end def valid? @@ -32,14 +33,45 @@ module Gitlab contexts << new_entry(:context, **args) end + def merge_yaml(kubeconfig_yaml) + return unless kubeconfig_yaml + + kubeconfig_yaml = YAML.safe_load(kubeconfig_yaml, symbolize_names: true) + kubeconfig_yaml[:users].each do |user| + add_user( + name: user[:name], + token: user.dig(:user, :token) + ) + end + kubeconfig_yaml[:clusters].each do |cluster| + ca_pem = cluster.dig(:cluster, :'certificate-authority-data')&.yield_self do |data| + Base64.strict_decode64(data) + end + + add_cluster( + name: cluster[:name], + url: cluster.dig(:cluster, :server), + ca_pem: ca_pem + ) + end + kubeconfig_yaml[:contexts].each do |context| + add_context( + name: context[:name], + **context[:context]&.slice(:cluster, :user, :namespace) + ) + end + @current_context = kubeconfig_yaml[:'current-context'] + end + def to_h { apiVersion: 'v1', kind: 'Config', clusters: clusters.map(&:to_h), users: users.map(&:to_h), - contexts: contexts.map(&:to_h) - } + contexts: contexts.map(&:to_h), + 'current-context': current_context + }.compact end def to_yaml @@ -48,7 +80,7 @@ module Gitlab private - attr_reader :clusters, :users, :contexts + attr_reader :clusters, :users, :contexts, :current_context def new_entry(entry, **args) ENTRIES.fetch(entry).new(**args) diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb index 6f7fa9fe03b..b259f58350b 100644 --- a/lib/gitlab/language_detection.rb +++ b/lib/gitlab/language_detection.rb @@ -63,7 +63,7 @@ module Gitlab @repository .languages .first(MAX_LANGUAGES) - .to_h { |l| [l[:label], l] } + .index_by { |l| l[:label] } end end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index e93a297cee4..ef5ca56a13b 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -12,6 +12,11 @@ module Gitlab module MailRoom RAILS_ROOT_DIR = Pathname.new('../..').expand_path(__dir__).freeze + DELIVERY_METHOD_SIDEKIQ = 'sidekiq' + DELIVERY_METHOD_WEBHOOK = 'webhook' + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Mailroom-Api-Request' + INTERNAL_API_REQUEST_JWT_ISSUER = 'gitlab-mailroom' + DEFAULT_CONFIG = { enabled: false, port: 143, @@ -20,7 +25,8 @@ module Gitlab mailbox: 'inbox', idle_timeout: 60, log_path: RAILS_ROOT_DIR.join('log', 'mail_room_json.log'), - expunge_deleted: false + expunge_deleted: false, + delivery_method: DELIVERY_METHOD_SIDEKIQ }.freeze # Email specific configuration which is merged with configuration @@ -63,7 +69,9 @@ module Gitlab return {} unless File.exist?(config_file) config = merged_configs(config_key) + config.merge!(redis_config) if enabled?(config) + config[:log_path] = File.expand_path(config[:log_path], RAILS_ROOT_DIR) config diff --git a/lib/gitlab/mail_room/authenticator.rb b/lib/gitlab/mail_room/authenticator.rb index 26ebdca8beb..ca583d4cddb 100644 --- a/lib/gitlab/mail_room/authenticator.rb +++ b/lib/gitlab/mail_room/authenticator.rb @@ -6,8 +6,6 @@ module Gitlab include JwtAuthenticatable SecretConfigurationError = Class.new(StandardError) - INTERNAL_API_REQUEST_HEADER = 'Gitlab-Mailroom-Api-Request' - INTERNAL_API_REQUEST_JWT_ISSUER = 'gitlab-mailroom' # Only allow token generated within the last 5 minutes EXPIRATION = 5.minutes @@ -18,9 +16,10 @@ module Gitlab return false if enabled_configs[mailbox_type].blank? decode_jwt( - request_headers[INTERNAL_API_REQUEST_HEADER], + request_headers[Gitlab::MailRoom::INTERNAL_API_REQUEST_HEADER], secret(mailbox_type), - issuer: INTERNAL_API_REQUEST_JWT_ISSUER, iat_after: Time.current - EXPIRATION + issuer: Gitlab::MailRoom::INTERNAL_API_REQUEST_JWT_ISSUER, + iat_after: Time.current - EXPIRATION ) rescue JWT::DecodeError => e ::Gitlab::AppLogger.warn("Fail to decode MailRoom JWT token: #{e.message}") if Rails.env.development? diff --git a/lib/gitlab/merge_requests/commit_message_generator.rb b/lib/gitlab/merge_requests/commit_message_generator.rb index 0515c17fe5d..ef5c63925c2 100644 --- a/lib/gitlab/merge_requests/commit_message_generator.rb +++ b/lib/gitlab/merge_requests/commit_message_generator.rb @@ -50,6 +50,19 @@ module Gitlab .except(commit_author&.commit_email_or_default) .map { |author_email, author_name| "Co-authored-by: #{author_name} <#{author_email}>" } .join("\n") + end, + 'all_commits' => -> (merge_request, _, _) do + merge_request + .recent_commits + .without_merge_commits + .map do |commit| + if commit.safe_message&.bytesize&.>(100.kilobytes) + "* #{commit.title}\n\n-- Skipped commit body exceeding 100KiB in size." + else + "* #{commit.safe_message&.strip}" + end + end + .join("\n\n") end }.freeze diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb index d0788c7d7c7..5284d20d423 100644 --- a/lib/gitlab/merge_requests/mergeability/check_result.rb +++ b/lib/gitlab/merge_requests/mergeability/check_result.rb @@ -22,8 +22,8 @@ module Gitlab def self.from_hash(data) new( - status: data.fetch(:status), - payload: data.fetch(:payload)) + status: data.fetch('status').to_sym, + payload: data.fetch('payload')) end def initialize(status:, payload: {}) diff --git a/lib/gitlab/merge_requests/mergeability/results_store.rb b/lib/gitlab/merge_requests/mergeability/results_store.rb index bb6489f8526..2f7b8888b2f 100644 --- a/lib/gitlab/merge_requests/mergeability/results_store.rb +++ b/lib/gitlab/merge_requests/mergeability/results_store.rb @@ -9,7 +9,11 @@ module Gitlab end def read(merge_check:) - interface.retrieve_check(merge_check: merge_check) + result_hash = interface.retrieve_check(merge_check: merge_check) + + return if result_hash.blank? + + CheckResult.from_hash(result_hash) end def write(merge_check:, result_hash:) diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb index 2c17982d299..31d75225972 100644 --- a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb @@ -74,7 +74,7 @@ module Gitlab def verify_params raise Errors::DashboardProcessingError, _('Cluster is required for Stages::ClusterEndpointInserter') unless params[:cluster] - raise Errors::DashboardProcessingError, _('Cluster type must be specificed for Stages::ClusterEndpointInserter') unless params[:cluster_type] + raise Errors::DashboardProcessingError, _('Cluster type must be specified for Stages::ClusterEndpointInserter') unless params[:cluster_type] end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 715dd86d93c..12576cabb19 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -134,11 +134,7 @@ module Gitlab :"gitlab_transaction_db_#{counter}_total" end - if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] - current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) - else - current_transaction&.increment(prometheus_key, 1) - end + current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 @@ -154,11 +150,7 @@ module Gitlab def observe(histogram, event, &block) db_config_name = db_config_name(event.payload) - if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] - current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block) - else - current_transaction&.observe(histogram, event.duration / 1000.0, &block) - end + current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block) end def current_transaction @@ -193,11 +185,9 @@ module Gitlab counters << compose_metric_key(metric, role) end - if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] - ::Gitlab::Database.db_config_names.each do |config_name| - counters << compose_metric_key(metric, nil, config_name) # main - counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica - end + ::Gitlab::Database.db_config_names.each do |config_name| + counters << compose_metric_key(metric, nil, config_name) # main + counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica end end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index a9ff186c7cb..f4984e11c14 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -3,6 +3,7 @@ module Gitlab class OmniauthInitializer OAUTH2_TIMEOUT_SECONDS = 10 + ConfigurationError = Class.new(StandardError) def initialize(devise_config) @devise_config = devise_config @@ -75,16 +76,29 @@ module Gitlab provider_arguments << provider[argument] if provider[argument] end - case provider['args'] + arguments = provider.fetch('args', {}) + defaults = provider_defaults(provider) + + case arguments when Array - # An Array from the configuration will be expanded. - provider_arguments.concat provider['args'] + # An Array from the configuration will be expanded + provider_arguments.concat arguments + provider_arguments << defaults unless defaults.empty? when Hash - defaults = provider_defaults(provider) - hash_arguments = provider['args'].deep_symbolize_keys.deep_merge(defaults) + hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults) + normalized = normalize_hash_arguments(hash_arguments) # A Hash from the configuration will be passed as is. - provider_arguments << normalize_hash_arguments(hash_arguments) + provider_arguments << normalized unless normalized.empty? + else + # this will prevent the application from starting in development mode. + # we still set defaults, and let the application start in prod. + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + ConfigurationError.new("Arguments were provided for #{provider['name']}, but not as an array or a hash"), + provider_name: provider['name'], + arguments_type: arguments.class.name + ) + provider_arguments << defaults unless defaults.empty? end provider_arguments diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb index b35683c9dec..7ada15cfe9a 100644 --- a/lib/gitlab/pages/settings.rb +++ b/lib/gitlab/pages/settings.rb @@ -16,7 +16,7 @@ module Gitlab def disk_access_denied? return true unless ::Settings.pages.local_store&.enabled - ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite? + ::Gitlab::Runtime.puma? && !::Gitlab::Runtime.test_suite? end def report_denied_disk_access diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index 99a3145104a..e76cab688cc 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -4,6 +4,7 @@ module Gitlab module Pagination class GitalyKeysetPager attr_reader :request_context, :project + delegate :params, to: :request_context def initialize(request_context, project) diff --git a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb index 18390f5b59d..e06d7e48ca3 100644 --- a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb +++ b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb @@ -6,6 +6,7 @@ module Gitlab class CursorBasedRequestContext DEFAULT_SORT_DIRECTION = :desc attr_reader :request_context + delegate :params, to: :request_context def initialize(request_context) diff --git a/lib/gitlab/pagination/keyset/header_builder.rb b/lib/gitlab/pagination/keyset/header_builder.rb index 888d93d5fe3..1036916e665 100644 --- a/lib/gitlab/pagination/keyset/header_builder.rb +++ b/lib/gitlab/pagination/keyset/header_builder.rb @@ -5,6 +5,7 @@ module Gitlab module Keyset class HeaderBuilder attr_reader :request_context + delegate :params, :header, :request, to: :request_context def initialize(request_context) diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 4f8a6ffb2cc..fca75d1fe01 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -4,6 +4,7 @@ module Gitlab module Pagination class OffsetPagination < Base attr_reader :request_context + delegate :params, :header, :request, to: :request_context def initialize(request_context) @@ -26,7 +27,7 @@ module Gitlab end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) + return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops, default_enabled: :yaml) limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT diff --git a/lib/gitlab/patch/action_cable_redis_listener.rb b/lib/gitlab/patch/action_cable_redis_listener.rb new file mode 100644 index 00000000000..b21bee45991 --- /dev/null +++ b/lib/gitlab/patch/action_cable_redis_listener.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Modifies https://github.com/rails/rails/blob/v6.1.4.6/actioncable/lib/action_cable/subscription_adapter/redis.rb +# so that it is resilient to Redis connection errors. +# See https://github.com/rails/rails/issues/27659. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Patch + module ActionCableRedisListener + private + + def ensure_listener_running + @thread ||= Thread.new do + Thread.current.abort_on_exception = true + + conn = @adapter.redis_connection_for_subscriptions + listen conn + rescue ::Redis::BaseConnectionError + @thread = @raw_client = nil + ::ActionCable.server.restart + end + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 06a26c4830f..6f497c6376d 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -255,7 +255,7 @@ module Gitlab end def container_image_regex - @container_image_regex ||= %r{([\w\.-]+\/){0,1}[\w\.-]+}.freeze + @container_image_regex ||= %r{([\w\.-]+\/){0,4}[\w\.-]+}.freeze end def container_image_blob_sha_regex diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index cf524e69454..8743772eef6 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -25,8 +25,8 @@ module Gitlab log_queries(id, data, 'active-record') log_queries(id, data, 'gitaly') log_queries(id, data, 'redis') - rescue StandardError => err - logger.error(message: "failed to process request id #{id}: #{err.message}") + rescue StandardError => e + logger.error(message: "failed to process request id #{id}: #{e.message}") end private @@ -34,6 +34,8 @@ module Gitlab def request(id) # Peek gem stores request data under peek:requests:request_id key json_data = @redis.get("peek:requests:#{id}") + raise "No data for #{id}" if json_data.nil? + Gitlab::Json.parse(json_data) end diff --git a/lib/gitlab/process_supervisor.rb b/lib/gitlab/process_supervisor.rb new file mode 100644 index 00000000000..18fd24aa582 --- /dev/null +++ b/lib/gitlab/process_supervisor.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module Gitlab + # Given a set of process IDs, the supervisor can monitor processes + # for being alive and invoke a callback if some or all should go away. + # The receiver of the callback can then act on this event, for instance + # by restarting those processes or performing clean-up work. + # + # The supervisor will also trap termination signals if provided and + # propagate those to the supervised processes. Any supervised processes + # that do not terminate within a specified grace period will be killed. + class ProcessSupervisor < Gitlab::Daemon + DEFAULT_HEALTH_CHECK_INTERVAL_SECONDS = 5 + DEFAULT_TERMINATE_INTERVAL_SECONDS = 1 + DEFAULT_TERMINATE_TIMEOUT_SECONDS = 10 + + attr_reader :alive + + def initialize( + health_check_interval_seconds: DEFAULT_HEALTH_CHECK_INTERVAL_SECONDS, + check_terminate_interval_seconds: DEFAULT_TERMINATE_INTERVAL_SECONDS, + terminate_timeout_seconds: DEFAULT_TERMINATE_TIMEOUT_SECONDS, + term_signals: %i(INT TERM), + forwarded_signals: [], + **options) + super(**options) + + @term_signals = term_signals + @forwarded_signals = forwarded_signals + @health_check_interval_seconds = health_check_interval_seconds + @check_terminate_interval_seconds = check_terminate_interval_seconds + @terminate_timeout_seconds = terminate_timeout_seconds + + @pids = [] + @alive = false + end + + # Starts a supervision loop for the given process ID(s). + # + # If any or all processes go away, the IDs of any dead processes will + # be yielded to the given block, so callers can act on them. + # + # If the block returns a non-empty list of IDs, the supervisor will + # start observing those processes instead. Otherwise it will shut down. + def supervise(pid_or_pids, &on_process_death) + @pids = Array(pid_or_pids) + @on_process_death = on_process_death + + trap_signals! + + start + end + + # Shuts down the supervisor and all supervised processes with the given signal. + def shutdown(signal = :TERM) + return unless @alive + + stop_processes(signal) + stop + end + + def supervised_pids + @pids + end + + private + + def start_working + @alive = true + end + + def stop_working + @alive = false + end + + def run_thread + while @alive + sleep(@health_check_interval_seconds) + + check_process_health + end + end + + def check_process_health + unless all_alive? + existing_pids = live_pids # Capture this value for the duration of the block. + dead_pids = @pids - existing_pids + new_pids = Array(@on_process_death.call(dead_pids)) + @pids = existing_pids + new_pids + @alive = @pids.any? + end + end + + def stop_processes(signal) + # Set this prior to shutting down so that shutdown hooks which read `alive` + # know the supervisor is about to shut down. + @alive = false + + # Shut down supervised processes. + signal_all(signal) + wait_for_termination + end + + def trap_signals! + ProcessManagement.trap_signals(@term_signals) do |signal| + stop_processes(signal) + end + + ProcessManagement.trap_signals(@forwarded_signals) do |signal| + signal_all(signal) + end + end + + def wait_for_termination + deadline = monotonic_time + @terminate_timeout_seconds + sleep(@check_terminate_interval_seconds) while continue_waiting?(deadline) + + hard_stop_stuck_pids + end + + def monotonic_time + Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) + end + + def continue_waiting?(deadline) + any_alive? && monotonic_time < deadline + end + + def signal_all(signal) + ProcessManagement.signal_processes(@pids, signal) + end + + def hard_stop_stuck_pids + ProcessManagement.signal_processes(live_pids, "-KILL") + end + + def any_alive? + ProcessManagement.any_alive?(@pids) + end + + def all_alive? + ProcessManagement.all_alive?(@pids) + end + + def live_pids + ProcessManagement.pids_alive(@pids) + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index b2179d80a18..3a5f1a1d480 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -28,7 +28,7 @@ module Gitlab ].freeze # Takes a URL to profile (can be a fully-qualified URL, or an absolute path) - # and returns the ruby-prof profile result. Formatting that result is the + # and returns the profiler result. Formatting that result is the # caller's responsibility. Requests are GET requests unless post_data is # passed. # @@ -43,7 +43,13 @@ module Gitlab # # - private_token: instead of providing a user instance, the token can be # given as a string. Takes precedence over the user option. - def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil) + # + # - sampling_mode: When true, uses a sampling profiler (StackProf) instead of a tracing profiler (RubyProf). + # + # - profiler_options: A keyword Hash of arguments passed to the profiler. Defaults by profiler type: + # RubyProf - {} + # StackProf - { mode: :wall, out: <some temporary file>, interval: 1000, raw: true } + def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil, sampling_mode: false, profiler_options: {}) app = ActionDispatch::Integration::Session.new(Rails.application) verb = :get headers = {} @@ -75,7 +81,9 @@ module Gitlab with_custom_logger(logger) do with_user(user) do - RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend + with_profiler(sampling_mode, profiler_options) do + app.public_send(verb, url, params: post_data, headers: headers) # rubocop:disable GitlabSecurity/PublicSend + end end end end @@ -172,5 +180,16 @@ module Gitlab RubyProf::FlatPrinter.new(result).print($stdout, default_options.merge(options)) end + + def self.with_profiler(sampling_mode, profiler_options) + if sampling_mode + require 'stackprof' + args = { mode: :wall, interval: 1000, raw: true }.merge!(profiler_options) + args[:out] ||= ::Tempfile.new(["profile-#{Time.now.to_i}-", ".dump"]).path + ::StackProf.run(**args) { yield } + else + RubyProf.profile(**profiler_options) { yield } + end + end end end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index 121626ced56..1d7b179baf0 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -22,7 +22,7 @@ module Gitlab user.projects_with_active_memberships.select_for_project_authorization, # The personal projects of the user. - user.personal_projects.select_as_maintainer_for_project_authorization, + user.personal_projects.select_project_owner_for_project_authorization, # Projects that belong directly to any of the groups the user has # access to. diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index 9ff414d5236..eabac6128b5 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -5,6 +5,7 @@ module Gitlab module Queries class BaseQuery attr_accessor :client + delegate :query_range, :query, :label_values, :series, to: :client, prefix: true def raw_memory_usage_query(environment_slug) diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index b44b47eca37..2f89774a257 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -291,7 +291,7 @@ module Gitlab types Issue condition do current_user.can?(:set_issue_crm_contacts, quick_action_target) && - CustomerRelations::Contact.exists_for_group?(quick_action_target.project.group) + CustomerRelations::Contact.exists_for_group?(quick_action_target.project.root_ancestor) end execution_message do _('One or more contacts were successfully added.') @@ -306,7 +306,7 @@ module Gitlab types Issue condition do current_user.can?(:set_issue_crm_contacts, quick_action_target) && - CustomerRelations::Contact.exists_for_group?(quick_action_target.project.group) + CustomerRelations::Contact.exists_for_group?(quick_action_target.project.root_ancestor) end execution_message do _('One or more contacts were successfully removed.') diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 842d4ef482b..e6a73c71e85 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -23,7 +23,11 @@ module Gitlab end end execution_message do - if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + if params[:merge_request_diff_head_sha].blank? + _("Merge request diff sha parameter is required for the merge quick action.") + elsif params[:merge_request_diff_head_sha] != quick_action_target.diff_head_sha + _("Branch has been updated since the merge was requested.") + elsif preferred_strategy = preferred_auto_merge_strategy(quick_action_target) _("Scheduled to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } else _('Merged this merge request.') @@ -35,6 +39,10 @@ module Gitlab merge_orchestration_service.can_merge?(quick_action_target) end command :merge do + next unless params[:merge_request_diff_head_sha].present? + + next unless params[:merge_request_diff_head_sha] == quick_action_target.diff_head_sha + @updates[:merge] = params[:merge_request_diff_head_sha] end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index a6491d23bf5..c9202c6c54c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -259,6 +259,15 @@ module Gitlab "It must start with a letter, digit, emoji, or '_'." end + # Project path must conform to this regex. See https://gitlab.com/gitlab-org/gitlab/-/issues/27483 + def oci_repository_path_regex + @oci_repository_path_regex ||= %r{\A[a-zA-Z0-9]+([._-][a-zA-Z0-9]+)*\z}.freeze + end + + def oci_repository_path_regex_message + 'must not start or end with a special character and must not contain consecutive special characters.' + end + def group_name_regex @group_name_regex ||= /\A#{group_name_regex_chars}\z/.freeze end @@ -459,6 +468,15 @@ module Gitlab "can contain only lowercase letters, digits, '_' and '-'. " \ "Must start with a letter, and cannot end with '-' or '_'" end + + def saved_reply_name_regex + @saved_reply_name_regex ||= /\A[a-z]([a-z0-9\-_]*[a-z0-9])?\z/.freeze + end + + def saved_reply_name_regex_message + "can contain only lowercase letters, digits, '_' and '-'. " \ + "Must start with a letter, and cannot end with '-' or '_'" + end end end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 574e05658bc..5b1341207fd 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -65,12 +65,15 @@ module Gitlab !!defined?(::Rails::Command::RunnerCommand) end - def web_server? - puma? + # Whether we are executing in an actual application context i.e. Puma or Sidekiq. + def application? + puma? || sidekiq? end + # Whether we are executing in a multi-threaded environment. For now this is equivalent + # to meaning Puma or Sidekiq, but this could change in the future. def multi_threaded? - puma? || sidekiq? + application? end def puma_in_clustered_mode? @@ -94,7 +97,7 @@ module Gitlab threads += Sidekiq.options[:concurrency] + 2 end - if web_server? + if puma? threads += Gitlab::ActionCable::Config.worker_pool_size end diff --git a/lib/gitlab/safe_request_loader.rb b/lib/gitlab/safe_request_loader.rb new file mode 100644 index 00000000000..89eca16c272 --- /dev/null +++ b/lib/gitlab/safe_request_loader.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + class SafeRequestLoader + def self.execute(args, &block) + new(**args).execute(&block) + end + + def initialize(resource_key:, resource_ids:, default_value: nil) + @resource_key = resource_key + @resource_ids = resource_ids.uniq + @default_value = default_value + @resource_data = {} + end + + def execute(&block) + raise ArgumentError, 'Block is mandatory' unless block_given? + + load_resource_data + remove_loaded_resource_ids + + update_resource_data(&block) + + resource_data + end + + private + + attr_reader :resource_key, :resource_ids, :default_value, :resource_data, :missing_resource_ids + + def load_resource_data + @resource_data = Gitlab::SafeRequestStore.fetch(resource_key) { resource_data } + end + + def remove_loaded_resource_ids + # Look up only the IDs we need + @missing_resource_ids = resource_ids - resource_data.keys + end + + def update_resource_data(&block) + return if missing_resource_ids.blank? + + reloaded_resource_data = yield(missing_resource_ids) + + @resource_data.merge!(reloaded_resource_data) + + mark_absent_values + end + + def mark_absent_values + absent = (missing_resource_ids - resource_data.keys).to_h { [_1, default_value] } + @resource_data.merge!(absent) + end + end +end diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index f607aff9d29..e302729df66 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -97,6 +97,28 @@ module Gitlab end end + def clean_existing_path(src_path, dry_run: false, content: nil, skip_unallowed_types: false) + content ||= File.read(src_path) + + if skip_unallowed_types + return unless check_for_allowed_types(content, raise_error: false) + else + check_for_allowed_types(content) + end + + to_remove = extra_tags(src_path) + + if to_remove.empty? + logger.info "#{src_path}: only whitelisted tags present, skipping" + return + end + + logger.info "#{src_path}: found exif tags to remove: #{to_remove}" + return if dry_run + + exec_remove_exif!(src_path) + end + private def extra_tags(path) @@ -146,12 +168,15 @@ module Gitlab filename end - def check_for_allowed_types(contents) + def check_for_allowed_types(contents, raise_error: true) mime_type = Gitlab::Utils::MimeType.from_string(contents) - unless ALLOWED_MIME_TYPES.include?(mime_type) + allowed = ALLOWED_MIME_TYPES.include?(mime_type) + if !allowed && raise_error raise "File type #{mime_type} not supported. Only supports #{ALLOWED_MIME_TYPES.join(", ")}." end + + allowed end def upload_ref(upload) diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 5671fce481f..e2df60c46f1 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -62,10 +62,6 @@ module Gitlab end def self.quiet - # Disable database insertion logs so speed isn't limited by ability to print to console - old_logger = ActiveRecord::Base.logger - ActiveRecord::Base.logger = nil - # Additional seed logic for models. Project.include(ProjectSeed) User.include(UserSeed) @@ -75,9 +71,11 @@ module Gitlab SeedFu.quiet = true - without_statement_timeout do - without_new_note_notifications do - yield + without_database_logging do + without_statement_timeout do + without_new_note_notifications do + yield + end end end @@ -85,7 +83,6 @@ module Gitlab ensure SeedFu.quiet = false ActionMailer::Base.perform_deliveries = old_perform_deliveries - ActiveRecord::Base.logger = old_logger end def self.without_gitaly_timeout @@ -112,10 +109,30 @@ module Gitlab end def self.without_statement_timeout - ActiveRecord::Base.connection.execute('SET statement_timeout=0') + Gitlab::Database::EachDatabase.each_database_connection do |connection| + connection.execute('SET statement_timeout=0') + end + yield + ensure + Gitlab::Database::EachDatabase.each_database_connection do |connection| + connection.execute('RESET statement_timeout') + end + end + + def self.without_database_logging + old_loggers = Gitlab::Database.database_base_models.transform_values do |model| + model.logger + end + + Gitlab::Database.database_base_models.each do |_, model| + model.logger = nil + end + yield ensure - ActiveRecord::Base.connection.execute('RESET statement_timeout') + Gitlab::Database.database_base_models.each do |connection_name, model| + model.logger = old_loggers[connection_name] + end end end end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 69802fd6217..fd3a5f715e8 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -33,7 +33,7 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::BatchLoader chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server - chain.add ::Gitlab::SidekiqMiddleware::QueryAnalyzer if Gitlab.dev_or_test_env? || Gitlab::Utils.to_boolean(ENV['GITLAB_ENABLE_QUERY_ANALYZERS'], default: false) + chain.add ::Gitlab::SidekiqMiddleware::QueryAnalyzer chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index f31262bfcc9..601c8d1c3cf 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -167,7 +167,6 @@ module Gitlab def idempotent? return false unless worker_klass return false unless worker_klass.respond_to?(:idempotent?) - return false unless preserve_wal_location? || !worker_klass.utilizes_load_balancing_capabilities? worker_klass.idempotent? end @@ -206,8 +205,6 @@ module Gitlab end def job_wal_locations - return {} unless preserve_wal_location? - job['wal_locations'] || {} end @@ -272,10 +269,6 @@ module Gitlab @existing_wal_locations ||= {} end - def preserve_wal_location? - Feature.enabled?(:preserve_latest_wal_locations_for_idempotent_jobs, default_enabled: :yaml) - end - def reschedulable? !scheduled? && options[:if_deduplicated] == :reschedule_once end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index bea98403997..f3e1d0af2aa 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -7,18 +7,26 @@ module Gitlab # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. - SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + # This information is better viewed in logs, but these buckets cover + # most of the durations for cpu, gitaly, db and elasticsearch + SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.5, 1, 2.5].freeze + + # These are the buckets we currently use for alerting, we will likely + # replace these histograms with Application SLIs + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1313 + SIDEKIQ_JOB_DURATION_BUCKETS = [10, 300].freeze + SIDEKIQ_QUEUE_DURATION_BUCKETS = [10, 60].freeze class << self include ::Gitlab::SidekiqMiddleware::MetricsHelper def metrics { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS), sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS), sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index 3de6c8df8aa..acc3e1712ab 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -29,7 +29,12 @@ module Gitlab # The worker classes aren't constants here, because that would force # Application Settings to be loaded earlier causing failures loading # the environment in rake tasks - EXEMPT_WORKER_NAMES = %w[BackgroundMigrationWorker BackgroundMigration::CiDatabaseWorker Database::BatchedBackgroundMigrationWorker].to_set + + EXEMPT_WORKER_NAMES = %w[BackgroundMigrationWorker + BackgroundMigration::CiDatabaseWorker + Database::BatchedBackgroundMigrationWorker + Database::BatchedBackgroundMigration::CiDatabaseWorker].to_set + JOB_STATUS_KEY = 'size_limiter' class << self diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 09236a7f1f0..c0730e7bd59 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -61,6 +61,16 @@ module Gitlab def self.with_fallback(pattern, multiline: false) UntrustedRegexp.new(pattern, multiline: multiline) rescue RegexpError + raise if Feature.enabled?(:disable_unsafe_regexp, default_enabled: :yaml) + + if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops, default_enabled: :yaml) + Gitlab::AppJsonLogger.info( + class: self.name, + regexp: pattern.to_s, + fabricated: 'unsafe ruby regexp' + ) + end + Regexp.new(pattern) end diff --git a/lib/gitlab/untrusted_regexp/ruby_syntax.rb b/lib/gitlab/untrusted_regexp/ruby_syntax.rb index 5176a6f6273..1f1da592ce0 100644 --- a/lib/gitlab/untrusted_regexp/ruby_syntax.rb +++ b/lib/gitlab/untrusted_regexp/ruby_syntax.rb @@ -16,40 +16,23 @@ module Gitlab # The regexp can match the pattern `/.../`, but may not be fabricatable: # it can be invalid or incomplete: `/match ( string/` - def self.valid?(pattern, fallback: false) - !!self.fabricate(pattern, fallback: fallback) + def self.valid?(pattern) + !!self.fabricate(pattern) end - def self.fabricate(pattern, fallback: false, project: nil) - self.fabricate!(pattern, fallback: fallback, project: project) + def self.fabricate(pattern, project: nil) + self.fabricate!(pattern, project: project) rescue RegexpError nil end - def self.fabricate!(pattern, fallback: false, project: nil) + def self.fabricate!(pattern, project: nil) raise RegexpError, 'Pattern is not string!' unless pattern.is_a?(String) matches = pattern.match(PATTERN) raise RegexpError, 'Invalid regular expression!' if matches.nil? - begin - create_untrusted_regexp(matches[:regexp], matches[:flags]) - rescue RegexpError - raise unless fallback && - Feature.enabled?(:allow_unsafe_ruby_regexp, default_enabled: :yaml) - - if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops, default_enabled: :yaml) - Gitlab::AppJsonLogger.info( - class: self.name, - regexp: pattern.to_s, - fabricated: 'unsafe ruby regexp', - project_id: project&.id, - project_path: project&.full_path - ) - end - - create_ruby_regexp(matches[:regexp], matches[:flags]) - end + create_untrusted_regexp(matches[:regexp], matches[:flags]) end def self.create_untrusted_regexp(pattern, flags) @@ -58,15 +41,6 @@ module Gitlab UntrustedRegexp.new(pattern, multiline: false) end private_class_method :create_untrusted_regexp - - def self.create_ruby_regexp(pattern, flags) - options = 0 - options += Regexp::IGNORECASE if flags&.include?('i') - options += Regexp::MULTILINE if flags&.include?('m') - - Regexp.new(pattern, options) - end - private_class_method :create_ruby_regexp end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 48228ede684..fe8c2227659 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -13,6 +13,7 @@ module Gitlab # ports - Raises error if the given URL port does is not between given ports. # allow_localhost - Raises error if URL resolves to a localhost IP address and argument is false. # allow_local_network - Raises error if URL resolves to a link-local address and argument is false. + # allow_object_storage - Avoid raising an error if URL resolves to an object storage endpoint and argument is true. # ascii_only - Raises error if URL has unicode characters and argument is true. # enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true. # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. @@ -25,6 +26,7 @@ module Gitlab schemes: [], allow_localhost: false, allow_local_network: true, + allow_object_storage: false, ascii_only: false, enforce_user: false, enforce_sanitization: false, @@ -58,6 +60,8 @@ module Gitlab # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) + return protected_uri_with_hostname if allow_object_storage && object_storage_endpoint?(uri) + validate_local_request( address_info: address_info, allow_localhost: allow_localhost, @@ -149,6 +153,7 @@ module Gitlab validate_local_network(address_info) validate_link_local(address_info) validate_shared_address(address_info) + validate_limited_broadcast_address(address_info) end end @@ -253,6 +258,17 @@ module Gitlab raise BlockedUrlError, "Requests to the link local network are not allowed" end + # Raises a BlockedUrlError if any IP in `addrs_info` is the limited + # broadcast address. + # https://datatracker.ietf.org/doc/html/rfc919#section-7 + def validate_limited_broadcast_address(addrs_info) + blocked_ips = ["255.255.255.255"] + + return if (blocked_ips & addrs_info.map(&:ip_address)).empty? + + raise BlockedUrlError, "Requests to the limited broadcast address are not allowed" + end + def internal?(uri) internal_web?(uri) || internal_shell?(uri) end @@ -269,6 +285,30 @@ module Gitlab get_port(uri) == config.gitlab_shell.ssh_port end + def enabled_object_storage_endpoints + ObjectStoreSettings::SUPPORTED_TYPES.collect do |type| + section_setting = config.try(type) + + next unless section_setting + + object_store_setting = section_setting['object_store'] + + next unless object_store_setting && object_store_setting['enabled'] + + object_store_setting.dig('connection', 'endpoint') + end.compact.uniq + end + + def object_storage_endpoint?(uri) + enabled_object_storage_endpoints.any? do |endpoint| + endpoint_uri = URI(endpoint) + + uri.scheme == endpoint_uri.scheme && + uri.hostname == endpoint_uri.hostname && + get_port(uri) == get_port(endpoint_uri) + end + end + def domain_allowed?(uri) Gitlab::UrlBlockers::UrlAllowlist.domain_allowed?(uri.normalized_host, port: get_port(uri)) end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 6e5196ecdbd..1031f38792b 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -80,6 +80,10 @@ module Gitlab @all ||= definitions.map { |_key_path, definition| definition } end + def not_removed + all.select { |definition| definition.attributes[:status] != 'removed' }.index_by(&:key_path) + end + def with_instrumentation_class all.select { |definition| definition.attributes[:instrumentation_class].present? && definition.available? } end diff --git a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb new file mode 100644 index 00000000000..6df6fef5d3a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CertBasedClustersFfMetric < GenericMetric + value do + Feature.enabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index d7fc798ebe2..34a8bfd08b5 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -33,6 +33,12 @@ module Gitlab @metric_relation = block end + def metric_options(&block) + return @metric_options&.call.to_h unless block_given? + + @metric_options = block + end + def operation(symbol, column: nil, &block) @metric_operation = symbol @column = column @@ -54,6 +60,7 @@ module Gitlab self.class.column, start: start, finish: finish, + **self.class.metric_options, &self.class.metric_operation_block) end diff --git a/lib/gitlab/usage/service_ping/instrumented_payload.rb b/lib/gitlab/usage/service_ping/instrumented_payload.rb new file mode 100644 index 00000000000..e04e2e589b2 --- /dev/null +++ b/lib/gitlab/usage/service_ping/instrumented_payload.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Service Ping payload build using the instrumentation classes +# for given metrics key_paths and output method +module Gitlab + module Usage + module ServicePing + class InstrumentedPayload + attr_reader :metrics_key_paths + attr_reader :output_method + + def initialize(metrics_key_paths, output_method) + @metrics_key_paths = metrics_key_paths + @output_method = output_method + end + + def build + metrics_key_paths.map do |key_path| + compute_instrumental_value(key_path, output_method) + end.reduce({}, :deep_merge) + end + + private + + # Not all metrics defintions have instrumentation classes + # The value can be computed only for those that have it + def instrumented_metrics_defintions + Gitlab::Usage::MetricDefinition.with_instrumentation_class + end + + def compute_instrumental_value(key_path, output_method) + definition = instrumented_metrics_defintions.find { |df| df.key_path == key_path } + + return {} unless definition.present? + + Gitlab::Usage::Metric.new(definition).method(output_method).call + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping/payload_keys_processor.rb b/lib/gitlab/usage/service_ping/payload_keys_processor.rb new file mode 100644 index 00000000000..ea2043ffb83 --- /dev/null +++ b/lib/gitlab/usage/service_ping/payload_keys_processor.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# Process the UsageData payload to get the keys that have a metric defintion +# Get the missing keys from the payload +module Gitlab + module Usage + module ServicePing + class PayloadKeysProcessor + attr_reader :old_payload + + def initialize(old_payload) + @old_payload = old_payload + end + + def key_paths + @key_paths ||= payload_keys.to_a.flatten.compact + end + + def missing_instrumented_metrics_key_paths + @missing_key_paths ||= metrics_with_instrumentation.map(&:key) - key_paths + end + + private + + def payload_keys(payload = old_payload, parents = []) + return unless payload.is_a?(Hash) + + payload.map do |key, value| + if has_metric_definition?(key, parents) + parents.dup.append(key).join('.') + else + payload_keys(value, parents.dup << key) if value.is_a?(Hash) + end + end + end + + def has_metric_definition?(key, parent_keys) + key_path = parent_keys.dup.append(key).join('.') + metric_definitions.key?(key_path) + end + + def metric_definitions + ::Gitlab::Usage::MetricDefinition.not_removed + end + + def metrics_with_instrumentation + ::Gitlab::Usage::MetricDefinition.with_instrumentation_class + end + end + end + end +end + +Gitlab::Usage::ServicePing::PayloadKeysProcessor.prepend_mod_with('Gitlab::Usage::ServicePing::PayloadKeysProcessor') diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index d9e30c46498..794f3373043 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -7,16 +7,29 @@ module Gitlab def for(output:, cached: false) case output.to_sym when :all_metrics_values - all_metrics_values(cached) + with_instrumentation_classes(all_metrics_values(cached), :with_value) when :metrics_queries - metrics_queries + with_instrumentation_classes(metrics_queries, :with_instrumentation) when :non_sql_metrics_values - non_sql_metrics_values + with_instrumentation_classes(non_sql_metrics_values, :with_instrumentation) end end private + def with_instrumentation_classes(old_payload, output_method) + if Feature.enabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) + + instrumented_metrics_key_paths = Gitlab::Usage::ServicePing::PayloadKeysProcessor.new(old_payload).missing_instrumented_metrics_key_paths + + instrumented_payload = Gitlab::Usage::ServicePing::InstrumentedPayload.new(instrumented_metrics_key_paths, output_method).build + + old_payload.deep_merge(instrumented_payload) + else + old_payload + end + end + def all_metrics_values(cached) Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do Gitlab::UsageData.data diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e66a565246b..951ec5ea5c3 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -70,10 +70,9 @@ module Gitlab def system_usage_data issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) - { + counts = { counts: { assignee_lists: count(List.assignee), - boards: add_metric('CountBoardsMetric', time_frame: 'all'), ci_builds: count(::Ci::Build), ci_internal_pipelines: count(::Ci::Pipeline.internal), ci_external_pipelines: count(::Ci::Pipeline.external), @@ -167,6 +166,12 @@ module Gitlab data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } + + if Feature.disabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) + counts[:counts][:boards] = add_metric('CountBoardsMetric', time_frame: 'all') + end + + counts end # rubocop: enable Metrics/AbcSize @@ -219,7 +224,8 @@ module Gitlab collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none'), service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none'), snowplow_enabled: add_metric('SnowplowEnabledMetric', time_frame: 'none'), - snowplow_configured_to_gitlab_collector: add_metric('SnowplowConfiguredToGitlabCollectorMetric', time_frame: 'none') + snowplow_configured_to_gitlab_collector: add_metric('SnowplowConfiguredToGitlabCollectorMetric', time_frame: 'none'), + certificate_based_clusters_ff: add_metric('CertBasedClustersFfMetric') } } end diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index cecc24a38d5..cdcad8fdc7b 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -16,7 +16,8 @@ module Gitlab DesignsCounter, KubernetesAgentCounter, StaticSiteEditorCounter, - DiffsCounter + DiffsCounter, + ServiceUsageDataCounter ].freeze UsageDataCounterError = Class.new(StandardError) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index c6e9db6a314..474ab9a4dd9 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -26,6 +26,7 @@ module Gitlab ecosystem epic_boards_usage epics_usage + error_tracking ide_edit incident_management issues_edit diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml new file mode 100644 index 00000000000..63498a35858 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -0,0 +1,5 @@ +- name: ci_users_executing_deployment_job + category: ci_users + redis_slot: ci_users + aggregation: weekly + feature_flag: job_deployment_count diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 96755db8439..fdf4bc58525 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -324,7 +324,6 @@ category: snippets redis_slot: snippets aggregation: weekly - feature_flag: usage_data_i_snippets_show # Terraform - name: p_terraform_state_api_unique_users category: terraform diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml new file mode 100644 index 00000000000..a56e0a6d370 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml @@ -0,0 +1,11 @@ +--- +- name: error_tracking_view_details + category: error_tracking + redis_slot: error_tracking + aggregation: weekly + feature_flag: track_error_tracking_activity +- name: error_tracking_view_list + category: error_tracking + redis_slot: error_tracking + aggregation: weekly + feature_flag: track_error_tracking_activity diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 49891080b03..4ba7ea2d407 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -127,6 +127,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_page + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_publish category: quickactions redis_slot: quickactions diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml new file mode 100644 index 00000000000..0c9c6026c46 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -0,0 +1,11 @@ +--- +- name: users_updating_work_item_title + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity +- name: users_creating_work_items + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity diff --git a/lib/gitlab/usage_data_counters/service_usage_data_counter.rb b/lib/gitlab/usage_data_counters/service_usage_data_counter.rb new file mode 100644 index 00000000000..aa1d9583ea5 --- /dev/null +++ b/lib/gitlab/usage_data_counters/service_usage_data_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class ServiceUsageDataCounter < BaseCounter + KNOWN_EVENTS = %w[download_payload_click].freeze + PREFIX = 'service_usage_data' + end +end diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb new file mode 100644 index 00000000000..6f5300405c7 --- /dev/null +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module WorkItemActivityUniqueCounter + WORK_ITEM_CREATED = 'users_creating_work_items' + WORK_ITEM_TITLE_CHANGED = 'users_updating_work_item_title' + + class << self + def track_work_item_created_action(author:) + track_unique_action(WORK_ITEM_CREATED, author) + end + + def track_work_item_title_changed_action(author:) + track_unique_action(WORK_ITEM_TITLE_CHANGED, author) + end + + private + + def track_unique_action(action, author) + return unless author + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index c6490ba7374..d40ac71afc6 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -41,9 +41,19 @@ module Gitlab end def maximum_id(model, column = nil) + # no-op: shadowing super for performance reasons end def minimum_id(model, column = nil) + # no-op: shadowing super for performance reasons + end + + def alt_usage_data(value = nil, fallback: FALLBACK, &block) + if block_given? + { alt_usage_data_block: block.to_s } + else + { alt_usage_data_value: value } + end end def redis_usage_data(counter = nil, &block) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 608545baf74..816ede4136a 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -5,6 +5,10 @@ module Gitlab extend self PathTraversalAttackError ||= Class.new(StandardError) + private_class_method def logger + @logger ||= Gitlab::AppLogger + end + # Ensure that the relative path will not traverse outside the base directory # We url decode the path to avoid passing invalid paths forward in url encoded format. # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 @@ -16,6 +20,7 @@ module Gitlab path_regex = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)} if path.match?(path_regex) + logger.warn(message: "Potential path traversal attempt detected", path: "#{path}") raise PathTraversalAttackError, 'Invalid path' end @@ -37,6 +42,13 @@ module Gitlab raise StandardError, "path #{path} is not allowed" end + def check_allowed_absolute_path_and_path_traversal!(path, path_allowlist) + traversal_path = check_path_traversal!(path) + raise StandardError, "path is not a string!" unless traversal_path.is_a?(String) + + check_allowed_absolute_path!(traversal_path, path_allowlist) + end + def decode_path(encoded_path) decoded = CGI.unescape(encoded_path) if decoded != CGI.unescape(decoded) diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 255fa0169bf..3c954f817a7 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -22,10 +22,12 @@ module Gitlab # end # def strong_memoize(name) - if strong_memoized?(name) - instance_variable_get(ivar(name)) + key = ivar(name) + + if instance_variable_defined?(key) + instance_variable_get(key) else - instance_variable_set(ivar(name), yield) + instance_variable_set(key, yield) end end @@ -34,13 +36,23 @@ module Gitlab end def clear_memoization(name) - remove_instance_variable(ivar(name)) if instance_variable_defined?(ivar(name)) + key = ivar(name) + remove_instance_variable(key) if instance_variable_defined?(key) end private + # Convert `"name"`/`:name` into `:@name` + # + # Depending on a type ensure that there's a single memory allocation def ivar(name) - "@#{name}" + if name.is_a?(Symbol) + name.to_s.prepend("@").to_sym + elsif name.is_a?(String) + :"@#{name}" + else + raise ArgumentError, "Invalid type of '#{name}'" + end end end end diff --git a/lib/gitlab/wiki_pages/front_matter_parser.rb b/lib/gitlab/wiki_pages/front_matter_parser.rb index 0ceec39782c..ee30fa907f4 100644 --- a/lib/gitlab/wiki_pages/front_matter_parser.rb +++ b/lib/gitlab/wiki_pages/front_matter_parser.rb @@ -3,8 +3,6 @@ module Gitlab module WikiPages class FrontMatterParser - FEATURE_FLAG = :wiki_front_matter - # We limit the maximum length of text we are prepared to parse as YAML, to # avoid exploitations and attempts to consume memory and CPU. We allow for: # - a title line @@ -30,18 +28,12 @@ module Gitlab end # @param [String] wiki_content - # @param [FeatureGate] feature_gate The scope for feature availability - def initialize(wiki_content, feature_gate) + def initialize(wiki_content) @wiki_content = wiki_content - @feature_gate = feature_gate - end - - def self.enabled?(gate = nil) - Feature.enabled?(FEATURE_FLAG, gate) end def parse - return empty_result unless enabled? && wiki_content.present? + return empty_result unless wiki_content.present? return empty_result(block.error) unless block.valid? Result.new(front_matter: block.data, content: strip_front_matter_block) @@ -94,22 +86,18 @@ module Gitlab private - attr_reader :wiki_content, :feature_gate + attr_reader :wiki_content def empty_result(reason = nil, error = nil) Result.new(content: wiki_content, reason: reason, error: error) end - def enabled? - self.class.enabled?(feature_gate) - end - def block @block ||= parse_front_matter_block end def parse_front_matter_block - wiki_content.match(Gitlab::FrontMatter::PATTERN) { |m| Block.new(*m.captures) } || Block.new + wiki_content.match(Gitlab::FrontMatter::PATTERN) { |m| Block.new(m[:delim], m[:lang], m[:front_matter]) } || Block.new end def strip_front_matter_block diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 43a2480d5b7..360c9a6c52f 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -22,6 +22,7 @@ module GoogleApi "https://www.googleapis.com/auth/monitoring" ].freeze ROLES_LIST = %w[roles/iam.serviceAccountUser roles/artifactregistry.admin roles/cloudbuild.builds.builder roles/run.admin roles/storage.admin roles/cloudsql.admin roles/browser].freeze + REVOKE_URL = 'https://oauth2.googleapis.com/revoke' class << self def session_key_for_token @@ -146,6 +147,11 @@ module GoogleApi enable_service(gcp_project_id, 'cloudbuild.googleapis.com') end + def revoke_authorizations + uri = URI(REVOKE_URL) + Gitlab::HTTP.post(uri, body: { 'token' => access_token }) + end + private def enable_service(gcp_project_id, service_name) @@ -211,7 +217,7 @@ module GoogleApi end def cloud_resource_manager_service - @gpc_service ||= Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new.tap { |s| s. authorization = access_token } + @gpc_service ||= Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new.tap { |s| s.authorization = access_token } end end end diff --git a/lib/learn_gitlab/onboarding.rb b/lib/learn_gitlab/onboarding.rb index 38ffa9eb2e6..42415aacbee 100644 --- a/lib/learn_gitlab/onboarding.rb +++ b/lib/learn_gitlab/onboarding.rb @@ -5,19 +5,19 @@ module LearnGitlab include Gitlab::Utils::StrongMemoize ACTION_ISSUE_IDS = { - issue_created: 4, - git_write: 6, pipeline_created: 7, - merge_request_created: 9, - user_added: 8, trial_started: 2, required_mr_approvals_enabled: 11, code_owners_enabled: 10 }.freeze - ACTION_DOC_URLS = { - security_scan_enabled: 'https://docs.gitlab.com/ee/user/application_security/security_dashboard/#gitlab-security-dashboard-security-center-and-vulnerability-reports' - }.freeze + ACTION_PATHS = [ + :issue_created, + :git_write, + :merge_request_created, + :user_added, + :security_scan_enabled + ].freeze def initialize(namespace) @namespace = namespace @@ -49,7 +49,7 @@ module LearnGitlab end def tracked_actions - ACTION_ISSUE_IDS.keys + ACTION_DOC_URLS.keys + ACTION_ISSUE_IDS.keys + ACTION_PATHS end attr_reader :namespace diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index 6a41de8f0b0..75346626255 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -75,14 +75,6 @@ module Peek "Role: #{role.to_s.capitalize}" end - - def format_call_details(call) - if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] - super - else - super.except(:db_config_name) - end - end end end end diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb index 4cc2e85c7bb..1301c6aa6fd 100644 --- a/lib/peek/views/detailed_view.rb +++ b/lib/peek/views/detailed_view.rb @@ -23,7 +23,7 @@ module Peek private def duration - detail_store.map { |entry| entry[:duration] }.sum * 1000 + detail_store.sum { |entry| entry[:duration] } * 1000 end def calls diff --git a/lib/security/ci_configuration/base_build_action.rb b/lib/security/ci_configuration/base_build_action.rb index 6012067fb53..8f0765a35c2 100644 --- a/lib/security/ci_configuration/base_build_action.rb +++ b/lib/security/ci_configuration/base_build_action.rb @@ -3,9 +3,10 @@ module Security module CiConfiguration class BaseBuildAction - def initialize(auto_devops_enabled, existing_gitlab_ci_content) + def initialize(auto_devops_enabled, existing_gitlab_ci_content, ci_config_path = ::Ci::Pipeline::DEFAULT_CONFIG_PATH) @auto_devops_enabled = auto_devops_enabled @existing_gitlab_ci_content = existing_gitlab_ci_content || {} + @ci_config_path = ci_config_path.presence || ::Ci::Pipeline::DEFAULT_CONFIG_PATH end def generate @@ -13,7 +14,7 @@ module Security update_existing_content! - { action: action, file_path: '.gitlab-ci.yml', content: prepare_existing_content, default_values_overwritten: @default_values_overwritten } + { action: action, file_path: @ci_config_path, content: prepare_existing_content, default_values_overwritten: @default_values_overwritten } end private diff --git a/lib/security/ci_configuration/sast_build_action.rb b/lib/security/ci_configuration/sast_build_action.rb index 3fa5e9c7177..63f16a1bebe 100644 --- a/lib/security/ci_configuration/sast_build_action.rb +++ b/lib/security/ci_configuration/sast_build_action.rb @@ -3,8 +3,8 @@ module Security module CiConfiguration class SastBuildAction < BaseBuildAction - def initialize(auto_devops_enabled, params, existing_gitlab_ci_content) - super(auto_devops_enabled, existing_gitlab_ci_content) + def initialize(auto_devops_enabled, params, existing_gitlab_ci_content, ci_config_path = ::Ci::Pipeline::DEFAULT_CONFIG_PATH) + super(auto_devops_enabled, existing_gitlab_ci_content, ci_config_path) @variables = variables(params) @default_sast_values = default_sast_values(params) @default_values_overwritten = false diff --git a/lib/serializers/unsafe_json.rb b/lib/serializers/unsafe_json.rb new file mode 100644 index 00000000000..25ec52e4581 --- /dev/null +++ b/lib/serializers/unsafe_json.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Serializers + class UnsafeJson + class << self + def dump(obj) + obj.to_json(unsafe: true) + end + + delegate :load, to: :JSON + end + end +end diff --git a/lib/sidebars/concerns/work_item_hierarchy.rb b/lib/sidebars/concerns/work_item_hierarchy.rb deleted file mode 100644 index a4153bb5120..00000000000 --- a/lib/sidebars/concerns/work_item_hierarchy.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -# This module has the necessary methods to render -# work items hierarchy menu -module Sidebars - module Concerns - module WorkItemHierarchy - def hierarchy_menu_item(container, url, path) - unless show_hierarachy_menu_item?(container) - return ::Sidebars::NilMenuItem.new(item_id: :hierarchy) - end - - ::Sidebars::MenuItem.new( - title: _('Planning hierarchy'), - link: url, - active_routes: { path: path }, - item_id: :hierarchy - ) - end - - def show_hierarachy_menu_item?(container) - can?(context.current_user, :read_planning_hierarchy, container) - end - end - end -end diff --git a/lib/sidebars/groups/menus/ci_cd_menu.rb b/lib/sidebars/groups/menus/ci_cd_menu.rb index a1f98b918e6..c1d80458f49 100644 --- a/lib/sidebars/groups/menus/ci_cd_menu.rb +++ b/lib/sidebars/groups/menus/ci_cd_menu.rb @@ -29,7 +29,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Runners'), link: group_runners_path(context.group), - active_routes: { path: 'groups/runners#index' }, + active_routes: { controller: 'groups/runners' }, item_id: :runners ) end diff --git a/lib/sidebars/groups/menus/customer_relations_menu.rb b/lib/sidebars/groups/menus/customer_relations_menu.rb index 002197965d1..0aaa6ec45f1 100644 --- a/lib/sidebars/groups/menus/customer_relations_menu.rb +++ b/lib/sidebars/groups/menus/customer_relations_menu.rb @@ -24,6 +24,8 @@ module Sidebars override :render? def render? + return false unless context.group.root? + can_read_contact? || can_read_organization? end diff --git a/lib/sidebars/groups/menus/kubernetes_menu.rb b/lib/sidebars/groups/menus/kubernetes_menu.rb index 4ea294a4837..98ca7865995 100644 --- a/lib/sidebars/groups/menus/kubernetes_menu.rb +++ b/lib/sidebars/groups/menus/kubernetes_menu.rb @@ -21,7 +21,10 @@ module Sidebars override :render? def render? - can?(context.current_user, :read_cluster, context.group) + clusterable = context.group + + Feature.enabled?(:certificate_based_clusters, clusterable, default_enabled: :yaml, type: :ops) && + can?(context.current_user, :read_cluster, clusterable) end override :extra_container_html_options diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb index 60d91c8fd10..4c21845ef18 100644 --- a/lib/sidebars/groups/menus/packages_registries_menu.rb +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -8,8 +8,8 @@ module Sidebars def configure_menu_items add_item(packages_registry_menu_item) add_item(container_registry_menu_item) + add_item(harbor_registry__menu_item) add_item(dependency_proxy_menu_item) - true end @@ -49,6 +49,17 @@ module Sidebars ) end + def harbor_registry__menu_item + return nil_menu_item(:harbor_registry) if Feature.disabled?(:harbor_registry_integration) + + ::Sidebars::MenuItem.new( + title: _('Harbor Registry'), + link: group_harbor_registries_path(context.group), + active_routes: { controller: 'groups/harbor/repositories' }, + item_id: :harbor_registry + ) + end + def dependency_proxy_menu_item setting_does_not_exist_or_is_enabled = !context.group.dependency_proxy_setting || context.group.dependency_proxy_setting.enabled diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index 810b467ed2d..09226256476 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -89,10 +89,16 @@ module Sidebars end def ci_cd_menu_item + active_routes_path = if Feature.enabled?(:runner_list_group_view_vue_ui, context.group, default_enabled: :yaml) + 'ci_cd#show' + else + %w[ci_cd#show groups/runners#show groups/runners#edit] + end + ::Sidebars::MenuItem.new( title: _('CI/CD'), link: group_settings_ci_cd_path(context.group), - active_routes: { path: %w[ci_cd#show groups/runners#show groups/runners#edit] }, + active_routes: { path: active_routes_path }, item_id: :ci_cd ) end diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb index 1af3d024291..d9d294ff982 100644 --- a/lib/sidebars/menu.rb +++ b/lib/sidebars/menu.rb @@ -15,6 +15,7 @@ module Sidebars include ::Sidebars::Concerns::HasPartial attr_reader :context + delegate :current_user, :container, to: :@context def initialize(context) diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index 060a5be5f57..c012b3bb627 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -64,7 +64,7 @@ module Sidebars end def serverless_menu_item - unless can?(context.current_user, :read_cluster, context.project) + unless Feature.enabled?(:deprecated_serverless, context.project, default_enabled: :yaml, type: :ops) && can?(context.current_user, :read_cluster, context.project) return ::Sidebars::NilMenuItem.new(item_id: :serverless) end @@ -100,7 +100,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Google Cloud'), link: project_google_cloud_index_path(context.project), - active_routes: { controller: [:google_cloud, :service_accounts, :deployments] }, + active_routes: { controller: [:google_cloud, :service_accounts, :deployments, :gcp_regions] }, item_id: :google_cloud ) end diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index f5f0da2992e..77f09986b19 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -9,7 +9,7 @@ module Sidebars add_item(packages_registry_menu_item) add_item(container_registry_menu_item) add_item(infrastructure_registry_menu_item) - + add_item(harbor_registry__menu_item) true end @@ -65,6 +65,17 @@ module Sidebars ) end + def harbor_registry__menu_item + return ::Sidebars::NilMenuItem.new(item_id: :harbor_registry) if Feature.disabled?(:harbor_registry_integration) + + ::Sidebars::MenuItem.new( + title: _('Harbor Registry'), + link: project_harbor_registry_index_path(context.project), + active_routes: { controller: :harbor_registry }, + item_id: :harbor_registry + ) + end + def packages_registry_disabled? !::Gitlab.config.packages.enabled || !can?(context.current_user, :read_package, context.project) end diff --git a/lib/sidebars/projects/menus/project_information_menu.rb b/lib/sidebars/projects/menus/project_information_menu.rb index 4056d50d324..44b94ee3522 100644 --- a/lib/sidebars/projects/menus/project_information_menu.rb +++ b/lib/sidebars/projects/menus/project_information_menu.rb @@ -4,13 +4,10 @@ module Sidebars module Projects module Menus class ProjectInformationMenu < ::Sidebars::Menu - include ::Sidebars::Concerns::WorkItemHierarchy - override :configure_menu_items def configure_menu_items add_item(activity_menu_item) add_item(labels_menu_item) - add_item(hierarchy_menu_item(context.project, project_planning_hierarchy_path(context.project), 'projects#planning_hierarchy')) add_item(members_menu_item) true diff --git a/lib/spam/concerns/has_spam_action_response_fields.rb b/lib/spam/concerns/has_spam_action_response_fields.rb index 6688ae56cb0..b33c922f7e6 100644 --- a/lib/spam/concerns/has_spam_action_response_fields.rb +++ b/lib/spam/concerns/has_spam_action_response_fields.rb @@ -7,7 +7,7 @@ module Spam module HasSpamActionResponseFields extend ActiveSupport::Concern - # spam_action_response_fields(spammable) -> hash + # spam_action_response_fields(spammable) -> hash # # Takes a Spammable as an argument and returns response fields necessary to display a CAPTCHA on # the client. diff --git a/lib/tasks/ci/build_artifacts.rake b/lib/tasks/ci/build_artifacts.rake new file mode 100644 index 00000000000..4f4faef5a62 --- /dev/null +++ b/lib/tasks/ci/build_artifacts.rake @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'httparty' +require 'csv' + +namespace :ci do + namespace :build_artifacts do + desc "GitLab | CI | Fetch projects with incorrect artifact size on GitLab.com" + task :project_with_incorrect_artifact_size do + csv_url = ENV['SISENSE_PROJECT_IDS_WITH_INCORRECT_ARTIFACTS_URL'] + + # rubocop: disable Gitlab/HTTParty + body = HTTParty.get(csv_url) + # rubocop: enable Gitlab/HTTParty + + table = CSV.parse(body.parsed_response, headers: true) + puts table['PROJECT_ID'].join(' ') + end + end +end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index cb01f229cd3..99ffeb4ec0b 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -8,8 +8,10 @@ namespace :dev do ENV['force'] = 'yes' Rake::Task["gitlab:setup"].invoke - # Make sure DB statistics are up to date. - ActiveRecord::Base.connection.execute('ANALYZE') + Gitlab::Database::EachDatabase.each_database_connection do |connection| + # Make sure DB statistics are up to date. + connection.execute('ANALYZE') + end Rake::Task["gitlab:shell:setup"].invoke end diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake index c7f3d003f9f..033427fa799 100644 --- a/lib/tasks/gitlab/background_migrations.rake +++ b/lib/tasks/gitlab/background_migrations.rake @@ -1,41 +1,110 @@ # frozen_string_literal: true +databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml + namespace :gitlab do namespace :background_migrations do desc 'Synchronously finish executing a batched background migration' task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| - [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument| - unless args[argument] - puts "Must specify #{argument} as an argument".color(:red) - exit 1 - end + if Gitlab::Database.db_config_names.size > 1 + puts "Please specify the database".color(:red) + exit 1 end - Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize( + validate_finalization_arguments!(args) + + main_model = Gitlab::Database.database_base_models[:main] + + finalize_migration( args[:job_class_name], args[:table_name], args[:column_name], - Gitlab::Json.parse(args[:job_arguments]) + Gitlab::Json.parse(args[:job_arguments]), + connection: main_model.connection ) + end - puts "Done.".color(:green) + namespace :finalize do + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + next if name.to_s == 'geo' + + desc "Gitlab | DB | Synchronously finish executing a batched background migration on #{name} database" + task name, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| + validate_finalization_arguments!(args) + + model = Gitlab::Database.database_base_models[name] + + finalize_migration( + args[:job_class_name], + args[:table_name], + args[:column_name], + Gitlab::Json.parse(args[:job_arguments]), + connection: model.connection + ) + end + end end desc 'Display the status of batched background migrations' - task status: :environment do - statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses - max_status_length = statuses.keys.map(&:length).max - format_string = "%-#{max_status_length}s | %s\n" - - Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration| - identification_fields = [ - migration.job_class_name, - migration.table_name, - migration.column_name, - migration.job_arguments.to_json - ].join(',') - - printf(format_string, migration.status, identification_fields) + task status: :environment do |_, args| + Gitlab::Database.database_base_models.each do |name, model| + display_migration_status(name, model.connection) + end + end + + namespace :status do + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + next if name.to_s == 'geo' + + desc "Gitlab | DB | Display the status of batched background migrations on #{name} database" + task name => :environment do |_, args| + model = Gitlab::Database.database_base_models[name] + display_migration_status(name, model.connection) + end + end + end + + private + + def finalize_migration(class_name, table_name, column_name, job_arguments, connection:) + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize( + class_name, + table_name, + column_name, + Gitlab::Json.parse(job_arguments), + connection: connection + ) + + puts "Done.".color(:green) + end + + def display_migration_status(database_name, connection) + Gitlab::Database::SharedModel.using_connection(connection) do + statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses + max_status_length = statuses.keys.map(&:length).max + format_string = "%-#{max_status_length}s | %s\n" + + puts "Database: #{database_name}\n" + + Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration| + identification_fields = [ + migration.job_class_name, + migration.table_name, + migration.column_name, + migration.job_arguments.to_json + ].join(',') + + printf(format_string, migration.status, identification_fields) + end + end + end + + def validate_finalization_arguments!(args) + [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument| + unless args[argument] + puts "Must specify #{argument} as an argument".color(:red) + exit 1 + end end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 6d4af9d166f..50ceb11581e 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -4,30 +4,28 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml namespace :gitlab do namespace :db do - desc 'GitLab | DB | Manually insert schema migration version' + desc 'GitLab | DB | Manually insert schema migration version on all configured databases' task :mark_migration_complete, [:version] => :environment do |_, args| mark_migration_complete(args[:version]) end namespace :mark_migration_complete do - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| - desc "Gitlab | DB | Manually insert schema migration version on #{name} database" - task name, [:version] => :environment do |_, args| - mark_migration_complete(args[:version], database: name) + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| + desc "Gitlab | DB | Manually insert schema migration version on #{database} database" + task database, [:version] => :environment do |_, args| + mark_migration_complete(args[:version], only_on: database) end end end - def mark_migration_complete(version, database: nil) + def mark_migration_complete(version, only_on: nil) if version.to_i == 0 puts 'Must give a version argument that is a non-zero integer'.color(:red) exit 1 end - Gitlab::Database.database_base_models.each do |name, model| - next if database && database.to_s != name - - model.connection.execute("INSERT INTO schema_migrations (version) VALUES (#{model.connection.quote(version)})") + Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |connection, name| + connection.execute("INSERT INTO schema_migrations (version) VALUES (#{connection.quote(version)})") puts "Successfully marked '#{version}' as complete on database #{name}".color(:green) rescue ActiveRecord::RecordNotUnique @@ -35,32 +33,44 @@ namespace :gitlab do end end - desc 'GitLab | DB | Drop all tables' + desc 'GitLab | DB | Drop all tables on all configured databases' task drop_tables: :environment do - connection = ActiveRecord::Base.connection + drop_tables + end - # In PostgreSQLAdapter, data_sources returns both views and tables, so use - # #tables instead - tables = connection.tables + namespace :drop_tables do + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| + desc "GitLab | DB | Drop all tables on the #{database} database" + task database => :environment do + drop_tables(only_on: database) + end + end + end - # Removes the entry from the array - tables.delete 'schema_migrations' - # Truncate schema_migrations to ensure migrations re-run - connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' + def drop_tables(only_on: nil) + Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |connection, name| + # In PostgreSQLAdapter, data_sources returns both views and tables, so use tables instead + tables = connection.tables - # Drop any views - connection.views.each do |view| - connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE") - end + # Removes the entry from the array + tables.delete 'schema_migrations' + # Truncate schema_migrations to ensure migrations re-run + connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' - # Drop tables with cascade to avoid dependent table errors - # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html - # Add `IF EXISTS` because cascade could have already deleted a table. - tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } + # Drop any views + connection.views.each do |view| + connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE") + end - # Drop all extra schema objects GitLab owns - Gitlab::Database::EXTRA_SCHEMAS.each do |schema| - connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE") + # Drop tables with cascade to avoid dependent table errors + # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html + # Add `IF EXISTS` because cascade could have already deleted a table. + tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } + + # Drop all extra schema objects GitLab owns + Gitlab::Database::EXTRA_SCHEMAS.each do |schema| + connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE") + end end end @@ -152,6 +162,17 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + # We'll temporarily skip this enhancement for geo, since in some situations we + # wish to setup the geo database before the other databases have been setup, + # and partition management attempts to connect to the main database. + next if name == 'geo' + + Rake::Task["db:migrate:#{name}"].enhance do + Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + end + end + # When we load the database schema from db/structure.sql # we don't have any dynamic partitions created. We don't really need to # because application initializers/sidekiq take care of that, too. @@ -160,16 +181,29 @@ namespace :gitlab do # # Other than that it's helpful to create partitions early when bootstrapping # a new installation. - # - # Rails 6.1 deprecates db:structure:load in favor of db:schema:load - Rake::Task['db:structure:load'].enhance do + Rake::Task['db:schema:load'].enhance do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - Rake::Task['db:schema:load'].enhance do - Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + # We'll temporarily skip this enhancement for geo, since in some situations we + # wish to setup the geo database before the other databases have been setup, + # and partition management attempts to connect to the main database. + next if name == 'geo' + + Rake::Task["db:schema:load:#{name}"].enhance do + Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + end + end + + desc "Clear all connections" + task :clear_all_connections do + ActiveRecord::Base.clear_all_connections! end + Rake::Task['db:test:purge'].enhance(['gitlab:db:clear_all_connections']) + Rake::Task['db:drop'].enhance(['gitlab:db:clear_all_connections']) + # During testing, db:test:load restores the database schema from scratch # which does not include dynamic partitions. We cannot rely on application # initializers here as the application can continue to run while @@ -195,8 +229,6 @@ namespace :gitlab do end namespace :reindex do - databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name| desc "Reindex #{database_name} database without downtime to eliminate bloat" task database_name => :environment do diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake index e7ece9e0fdd..2d234fcdb36 100644 --- a/lib/tasks/gitlab/docs/redirect.rake +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -54,7 +54,9 @@ namespace :gitlab do post.puts "This document was moved to [another location](#{new_path})." post.puts post.puts "<!-- This redirect file can be deleted after <#{date}>. -->" - post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->" + post.puts "<!-- Redirects that point to other docs in the same project expire in three months. -->" + post.puts "<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. -->" + post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->" end end end diff --git a/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake new file mode 100644 index 00000000000..1cc18d14d78 --- /dev/null +++ b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + desc "GitLab | Refresh build artifacts size project statistics for given project IDs" + + BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE = 500 + + task :refresh_project_statistics_build_artifacts_size, [:project_ids] => :environment do |_t, args| + project_ids = [] + project_ids = $stdin.read.split unless $stdin.tty? + project_ids = args.project_ids.to_s.split unless project_ids.any? + + if project_ids.any? + project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE) do |ids| + projects = Project.where(id: ids) + Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects) + end + puts 'Done.'.green + else + puts 'Please provide a string of space-separated project IDs as the argument or through the STDIN'.red + end + end +end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 705519d1741..a5289476378 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -47,13 +47,15 @@ namespace :gitlab do # will work. def self.terminate_all_connections cmd = <<~SQL - SELECT pg_terminate_backend(pg_stat_activity.pid) - FROM pg_stat_activity - WHERE datname = current_database() + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid(); SQL - ActiveRecord::Base.connection.execute(cmd)&.result_status == PG::PGRES_TUPLES_OK - rescue ActiveRecord::NoDatabaseError + Gitlab::Database::EachDatabase.each_database_connection do |connection| + connection.execute(cmd) + rescue ActiveRecord::NoDatabaseError + end end end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 43fd4f8685a..358bc6c31eb 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -22,7 +22,7 @@ namespace :tw do CodeOwnerRule.new('Container Security', '@ngaskill'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Conversion', '@kpaizee'), - CodeOwnerRule.new('Database', '@marcia'), + CodeOwnerRule.new('Database', '@aqualls'), CodeOwnerRule.new('Development', '@marcia'), CodeOwnerRule.new('Distribution', '@axil'), CodeOwnerRule.new('Distribution (Charts)', '@axil'), diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index 8c5edb5de8a..6eabdf51dcd 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -1,4 +1,5 @@ # frozen_string_literal: true +# rubocop:disable Rails/RakeEnvironment unless Rails.env.production? require 'rubocop/rake_task' @@ -8,18 +9,59 @@ unless Rails.env.production? namespace :rubocop do namespace :todo do desc 'Generate RuboCop todos' - task :generate do # rubocop:disable Rails/RakeEnvironment + task :generate do |_task, args| require 'rubocop' + require 'active_support/inflector/inflections' + require_relative '../../rubocop/todo_dir' + require_relative '../../rubocop/formatter/todo_formatter' + + # Reveal all pending TODOs so RuboCop can pick them up and report + # during scan. + ENV['REVEAL_RUBOCOP_TODO'] = '1' + + # Save cop configuration like `RSpec/ContextWording` into + # `rspec/context_wording.yml` and not into + # `r_spec/context_wording.yml`. + ActiveSupport::Inflector.inflections(:en) do |inflect| + inflect.acronym 'RSpec' + inflect.acronym 'GraphQL' + end options = %w[ - --auto-gen-config - --auto-gen-only-exclude - --exclude-limit=100000 - --no-offense-counts + --parallel + --format RuboCop::Formatter::TodoFormatter ] + # Convert from Rake::TaskArguments into an Array to make `any?` work as + # expected. + cop_names = args.to_a + + todo_dir = RuboCop::TodoDir.new(RuboCop::TodoDir::DEFAULT_TODO_DIR) + + if cop_names.any? + # We are sorting the cop names to benefit from RuboCop cache which + # also takes passed parameters into account. + list = cop_names.sort.join(',') + options.concat ['--only', list] + + cop_names.each { |cop_name| todo_dir.inspect(cop_name) } + else + todo_dir.inspect_all + end + + puts <<~MSG + Generating RuboCop TODOs with: + rubocop #{options.join(' ')} + + This might take a while... + MSG + RuboCop::CLI.new.run(options) + + todo_dir.delete_inspected end end end end + +# rubocop:enable Rails/RakeEnvironment diff --git a/lib/tasks/tanuki_emoji.rake b/lib/tasks/tanuki_emoji.rake index 98d3920c07f..0dc7dd4e701 100644 --- a/lib/tasks/tanuki_emoji.rake +++ b/lib/tasks/tanuki_emoji.rake @@ -3,12 +3,20 @@ namespace :tanuki_emoji do desc 'Generates Emoji aliases fixtures' task aliases: :environment do + ALLOWED_ALIASES = [':)', ':('].freeze aliases = {} TanukiEmoji.index.all.each do |emoji| emoji.aliases.each do |emoji_alias| aliases[TanukiEmoji::Character.format_name(emoji_alias)] = emoji.name end + + emoji.ascii_aliases.intersection(ALLOWED_ALIASES).each do |ascii_alias| + # We add an extra space at the end so that when a user types ":) " + # we'd still match this alias and not show "cocos (keeling) islands" as the first result. + # The initial ":" is ignored when matching because it's our emoji prefix in Markdown. + aliases[ascii_alias + ' '] = emoji.name + end end aliases_json_file = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') |