diff options
Diffstat (limited to 'lib')
369 files changed, 5334 insertions, 1908 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 5e449022676..4b7fe6bdc7a 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -164,7 +164,7 @@ module API namespace do after do - ::Users::ActivityService.new(@current_user).execute + ::Users::ActivityService.new(author: @current_user, project: @project, namespace: @group).execute end # Mount endpoints to include in the OpenAPI V2 documentation here @@ -209,6 +209,7 @@ module API mount ::API::DeployKeys mount ::API::DeployTokens mount ::API::Deployments + mount ::API::DraftNotes mount ::API::Environments mount ::API::ErrorTracking::ClientKeys mount ::API::ErrorTracking::ProjectSettings diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index df550f12c0d..81a640d9a93 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -57,10 +57,7 @@ module API user = find_user_from_sources return unless user - if user.is_a?(User) && Gitlab::CurrentSettings.admin_mode - # Sessions are enforced to be unavailable for API calls, so ignore them for admin mode - Gitlab::Auth::CurrentUserMode.bypass_session!(user.id) - end + Gitlab::Auth::CurrentUserMode.bypass_session!(user.id) if bypass_session_for_admin_mode?(user) unless api_access_allowed?(user) forbidden!(api_access_denied_message(user)) @@ -85,6 +82,16 @@ module API private + def bypass_session_for_admin_mode?(user) + return user.is_a?(User) && Gitlab::CurrentSettings.admin_mode if Feature.disabled?(:admin_mode_for_api) + + return false unless Gitlab::CurrentSettings.admin_mode + return false unless user.is_a?(User) + + Gitlab::Session.with_session(current_request.session) { Gitlab::Auth::CurrentUserMode.new(user).admin_mode? } || + Gitlab::Auth::RequestAuthenticator.new(current_request).valid_access_token?(scopes: [:admin_mode]) + end + # An array of scopes that were registered (using `allow_access_with_scope`) # for the current endpoint class. It also returns scopes registered on # `API::API`, since these are meant to apply to all API routes. diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb index 99278bdf8b0..f8e4f5d2ab2 100644 --- a/lib/api/appearance.rb +++ b/lib/api/appearance.rb @@ -26,13 +26,15 @@ module API end params do optional :title, type: String, desc: 'Instance title on the sign in / sign up page' - optional :pwa_short_name, type: String, desc: 'Optional, short name for Progressive Web App' optional :description, type: String, desc: 'Markdown text shown on the sign in / sign up page' + optional :pwa_name, type: String, desc: 'Name of the Progressive Web App' + optional :pwa_short_name, type: String, desc: 'Optional, short name for Progressive Web App' + optional :pwa_description, type: String, desc: 'An explanation of what the Progressive Web App does' # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 - optional :logo, type: File, desc: 'Instance image used on the sign in / sign up page' # rubocop:disable Scalability/FileUploads - optional :pwa_icon, type: File, desc: 'Icon used for Progressive Web App' # rubocop:disable Scalability/FileUploads - optional :header_logo, type: File, desc: 'Instance image used for the main navigation bar' # rubocop:disable Scalability/FileUploads - optional :favicon, type: File, desc: 'Instance favicon in .ico/.png format' # rubocop:disable Scalability/FileUploads + optional :logo, type: File, desc: 'Instance image used on the sign in / sign up page' # rubocop:todo Scalability/FileUploads + optional :pwa_icon, type: File, desc: 'Icon used for Progressive Web App' # rubocop:todo Scalability/FileUploads + optional :header_logo, type: File, desc: 'Instance image used for the main navigation bar' # rubocop:todo Scalability/FileUploads + optional :favicon, type: File, desc: 'Instance favicon in .ico/.png format' # rubocop:todo Scalability/FileUploads optional :new_project_guidelines, type: String, desc: 'Markdown text shown on the new project page' optional :profile_image_guidelines, type: String, desc: 'Markdown text shown on the profile page below Public Avatar' optional :header_message, type: String, desc: 'Message within the system header bar' diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb index 0fb7a4cd435..addea780eb0 100644 --- a/lib/api/avatar.rb +++ b/lib/api/avatar.rb @@ -2,7 +2,7 @@ module API class Avatar < ::API::Base - feature_category :users + feature_category :user_profile urgency :medium resource :avatar do diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index 6c07b15329e..e3dc9ea52cb 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -91,6 +91,8 @@ module API end end post do + check_rate_limit!(:bulk_import, scope: current_user) + params[:entities].each do |entity| if entity[:destination_name] entity[:destination_slug] ||= entity[:destination_name] diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index be4d82bc500..96f5265ce23 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -10,23 +10,32 @@ module API JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN' JOB_TOKEN_PARAM = :token + LEGACY_SYSTEM_XID = '<legacy>' def authenticate_runner! track_runner_authentication forbidden! unless current_runner - current_runner - .heartbeat(get_runner_details_from_request) + runner_details = get_runner_details_from_request + current_runner.heartbeat(runner_details) + current_runner_machine&.heartbeat(runner_details) end def get_runner_details_from_request return get_runner_ip unless params['info'].present? attributes_for_keys(%w(name version revision platform architecture executor), params['info']) + .merge(get_system_id_from_request) .merge(get_runner_config_from_request) .merge(get_runner_ip) end + def get_system_id_from_request + return { system_id: params[:system_id] } if params.include?(:system_id) + + {} + end + def get_runner_ip { ip_address: ip_address } end @@ -43,6 +52,15 @@ module API end end + def current_runner_machine + return if Feature.disabled?(:create_runner_machine) + + strong_memoize(:current_runner_machine) do + system_xid = params.fetch(:system_id, LEGACY_SYSTEM_XID) + current_runner&.ensure_machine(system_xid) { |m| m.contacted_at = Time.current } + end + end + def track_runner_authentication if current_runner metrics.increment_runner_authentication_success_counter(runner_type: current_runner.runner_type) @@ -78,6 +96,7 @@ module API # the heartbeat should be triggered. if heartbeat_runner job.runner&.heartbeat(get_runner_ip) + job.runner_machine&.heartbeat(get_runner_ip) end job diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index ed1c7dfbfa2..30d12864bf8 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -57,11 +57,7 @@ module API builds = filter_builds(builds, params[:scope]) builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project) - if Feature.enabled?(:jobs_api_keyset_pagination, user_project) - present paginate_with_strategies(builds, paginator_params: { without_count: true }), with: Entities::Ci::Job - else - present paginate(builds, without_count: true), with: Entities::Ci::Job - end + present paginate_with_strategies(builds, paginator_params: { without_count: true }), with: Entities::Ci::Job end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 6b4394114df..1c81db39bb1 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -77,15 +77,18 @@ module API desc 'Validate authentication credentials' do summary "Verify authentication for a registered runner" + success Entities::Ci::RunnerRegistrationDetails http_codes [[200, 'Credentials are valid'], [403, 'Forbidden']] end params do requires :token, type: String, desc: %q(The runner's authentication token) + optional :system_id, type: String, desc: %q(The runner's system identifier) end post '/verify', urgency: :low, feature_category: :runner do authenticate_runner! status 200 - body "200" + + present current_runner, with: Entities::Ci::RunnerRegistrationDetails end desc 'Reset runner authentication token with current token' do @@ -115,6 +118,7 @@ module API end params do requires :token, type: String, desc: %q(Runner's authentication token) + optional :system_id, type: String, desc: %q(Runner's system identifier) optional :last_update, type: String, desc: %q(Runner's queue last_update token) optional :info, type: Hash, desc: %q(Runner's metadata) do optional :name, type: String, desc: %q(Runner's name) @@ -166,7 +170,7 @@ module API end new_update = current_runner.ensure_runner_queue_value - result = ::Ci::RegisterJobService.new(current_runner).execute(runner_params) + result = ::Ci::RegisterJobService.new(current_runner, current_runner_machine).execute(runner_params) if result.valid? if result.build_json @@ -192,7 +196,7 @@ module API [403, 'Forbidden']] end params do - requires :token, type: String, desc: %q(Runner's authentication token) + requires :token, type: String, desc: %q(Job token) requires :id, type: Integer, desc: %q(Job's ID) optional :state, type: String, desc: %q(Job's status: success, failed) optional :checksum, type: String, desc: %q(Job's trace CRC32 checksum) diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 4a6c58b4987..f2f0f32261a 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -13,7 +13,7 @@ module API helpers do params :deprecated_filter_params do optional :scope, type: String, values: ::Ci::Runner::AVAILABLE_SCOPES, - desc: 'Deprecated: Use `type` or `status` instead. The scope of specific runners to return' + desc: 'Deprecated: Use `type` or `status` instead. The scope of runners to return' end params :filter_params do @@ -111,9 +111,9 @@ module API present paginate(runners), with: Entities::Ci::Runner end - desc 'Get all runners - shared and specific' do + desc 'Get all runners - shared and project' do summary 'List all runners' - detail 'Get a list of all runners in the GitLab instance (specific and shared). ' \ + detail 'Get a list of all runners in the GitLab instance (shared and project). ' \ 'Access is restricted to users with administrator access.' success Entities::Ci::Runner failure [[400, 'Scope contains invalid value'], [401, 'Unauthorized']] @@ -286,7 +286,7 @@ module API end desc 'Enable a runner in project' do - detail "Enable an available specific runner in the project." + detail "Enable an available project runner in the project." success Entities::Ci::Runner failure [[400, 'Bad Request'], [403, 'No access granted'], [403, 'Runner is a group runner'], [403, 'Runner is locked'], @@ -308,7 +308,7 @@ module API end desc "Disable project's runner" do - summary "Disable a specific runner from the project" + summary "Disable a project runner from the project" detail "It works only if the project isn't the only project associated with the specified runner. " \ "If so, an error is returned. Use the call to delete a runner instead." success Entities::Ci::Runner diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index 6483abcc74e..41faaf80c82 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -10,7 +10,7 @@ module API authorize! :read_secure_files, user_project end - feature_category :pipeline_authoring + feature_category :mobile_devops default_format :json diff --git a/lib/api/concerns/packages/debian_distribution_endpoints.rb b/lib/api/concerns/packages/debian_distribution_endpoints.rb index 6fe3f432edb..7aad3a56422 100644 --- a/lib/api/concerns/packages/debian_distribution_endpoints.rb +++ b/lib/api/concerns/packages/debian_distribution_endpoints.rb @@ -76,7 +76,7 @@ module API end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'sid' } use :optional_distribution_params end post '/' do @@ -107,7 +107,7 @@ module API params do use :pagination - optional :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } + optional :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'sid' } use :optional_distribution_params end get '/' do @@ -132,7 +132,7 @@ module API end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'sid' } end get '/:codename' do authorize_read_package!(project_or_group) @@ -153,7 +153,7 @@ module API end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'sid' } end get '/:codename/key.asc' do authorize_read_package!(project_or_group) @@ -179,7 +179,7 @@ module API end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'sid' } use :optional_distribution_params end put '/:codename' do @@ -210,7 +210,7 @@ module API end params do - requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'unstable' } + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename', documentation: { example: 'sid' } use :optional_distribution_params end delete '/:codename' do diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index 181759a7f38..db31f2e35f1 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -44,6 +44,8 @@ module API end def present_index_file!(file_type) + not_found!("Format #{params[:format]} is not supported") unless params[:format].nil? + relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize relation = relation diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 483d0dd9c90..67416e62f7d 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -6,10 +6,6 @@ module API project_id: %r{[0-9]+}.freeze ).freeze - before do - not_found! if Gitlab::FIPS.enabled? - end - resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def project_or_group diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 353f64b8dd1..21c0c219046 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -14,10 +14,6 @@ module API file_name: API::NO_SLASH_URL_PART_REGEX }.freeze - before do - not_found! if Gitlab::FIPS.enabled? - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def project_or_group @@ -69,6 +65,7 @@ module API format :txt content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + # PUT {projects|groups}/:id/packages/debian/:file_name desc 'Upload Debian package' do detail 'This feature was introduced in GitLab 14.0' success code: 201 @@ -80,12 +77,14 @@ module API ] tags %w[debian_packages] end - - # PUT {projects|groups}/:id/packages/debian/:file_name params do requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } + optional :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex + given :distribution do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :file_name, type: String, desc: 'The filename', regexp: { value: Gitlab::Regex.debian_direct_upload_filename_regex, message: 'Only debs and udebs can be directly added to a distribution' } + end end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true put do authorize_upload!(authorized_user_project) @@ -95,10 +94,18 @@ module API file: params['file'], file_name: params['file_name'], file_sha1: params['file.sha1'], - file_md5: params['file.md5'] + file_md5: params['file.md5'], + distribution: params['distribution'], + component: params['component'] } - package = ::Packages::Debian::FindOrCreateIncomingService.new(authorized_user_project, current_user).execute + package = if params[:distribution].present? + ::Packages::CreateTemporaryPackageService.new( + authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job) + ).execute(:debian, name: ::Packages::Debian::TEMPORARY_PACKAGE_NAME) + else + ::Packages::Debian::FindOrCreateIncomingService.new(authorized_user_project, current_user).execute + end ::Packages::Debian::CreatePackageFileService.new(package: package, current_user: current_user, params: file_params).execute @@ -114,12 +121,20 @@ module API detail 'This feature was introduced in GitLab 13.5' success code: 200 failure [ + { code: 400, message: 'Bad Request' }, { code: 401, message: 'Unauthorized' }, { code: 403, message: 'Forbidden' }, { code: 404, message: 'Not Found' } ] tags %w[debian_packages] end + params do + optional :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex + given :distribution do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :file_name, type: String, desc: 'The filename', regexp: { value: Gitlab::Regex.debian_direct_upload_filename_regex, message: 'Only debs and udebs can be directly added to a distribution' } + end + end route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true put 'authorize' do authorize_workhorse!( diff --git a/lib/api/draft_notes.rb b/lib/api/draft_notes.rb new file mode 100644 index 00000000000..842180652c4 --- /dev/null +++ b/lib/api/draft_notes.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +module API + class DraftNotes < ::API::Base + before { authenticate! } + + urgency :low + + helpers do + def merge_request(params:) + strong_memoize(:merge_request) do + find_project_merge_request(params[:merge_request_iid]) + end + end + + def load_draft_notes(params:) + merge_request(params: params).draft_notes.authored_by(current_user) + end + + def get_draft_note(params:) + load_draft_notes(params: params).find(params[:draft_note_id]) + end + + def delete_draft_note(draft_note) + ::DraftNotes::DestroyService.new(user_project, current_user).execute(draft_note) + end + + def publish_draft_note(params:) + ::DraftNotes::PublishService + .new(merge_request(params: params), current_user) + .execute(get_draft_note(params: params)) + end + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc "Get a list of merge request draft notes" do + success Entities::DraftNote + is_array true + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + end + params do + requires :id, type: String, desc: "The ID of a project" + requires :merge_request_iid, type: Integer, desc: "The ID of a merge request" + end + get ":id/merge_requests/:merge_request_iid/draft_notes", feature_category: :code_review_workflow do + present load_draft_notes(params: params), with: Entities::DraftNote + end + + desc "Get a single draft note" do + success Entities::DraftNote + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + end + params do + requires :id, type: String, desc: "The ID of a project" + requires :merge_request_iid, type: Integer, desc: "The ID of a merge request" + requires :draft_note_id, type: Integer, desc: "The ID of a draft note" + end + get ":id/merge_requests/:merge_request_iid/draft_notes/:draft_note_id", feature_category: :code_review_workflow do + draft_note = get_draft_note(params: params) + + if draft_note + present draft_note, with: Entities::DraftNote + else + not_found!("Draft Note") + end + end + + desc "Delete a draft note" do + success Entities::DraftNote + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + end + params do + requires :id, type: String, desc: "The ID of a project" + requires :merge_request_iid, type: Integer, desc: "The ID of a merge request" + requires :draft_note_id, type: Integer, desc: "The ID of a draft note" + end + delete( + ":id/merge_requests/:merge_request_iid/draft_notes/:draft_note_id", + feature_category: :code_review_workflow) do + draft_note = get_draft_note(params: params) + + if draft_note + delete_draft_note(draft_note) + status 204 + body false + else + not_found!("Draft Note") + end + end + + desc "Publish a pending draft note" do + success code: 204 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not found' } + ] + end + params do + requires :id, type: String, desc: "The ID of a project" + requires :merge_request_iid, type: Integer, desc: "The ID of a merge request" + requires :draft_note_id, type: Integer, desc: "The ID of a draft note" + end + put( + ":id/merge_requests/:merge_request_iid/draft_notes/:draft_note_id/publish", + feature_category: :code_review_workflow) do + result = publish_draft_note(params: params) + + if result[:status] == :success + status 204 + body false + else + status 500 + end + end + end + end +end diff --git a/lib/api/entities/appearance.rb b/lib/api/entities/appearance.rb index cabdf68c23a..2825c84e01d 100644 --- a/lib/api/entities/appearance.rb +++ b/lib/api/entities/appearance.rb @@ -4,8 +4,10 @@ module API module Entities class Appearance < Grape::Entity expose :title - expose :pwa_short_name expose :description + expose :pwa_name + expose :pwa_short_name + expose :pwa_description expose :logo do |appearance, options| appearance.logo.url diff --git a/lib/api/entities/ci/job_basic.rb b/lib/api/entities/ci/job_basic.rb index 3cbb8aad313..526dfba05f2 100644 --- a/lib/api/entities/ci/job_basic.rb +++ b/lib/api/entities/ci/job_basic.rb @@ -15,6 +15,7 @@ module API expose :created_at, documentation: { type: 'dateTime', example: '2015-12-24T15:51:21.880Z' } expose :started_at, documentation: { type: 'dateTime', example: '2015-12-24T17:54:30.733Z' } expose :finished_at, documentation: { type: 'dateTime', example: '2015-12-24T17:54:31.198Z' } + expose :erased_at, documentation: { type: 'dateTime', example: '2015-12-24T18:00:29.728Z' } expose :duration, documentation: { type: 'number', format: 'float', desc: 'Time spent running', example: 0.465 } expose :queued_duration, diff --git a/lib/api/entities/ci/secure_file.rb b/lib/api/entities/ci/secure_file.rb index a234ada6f82..bf8d2776db1 100644 --- a/lib/api/entities/ci/secure_file.rb +++ b/lib/api/entities/ci/secure_file.rb @@ -12,6 +12,7 @@ documentation: { type: 'string', example: '16630b189ab34b2e3504f4758e1054d2e478d expose :created_at, documentation: { type: 'dateTime', example: '2022-02-22T22:22:22.222Z' } expose :expires_at, documentation: { type: 'dateTime', example: '2022-09-21T14:56:00.000Z' } expose :metadata, documentation: { type: 'Hash', example: { "id" => "75949910542696343243264405377658443914" } } + expose :file_extension, documentation: { type: 'string', example: 'jks' } end end end diff --git a/lib/api/entities/draft_note.rb b/lib/api/entities/draft_note.rb new file mode 100644 index 00000000000..70b32bac502 --- /dev/null +++ b/lib/api/entities/draft_note.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module API + module Entities + class DraftNote < Grape::Entity + expose :id, documentation: { type: 'integer', example: 2 } + expose :author_id, documentation: { type: 'integer', example: 4 } + expose :merge_request_id, documentation: { type: 'integer', example: 52 } + expose :resolve_discussion, documentation: { type: 'boolean', example: true } + expose :discussion_id, documentation: { type: 'integer', example: 613 } + expose :note, documentation: { type: 'string', example: 'This is a note' } + expose :commit_id, documentation: { type: 'integer', example: 4 } + expose :line_code, documentation: { type: 'string', example: '1c497fbb3a46b78edf0_2_4' } + expose :position, documentation: { + type: 'Hash', + example: { + base_sha: "aa149113", + start_sha: "b3a0a8c4", + head_sha: "be3020c7", + old_path: "example.md", + new_path: "example.md", + position_type: "text", + old_line: 2, + new_line: 4, + line_range: { + start: { + line_code: "1c497fbb3a46b78edf04cc2a2fa33f67e3ffbe2a_2_4", + type: nil, + old_line: 2, + new_line: 4 + }, + end: { + line_code: "1c497fbb3a46b78edf04cc2a2fa33f67e3ffbe2a_2_4", + type: nil, + old_line: 2, + new_line: 4 + } + } + } + } do |note| + note.position.to_h + end + end + end +end diff --git a/lib/api/entities/issue.rb b/lib/api/entities/issue.rb index 7630fd1e94e..56e942a0383 100644 --- a/lib/api/entities/issue.rb +++ b/lib/api/entities/issue.rb @@ -6,11 +6,11 @@ module API include ::API::Helpers::RelatedResourcesHelpers expose(:has_tasks) do |issue, _| - !issue.task_list_items.empty? + !issue.tasks? end expose :task_status, if: -> (issue, _) do - !issue.task_list_items.empty? + !issue.tasks? end expose :_links do diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb index 27f6e6ade06..16afc6c1f6a 100644 --- a/lib/api/entities/merge_request_basic.rb +++ b/lib/api/entities/merge_request_basic.rb @@ -91,6 +91,7 @@ module API end expose :squash + expose :squash_on_merge?, as: :squash_on_merge expose :task_completion_status expose :cannot_be_merged?, as: :has_conflicts expose :mergeable_discussions_state?, as: :blocking_discussions_resolved diff --git a/lib/api/entities/packages/debian/distribution.rb b/lib/api/entities/packages/debian/distribution.rb index a11f4337f38..82ee95c198b 100644 --- a/lib/api/entities/packages/debian/distribution.rb +++ b/lib/api/entities/packages/debian/distribution.rb @@ -6,7 +6,7 @@ module API module Debian class Distribution < Grape::Entity expose :id, documentation: { type: 'integer', example: 1 } - expose :codename, documentation: { type: 'string', example: 'unstable' } + expose :codename, documentation: { type: 'string', example: 'sid' } expose :suite, documentation: { type: 'string', example: 'unstable' } expose :origin, documentation: { type: 'string', example: 'Grep' } expose :label, documentation: { type: 'string', example: 'grep.be' } diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 37be6903d8b..fcb7ddb9567 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -88,6 +88,7 @@ module API expose :emails_disabled, documentation: { type: 'boolean' } expose :shared_runners_enabled, documentation: { type: 'boolean' } + expose :group_runners_enabled, documentation: { type: 'boolean' } expose :lfs_enabled?, as: :lfs_enabled, documentation: { type: 'boolean' } expose :creator_id, documentation: { type: 'integer', example: 1 } expose :forked_from_project, using: Entities::BasicProjectDetails, if: ->(project, options) do diff --git a/lib/api/entities/project_with_access.rb b/lib/api/entities/project_with_access.rb index 9722b8806d4..47706165806 100644 --- a/lib/api/entities/project_with_access.rb +++ b/lib/api/entities/project_with_access.rb @@ -25,32 +25,10 @@ module API # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) - if ::Feature.enabled?(:projects_preloader_fix) - super(projects_relation, options) - else - relation = super(projects_relation, options) - # use reselect to override the existing select and - # prevent an error `subquery has too many columns` - project_ids = relation.reselect('projects.id') - namespace_ids = relation.reselect(:namespace_id) - - options[:project_members] = options[:current_user] - .project_members - .where(source_id: project_ids) - .preload(:source, user: [notification_settings: :source]) - - options[:group_members] = options[:current_user] - .group_members - .where(source_id: namespace_ids) - .preload(:source, user: [notification_settings: :source]) - - relation - end + super(projects_relation, options) end def self.postload_relation(projects_relation, options = {}) - return unless ::Feature.enabled?(:projects_preloader_fix) - options[:project_members] = options[:current_user] .project_members .where(source_id: projects_relation.subquery(:id)) diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb index c1a48a46d64..4e4a500718f 100644 --- a/lib/api/entities/release.rb +++ b/lib/api/entities/release.rb @@ -28,8 +28,13 @@ module API end expose :evidences, using: Entities::Releases::Evidence, expose_nil: false, if: ->(_, _) { can_read_code? } expose :_links do - expose :self_url, as: :self, expose_nil: false + expose :closed_issues_url, expose_nil: false + expose :closed_merge_requests_url, expose_nil: false expose :edit_url, expose_nil: false + expose :merged_merge_requests_url, expose_nil: false + expose :opened_issues_url, expose_nil: false + expose :opened_merge_requests_url, expose_nil: false + expose :self_url, as: :self, expose_nil: false end private diff --git a/lib/api/entities/releases/link.rb b/lib/api/entities/releases/link.rb index abf380e11d5..534510ec7e6 100644 --- a/lib/api/entities/releases/link.rb +++ b/lib/api/entities/releases/link.rb @@ -18,7 +18,7 @@ module API } do |link| ::Releases::LinkPresenter.new(link).direct_asset_url end - expose :external?, documentation: { type: 'boolean' }, as: :external + expose :external?, documentation: { type: 'boolean' }, as: :external # @deprecated expose :link_type, documentation: { type: 'string', example: 'other' } end end diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index a86039b856a..884f0f75d7a 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -7,7 +7,7 @@ module API include TimeZoneHelper expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } - expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization, :job_title, :pronouns + expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :discord, :website_url, :organization, :job_title, :pronouns expose :bot?, as: :bot expose :work_information do |user| work_information(user) diff --git a/lib/api/entities/wiki_page.rb b/lib/api/entities/wiki_page.rb index 07ef4a4a156..9d2a031cee8 100644 --- a/lib/api/entities/wiki_page.rb +++ b/lib/api/entities/wiki_page.rb @@ -25,3 +25,5 @@ module API end end end + +API::Entities::WikiPage.prepend_mod diff --git a/lib/api/events.rb b/lib/api/events.rb index d3e8892f3bc..5428692e3ec 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -8,7 +8,7 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } - feature_category :users + feature_category :user_profile urgency :low resource :events do diff --git a/lib/api/group_debian_distributions.rb b/lib/api/group_debian_distributions.rb index 8b6d4b8c4b2..9946199f7a8 100644 --- a/lib/api/group_debian_distributions.rb +++ b/lib/api/group_debian_distributions.rb @@ -6,10 +6,6 @@ module API requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the group' end - before do - not_found! if Gitlab::FIPS.enabled? - end - resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do after_validation do require_packages_enabled! diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 38430aac455..aadcbe38b15 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -504,12 +504,12 @@ module API def render_validation_error!(model, status = 400) if model.errors.any? - render_api_error!(model_error_messages(model) || '400 Bad Request', status) + render_api_error!(model_errors(model).messages || '400 Bad Request', status) end end - def model_error_messages(model) - model.errors.messages + def model_errors(model) + model.errors end def render_api_error_with_reason!(status, message, reason) @@ -608,7 +608,7 @@ module API if file.file_storage? present_disk_file!(file.path, file.filename) elsif supports_direct_download && file.class.direct_download_enabled? - return redirect(signed_head_url(file)) if head_request_on_aws_file?(file) + return redirect(ObjectStorage::S3.signed_head_url(file)) if request.head? && file.fog_credentials[:provider] == 'AWS' redirect(cdn_fronted_url(file)) else @@ -691,15 +691,7 @@ module API {} end - def validate_anonymous_search_access! - return if current_user.present? || Feature.disabled?(:disable_anonymous_search, type: :ops) - - unprocessable_entity!('User must be authenticated to use search') - end - def validate_search_rate_limit! - return unless Feature.enabled?(:rate_limit_issuable_searches) - if current_user check_rate_limit!(:search_rate_limit, scope: [current_user]) else @@ -709,19 +701,6 @@ module API private - def head_request_on_aws_file?(file) - request.head? && file.fog_credentials[:provider] == 'AWS' - end - - def signed_head_url(file) - fog_storage = ::Fog::Storage.new(file.fog_credentials) - fog_dir = fog_storage.directories.new(key: file.fog_directory) - fog_file = fog_dir.files.new(key: file.path) - expire_at = ::Fog::Time.now + file.fog_authenticated_url_expiration - - fog_file.collection.head_url(fog_file.key, expire_at) - end - # rubocop:disable Gitlab/ModuleWithInstanceVariables def initial_current_user return @initial_current_user if defined?(@initial_current_user) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 56db6ee4c5c..816b8deb461 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -58,7 +58,9 @@ module API def log_user_activity(actor) commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS - ::Users::ActivityService.new(actor).execute if commands.include?(params[:action]) + return unless commands.include?(params[:action]) + + ::Users::ActivityService.new(author: actor, namespace: project&.namespace, project: project).execute end def redis_ping diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index b0ea4388d9b..4b34a2bbe79 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -11,6 +11,9 @@ module API params :optional_state_filter_ee do end + params :optional_put_params_ee do + end + def find_source(source_type, id) public_send("find_#{source_type}!", id) # rubocop:disable GitlabSecurity/PublicSend end @@ -98,6 +101,10 @@ module API user_id.present? end + def self.member_access_levels + Gitlab::Access.all_values + end + private def member_already_exists?(source, user_id) diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index 1d35c316913..0fb3a19b8fd 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -86,7 +86,9 @@ module API end def track_package_event(action, scope, **args) - ::Packages::CreateEventService.new(nil, current_user, event_name: action, scope: scope).execute + service = ::Packages::CreateEventService.new(nil, current_user, event_name: action, scope: scope) + service.execute + category = args.delete(:category) || self.options[:for].name event_name = "i_package_#{scope}_user" ::Gitlab::Tracking.event( @@ -97,12 +99,40 @@ module API context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context], **args ) + + return unless Feature.enabled?(:route_hll_to_snowplow_phase3) + + if action.to_s == 'push_package' && service.originator_type == :deploy_token + track_snowplow_event("push_package_by_deploy_token", category, args) + elsif action.to_s == 'pull_package' && service.originator_type == :guest + track_snowplow_event("pull_package_by_guest", category, args) + end end def present_package_file!(package_file, supports_direct_download: true) package_file.package.touch_last_downloaded_at present_carrierwave_file!(package_file.file, supports_direct_download: supports_direct_download) end + + private + + def track_snowplow_event(action_name, category, args) + event_name = "i_package_#{action_name}" + key_path = "counts.package_events_i_package_#{action_name}" + service_ping_context = Gitlab::Tracking::ServicePingContext.new( + data_source: :redis, + key_path: key_path + ).to_context + + Gitlab::Tracking.event( + category, + action_name, + property: event_name, + label: key_path, + context: [service_ping_context], + **args + ) + end end end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index c5636fa06de..2700ea90d59 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -32,7 +32,7 @@ module API optional :builds_access_level, type: String, values: %w(disabled private enabled), desc: 'Builds access level. One of `disabled`, `private` or `enabled`' optional :snippets_access_level, type: String, values: %w(disabled private enabled), desc: 'Snippets access level. One of `disabled`, `private` or `enabled`' optional :pages_access_level, type: String, values: %w(disabled private enabled public), desc: 'Pages access level. One of `disabled`, `private`, `enabled` or `public`' - optional :operations_access_level, type: String, values: %w(disabled private enabled), desc: 'Operations access level. One of `disabled`, `private` or `enabled`' + optional :operations_access_level, type: String, values: %w(disabled private enabled), desc: 'Operations access level. One of `disabled`, `private` or `enabled`. Deprecated in GitLab 15.8, see https://gitlab.com/gitlab-org/gitlab/-/issues/385798.' optional :analytics_access_level, type: String, values: %w(disabled private enabled), desc: 'Analytics access level. One of `disabled`, `private` or `enabled`' optional :container_registry_access_level, type: String, values: %w(disabled private enabled), desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry' optional :security_and_compliance_access_level, type: String, values: %w(disabled private enabled), desc: 'Security and compliance access level. One of `disabled`, `private` or `enabled`' @@ -48,6 +48,7 @@ module API optional :warn_about_potentially_unwanted_characters, type: Boolean, desc: 'Warn about Potentially Unwanted Characters' optional :enforce_auth_checks_on_uploads, type: Boolean, desc: 'Enforce auth check on uploads' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :group_runners_enabled, type: Boolean, desc: 'Flag indication if group runners are enabled for that project' optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diff threads on lines changed with a push' optional :remove_source_branch_after_merge, type: Boolean, desc: 'Remove the source branch by default after merge' optional :container_registry_enabled, type: Boolean, desc: 'Deprecated: Use :container_registry_access_level instead. Flag indication if the container registry is enabled for that project' @@ -170,6 +171,7 @@ module API :security_and_compliance_access_level, :squash_option, :shared_runners_enabled, + :group_runners_enabled, :snippets_access_level, :tag_list, :topics, diff --git a/lib/api/helpers/users_helpers.rb b/lib/api/helpers/users_helpers.rb index e80b89488a2..f97071d9a97 100644 --- a/lib/api/helpers/users_helpers.rb +++ b/lib/api/helpers/users_helpers.rb @@ -12,10 +12,14 @@ module API params :optional_index_params_ee do end - def model_error_messages(model) - super.tap do |error_messages| + def model_errors(model) + super.tap do |errors| # Remapping errors from nested associations. - error_messages[:bio] = error_messages.delete(:"user_detail.bio") if error_messages.has_key?(:"user_detail.bio") + next unless errors.has_key?(:"user_detail.bio") + + errors.delete(:"user_detail.bio").each do |message| + errors.add(:bio, message) + end end end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 777d5019a29..5f12275b7a0 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -118,7 +118,7 @@ module API end end - namespace 'kubernetes/agent_configuration', urgency: :low do + namespace 'kubernetes/agent_configuration' do desc 'POST agent configuration' do detail 'Store configuration for an agent' end @@ -126,7 +126,7 @@ module API requires :agent_id, type: Integer, desc: 'ID of the configured Agent' requires :agent_config, type: JSON, desc: 'Configuration for the Agent' end - post '/', feature_category: :kubernetes_management do + post '/', feature_category: :kubernetes_management, urgency: :low do agent = ::Clusters::Agent.find(params[:agent_id]) ::Clusters::Agents::RefreshAuthorizationService.new(agent, config: params[:agent_config]).execute diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 872dab26469..828f4b419ef 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -4,7 +4,7 @@ module API class Invitations < ::API::Base include PaginationParams - feature_category :users + feature_category :user_profile before { authenticate! } @@ -21,7 +21,7 @@ module API tags %w[invitations] end params do - requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' + requires :access_level, type: Integer, values: ::API::Helpers::MembersHelpers.member_access_levels, desc: 'A valid access level (defaults: `30`, developer access level)' optional :email, type: Array[String], email_or_email_list: true, coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The email address to invite, or multiple emails separated by comma' optional :user_id, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index 020b02248a0..4c79aa2587f 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -55,7 +55,7 @@ module API requires :target_project_id, types: [String, Integer], desc: 'The ID or URL-encoded path of a target project' requires :target_issue_iid, types: [String, Integer], desc: 'The internal ID of a target project’s issue' - optional :link_type, type: String, values: IssueLink.link_types.keys, + optional :link_type, type: String, values: IssueLink.available_link_types, desc: 'The type of the relation (“relates_to”, “blocks”, “is_blocked_by”),'\ 'defaults to “relates_to”)' end @@ -72,9 +72,7 @@ module API .execute if result[:status] == :success - issue_link = IssueLink.find_by!(source: source_issue, target: target_issue) - - present issue_link, with: Entities::IssueLink + present result[:created_references].first, with: Entities::IssueLink else render_api_error!(result[:message], result[:http_status]) end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 7b6306938cf..88e99b29587 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -115,7 +115,6 @@ module API end get '/issues_statistics' do authenticate! unless params[:scope] == 'all' - validate_anonymous_search_access! if params[:search].present? validate_search_rate_limit! if declared_params[:search].present? present issues_statistics, with: Grape::Presenters::Presenter @@ -134,7 +133,6 @@ module API end get do authenticate! unless params[:scope] == 'all' - validate_anonymous_search_access! if params[:search].present? validate_search_rate_limit! if declared_params[:search].present? issues = paginate(find_issues) @@ -174,7 +172,6 @@ module API optional :non_archived, type: Boolean, desc: 'Return issues from non archived projects', default: true end get ":id/issues" do - validate_anonymous_search_access! if declared_params[:search].present? validate_search_rate_limit! if declared_params[:search].present? issues = paginate(find_issues(group_id: user_group.id, include_subgroups: true)) @@ -194,7 +191,6 @@ module API use :issues_stats_params end get ":id/issues_statistics" do - validate_anonymous_search_access! if declared_params[:search].present? validate_search_rate_limit! if declared_params[:search].present? present issues_statistics(group_id: user_group.id, include_subgroups: true), with: Grape::Presenters::Presenter @@ -214,7 +210,6 @@ module API use :issues_params end get ":id/issues" do - validate_anonymous_search_access! if declared_params[:search].present? validate_search_rate_limit! if declared_params[:search].present? issues = paginate(find_issues(project_id: user_project.id)) @@ -234,7 +229,6 @@ module API use :issues_stats_params end get ":id/issues_statistics" do - validate_anonymous_search_access! if declared_params[:search].present? validate_search_rate_limit! if declared_params[:search].present? present issues_statistics(project_id: user_project.id), with: Grape::Presenters::Presenter @@ -278,7 +272,7 @@ module API begin spam_params = ::Spam::SpamParams.new_from_request(request: request) - result = ::Issues::CreateService.new(project: user_project, + result = ::Issues::CreateService.new(container: user_project, current_user: current_user, params: issue_params, spam_params: spam_params).execute @@ -325,7 +319,7 @@ module API update_params = convert_parameters_from_legacy_format(update_params) spam_params = ::Spam::SpamParams.new_from_request(request: request) - issue = ::Issues::UpdateService.new(project: user_project, + issue = ::Issues::UpdateService.new(container: user_project, current_user: current_user, params: update_params, spam_params: spam_params).execute(issue) @@ -356,7 +350,7 @@ module API authorize! :update_issue, issue - if ::Issues::ReorderService.new(project: user_project, current_user: current_user, params: params).execute(issue) + if ::Issues::ReorderService.new(container: user_project, current_user: current_user, params: params).execute(issue) present issue, with: Entities::Issue, current_user: current_user, project: user_project else render_api_error!({ error: 'Unprocessable Entity' }, 422) @@ -382,7 +376,7 @@ module API not_found!('Project') unless new_project begin - issue = ::Issues::MoveService.new(project: user_project, current_user: current_user).execute(issue, new_project) + issue = ::Issues::MoveService.new(container: user_project, current_user: current_user).execute(issue, new_project) present issue, with: Entities::Issue, current_user: current_user, project: user_project rescue ::Issues::MoveService::MoveError => error render_api_error!(error.message, 400) @@ -409,7 +403,7 @@ module API not_found!('Project') unless target_project begin - issue = ::Issues::CloneService.new(project: user_project, current_user: current_user) + issue = ::Issues::CloneService.new(container: user_project, current_user: current_user) .execute(issue, target_project, with_notes: params[:with_notes]) present issue, with: Entities::Issue, current_user: current_user, project: target_project rescue ::Issues::CloneService::CloneError => error @@ -430,7 +424,7 @@ module API authorize!(:destroy_issue, issue) destroy_conditionally!(issue) do |issue| - Issuable::DestroyService.new(project: user_project, current_user: current_user).execute(issue) + Issuable::DestroyService.new(container: user_project, current_user: current_user).execute(issue) end end # rubocop: enable CodeReuse/ActiveRecord @@ -444,9 +438,10 @@ module API get ':id/issues/:issue_iid/related_merge_requests' do issue = find_project_issue(params[:issue_iid]) - merge_requests = ::Issues::ReferencedMergeRequestsService.new(project: user_project, current_user: current_user) - .execute(issue) - .first + merge_requests = ::Issues::ReferencedMergeRequestsService + .new(container: user_project, current_user: current_user) + .execute(issue) + .first present paginate(::Kaminari.paginate_array(merge_requests)), with: Entities::MergeRequest, diff --git a/lib/api/members.rb b/lib/api/members.rb index 32c5227a939..1e640a6542a 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -139,6 +139,7 @@ module API requires :user_id, type: Integer, desc: 'The user ID of the new member' requires :access_level, type: Integer, desc: 'A valid access level' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + use :optional_put_params_ee end # rubocop: disable CodeReuse/ActiveRecord put ":id/members/:user_id", feature_category: feature_category do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 25fbeca01dc..cd46b442b68 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -140,7 +140,6 @@ module API end get feature_category: :code_review_workflow, urgency: :low do authenticate! unless params[:scope] == 'all' - validate_anonymous_search_access! if params[:search].present? validate_search_rate_limit! if declared_params[:search].present? merge_requests = find_merge_requests @@ -169,7 +168,6 @@ module API desc: 'Returns merge requests from non archived projects only.' end get ":id/merge_requests", feature_category: :code_review_workflow, urgency: :low do - validate_anonymous_search_access! if declared_params[:search].present? validate_search_rate_limit! if declared_params[:search].present? merge_requests = find_merge_requests(group_id: user_group.id, include_subgroups: true) @@ -237,7 +235,6 @@ module API end get ":id/merge_requests", feature_category: :code_review_workflow, urgency: :low do authorize! :read_merge_request, user_project - validate_anonymous_search_access! if declared_params[:search].present? validate_search_rate_limit! if declared_params[:search].present? merge_requests = find_merge_requests(project_id: user_project.id) @@ -315,7 +312,7 @@ module API authorize!(:destroy_merge_request, merge_request) destroy_conditionally!(merge_request) do |merge_request| - Issuable::DestroyService.new(project: user_project, current_user: current_user).execute(merge_request) + Issuable::DestroyService.new(container: user_project, current_user: current_user).execute(merge_request) end end @@ -627,8 +624,9 @@ module API merge_request = find_project_merge_request(params[:merge_request_iid]) - # Merge request can not be merged - # because user dont have permissions to push into target branch + # Merge request can not be merged because the user doesn't have + # permissions to push into target branch + # unauthorized! unless merge_request.can_be_merged_by?(current_user) merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 2b1007e715a..c971f73ccbb 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -81,13 +81,13 @@ module API tags NAMESPACES_TAGS end params do - requires :namespace, type: String, desc: "Namespace’s path" + requires :id, type: String, desc: "Namespace’s path" optional :parent_id, type: Integer, desc: 'The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered.' end - get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do + get ':id/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do check_rate_limit!(:namespace_exists, scope: current_user) - namespace_path = params[:namespace] + namespace_path = params[:id] existing_namespaces_within_the_parent = Namespace.without_project_namespaces.by_parent(params[:parent_id]) exists = existing_namespaces_within_the_parent.filter_by_path(namespace_path).exists? diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 15c1a78839f..db156186458 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -94,7 +94,7 @@ module API end params do requires :domain, type: String, desc: 'The domain' - # rubocop:disable Scalability/FileUploads + # rubocop:todo Scalability/FileUploads # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 optional :certificate, types: [File, String], desc: 'The certificate', as: :user_provided_certificate optional :key, types: [File, String], desc: 'The key', as: :user_provided_key @@ -122,7 +122,7 @@ module API desc 'Updates a pages domain' params do requires :domain, type: String, desc: 'The domain' - # rubocop:disable Scalability/FileUploads + # rubocop:todo Scalability/FileUploads # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 optional :certificate, types: [File, String], desc: 'The certificate', as: :user_provided_certificate optional :key, types: [File, String], desc: 'The key', as: :user_provided_key diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index e5e6ccdf025..0f4f1dc7fa6 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -12,7 +12,7 @@ module API before { authorize_read_container_images! } - feature_category :package_registry + feature_category :container_registry urgency :low params do diff --git a/lib/api/project_debian_distributions.rb b/lib/api/project_debian_distributions.rb index 856b4097b5a..fc94107f88d 100644 --- a/lib/api/project_debian_distributions.rb +++ b/lib/api/project_debian_distributions.rb @@ -6,10 +6,6 @@ module API requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end - before do - not_found! if Gitlab::FIPS.enabled? - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do after_validation do require_packages_enabled! diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb index d90ce32c354..3e651cdda44 100644 --- a/lib/api/project_events.rb +++ b/lib/api/project_events.rb @@ -6,7 +6,7 @@ module API include APIGuard helpers ::API::Helpers::EventsHelpers - feature_category :users + feature_category :user_profile # TODO: Set higher urgency after resolving https://gitlab.com/gitlab-org/gitlab/-/issues/357839 urgency :low diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 5077f02fcc1..6eea56ea117 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -9,7 +9,11 @@ module API before { authenticate_non_get! } - feature_category :projects, ['/projects/:id/custom_attributes', '/projects/:id/custom_attributes/:key'] + feature_category :projects, %w[ + /projects/:id/custom_attributes + /projects/:id/custom_attributes/:key + /projects/:id/share_locations + ] PROJECT_ATTACHMENT_SIZE_EXEMPT = 1.gigabyte @@ -351,6 +355,20 @@ module API render_validation_error!(project) end end + + desc 'Returns group that can be shared with the given project' do + success Entities::Group + end + params do + requires :id, type: Integer, desc: 'The id of the project' + optional :search, type: String, desc: 'Return list of groups matching the search criteria' + end + get ':id/share_locations' do + groups = ::Groups::AcceptingProjectSharesFinder.new(current_user, user_project, declared_params(include_missing: false)).execute + + present_groups groups + end + # rubocop: enable CodeReuse/ActiveRecord end @@ -850,11 +868,23 @@ module API ] tags %w[projects] end + params do + optional :task, type: Symbol, default: :eager, values: %i[eager prune], desc: '`prune` to trigger manual prune of unreachable objects or `eager` to trigger eager housekeeping.' + end post ':id/housekeeping', feature_category: :source_code_management do authorize_admin_project begin - ::Repositories::HousekeepingService.new(user_project, :gc).execute + ::Repositories::HousekeepingService.new(user_project, params[:task]).execute do + ::Gitlab::Audit::Auditor.audit( + name: 'manually_trigger_housekeeping', + author: current_user, + scope: user_project, + target: user_project, + message: "Housekeeping task: #{params[:task]}", + created_at: DateTime.current + ) + end rescue ::Repositories::HousekeepingService::LeaseTaken => error conflict!(error.message) end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index e69dc756551..ebf1c03e86b 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -246,8 +246,14 @@ module API optional :milestones, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, - desc: 'The title of each milestone the release is associated with. GitLab Premium customers can specify group milestones', - default: [] + desc: 'The title of each milestone the release is associated with. GitLab Premium customers can specify group milestones. Cannot be combined with `milestone_ids` parameter.' + + optional :milestone_ids, + type: Array[String, Integer], + coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, + desc: 'The ID of each milestone the release is associated with. GitLab Premium customers can specify group milestones. Cannot be combined with `milestones` parameter.' + + mutually_exclusive :milestones, :milestone_ids, message: 'Cannot specify milestones and milestone_ids at the same time' optional :released_at, type: DateTime, @@ -292,7 +298,14 @@ module API optional :milestones, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, - desc: 'The title of each milestone to associate with the release. GitLab Premium customers can specify group milestones. To remove all milestones from the release, specify `[]`' + desc: 'The title of each milestone to associate with the release. GitLab Premium customers can specify group milestones. Cannot be combined with `milestone_ids` parameter. To remove all milestones from the release, specify `[]`' + + optional :milestone_ids, + type: Array[String, Integer], + coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, + desc: 'The ID of each milestone the release is associated with. GitLab Premium customers can specify group milestones. Cannot be combined with `milestones` parameter. To remove all milestones from the release, specify `[]`' + + mutually_exclusive :milestones, :milestone_ids, message: 'Cannot specify milestones and milestone_ids at the same time' end route_setting :authentication, job_token_allowed: true put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 06b576a982b..8efb848eb57 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -191,8 +191,8 @@ module API optional :group_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for group runners, in seconds' optional :project_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for project runners, in seconds' optional :pipeline_limit_per_project_user_sha, type: Integer, desc: "Maximum number of pipeline creation requests allowed per minute per user and commit. Set to 0 for unlimited requests per minute." - optional :jira_connect_application_key, type: String, desc: "Application ID of the OAuth application that should be used to authenticate with the GitLab.com for Jira Cloud app" - optional :jira_connect_proxy_url, type: String, desc: "URL of the GitLab instance that should be used as a proxy for the GitLab.com for Jira Cloud app" + optional :jira_connect_application_key, type: String, desc: "Application ID of the OAuth application that should be used to authenticate with the GitLab for Jira Cloud app" + optional :jira_connect_proxy_url, type: String, desc: "URL of the GitLab instance that should be used as a proxy for the GitLab for Jira Cloud app" optional :bulk_import_enabled, type: Boolean, desc: 'Enable migrating GitLab groups and projects by direct transfer' optional :allow_runner_registration_token, type: Boolean, desc: 'Allow registering runners using a registration token' diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index dd8ad2cc144..3534edc3831 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -35,7 +35,9 @@ module API custom_params = declared_params(include_missing: false) custom_params.merge!(attrs) - issuable = update_service.new(project: user_project, current_user: current_user, params: custom_params).execute(load_issuable) + issuable = update_service.new(**update_service.constructor_container_arg(user_project), + current_user: current_user, params: custom_params).execute(load_issuable) + if issuable.valid? present issuable, with: Entities::IssuableTimeStats else diff --git a/lib/api/users.rb b/lib/api/users.rb index a9b09596728..cc7eb63798a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -8,7 +8,7 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } - feature_category :users, + feature_category :user_profile, %w[ /users/:id/custom_attributes /users/:id/custom_attributes/:key @@ -46,6 +46,7 @@ module API optional :skype, type: String, desc: 'The Skype username' optional :linkedin, type: String, desc: 'The LinkedIn username' optional :twitter, type: String, desc: 'The Twitter username' + optional :discord, type: String, desc: 'The Discord user ID' optional :website_url, type: String, desc: 'The website of the user' optional :organization, type: String, desc: 'The organization of the user' optional :projects_limit, type: Integer, desc: 'The number of projects a user can create' @@ -131,7 +132,7 @@ module API use :optional_index_params_ee end # rubocop: disable CodeReuse/ActiveRecord - get feature_category: :users, urgency: :low do + get feature_category: :user_profile, urgency: :low do authenticated_as_admin! if params[:extern_uid].present? && params[:provider].present? unless current_user&.can_read_all_resources? @@ -175,7 +176,7 @@ module API use :with_custom_attributes end # rubocop: disable CodeReuse/ActiveRecord - get ":id", feature_category: :users, urgency: :low do + get ":id", feature_category: :user_profile, urgency: :low do forbidden!('Not authorized!') unless current_user unless current_user.can_read_all_resources? @@ -200,7 +201,7 @@ module API params do requires :user_id, type: String, desc: 'The ID or username of the user' end - get ":user_id/status", requirements: API::USER_REQUIREMENTS, feature_category: :users, urgency: :default do + get ":user_id/status", requirements: API::USER_REQUIREMENTS, feature_category: :user_profile, urgency: :default do user = find_user(params[:user_id]) not_found!('User') unless user && can?(current_user, :read_user, user) @@ -214,7 +215,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of the user' end - post ':id/follow', feature_category: :users do + post ':id/follow', feature_category: :user_profile do user = find_user(params[:id]) not_found!('User') unless user @@ -234,7 +235,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of the user' end - post ':id/unfollow', feature_category: :users do + post ':id/unfollow', feature_category: :user_profile do user = find_user(params[:id]) not_found!('User') unless user @@ -252,7 +253,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' use :pagination end - get ':id/following', feature_category: :users do + get ':id/following', feature_category: :user_profile do forbidden!('Not authorized!') unless current_user user = find_user(params[:id]) @@ -268,7 +269,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' use :pagination end - get ':id/followers', feature_category: :users do + get ':id/followers', feature_category: :user_profile do forbidden!('Not authorized!') unless current_user user = find_user(params[:id]) @@ -291,7 +292,7 @@ module API optional :force_random_password, type: Boolean, desc: 'Flag indicating a random password will be set' use :optional_attributes end - post feature_category: :users do + post feature_category: :user_profile do authenticated_as_admin! params = declared_params(include_missing: false) @@ -333,7 +334,7 @@ module API use :optional_attributes end # rubocop: disable CodeReuse/ActiveRecord - put ":id", feature_category: :users do + put ":id", feature_category: :user_profile do authenticated_as_admin! user = User.find_by(id: params.delete(:id)) @@ -644,7 +645,7 @@ module API optional :skip_confirmation, type: Boolean, desc: 'Skip confirmation of email and assume it is verified' end # rubocop: disable CodeReuse/ActiveRecord - post ":id/emails", feature_category: :users do + post ":id/emails", feature_category: :user_profile do authenticated_as_admin! user = User.find_by(id: params.delete(:id)) @@ -668,7 +669,7 @@ module API use :pagination end # rubocop: disable CodeReuse/ActiveRecord - get ':id/emails', feature_category: :users do + get ':id/emails', feature_category: :user_profile do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -685,7 +686,7 @@ module API requires :email_id, type: Integer, desc: 'The ID of the email' end # rubocop: disable CodeReuse/ActiveRecord - delete ':id/emails/:email_id', feature_category: :users do + delete ':id/emails/:email_id', feature_category: :user_profile do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -707,7 +708,7 @@ module API optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end # rubocop: disable CodeReuse/ActiveRecord - delete ":id", feature_category: :users do + delete ":id", feature_category: :user_profile do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -883,7 +884,7 @@ module API optional :type, type: String, values: %w[Project Namespace] use :pagination end - get ":user_id/memberships", feature_category: :users, urgency: :high do + get ":user_id/memberships", feature_category: :user_profile, urgency: :high do authenticated_as_admin! user = find_user_by_id(params) @@ -1021,7 +1022,7 @@ module API desc 'Get the currently authenticated user' do success Entities::UserPublic end - get feature_category: :users, urgency: :low do + get feature_category: :user_profile, urgency: :low do entity = # We're disabling Cop/UserAdmin because it checks if the given user is an admin. if current_user.admin? # rubocop:disable Cop/UserAdmin @@ -1202,7 +1203,7 @@ module API params do use :pagination end - get "emails", feature_category: :users, urgency: :high do + get "emails", feature_category: :user_profile, urgency: :high do present paginate(current_user.emails), with: Entities::Email end @@ -1244,7 +1245,7 @@ module API optional :show_whitespace_in_diffs, type: Boolean, desc: 'Flag indicating the user sees whitespace changes in diffs' at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs end - put "preferences", feature_category: :users, urgency: :high do + put "preferences", feature_category: :user_profile, urgency: :high do authenticate! preferences = current_user.user_preference @@ -1263,7 +1264,7 @@ module API success Entities::UserPreferences detail 'This feature was introduced in GitLab 14.0.' end - get "preferences", feature_category: :users do + get "preferences", feature_category: :user_profile do present current_user.user_preference, with: Entities::UserPreferences end @@ -1274,7 +1275,7 @@ module API requires :email_id, type: Integer, desc: 'The ID of the email' end # rubocop: disable CodeReuse/ActiveRecord - get "emails/:email_id", feature_category: :users do + get "emails/:email_id", feature_category: :user_profile do email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email @@ -1288,7 +1289,7 @@ module API params do requires :email, type: String, desc: 'The new email' end - post "emails", feature_category: :users do + post "emails", feature_category: :user_profile do email = Emails::CreateService.new(current_user, declared_params.merge(user: current_user)).execute if email.errors.blank? @@ -1303,7 +1304,7 @@ module API requires :email_id, type: Integer, desc: 'The ID of the email' end # rubocop: disable CodeReuse/ActiveRecord - delete "emails/:email_id", feature_category: :users do + delete "emails/:email_id", feature_category: :user_profile do email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email @@ -1319,7 +1320,7 @@ module API use :pagination end # rubocop: disable CodeReuse/ActiveRecord - get "activities", feature_category: :users do + get "activities", feature_category: :user_profile do authenticated_as_admin! activities = User @@ -1337,7 +1338,7 @@ module API params do use :set_user_status_params end - put "status", feature_category: :users do + put "status", feature_category: :user_profile do set_user_status(include_missing_params: true) end @@ -1348,7 +1349,7 @@ module API params do use :set_user_status_params end - patch "status", feature_category: :users do + patch "status", feature_category: :user_profile do if declared_params(include_missing: false).empty? status :ok @@ -1361,7 +1362,7 @@ module API desc 'get the status of the current user' do success Entities::UserStatus end - get 'status', feature_category: :users do + get 'status', feature_category: :user_profile do present current_user.status || {}, with: Entities::UserStatus end end diff --git a/lib/api/validations/validators/bulk_imports.rb b/lib/api/validations/validators/bulk_imports.rb index 8d49607f64c..4625f2f39cd 100644 --- a/lib/api/validations/validators/bulk_imports.rb +++ b/lib/api/validations/validators/bulk_imports.rb @@ -21,7 +21,7 @@ module API def validate_param!(attr_name, params) return if params[attr_name].blank? - unless params[attr_name] =~ Gitlab::Regex.bulk_import_namespace_path_regex # rubocop: disable Style/GuardClause + unless params[attr_name] =~ Gitlab::Regex.bulk_import_destination_namespace_path_regex # rubocop: disable Style/GuardClause raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], message: "cannot start with a dash or forward slash, or end with a period or forward slash. " \ @@ -34,7 +34,7 @@ module API class SourceFullPath < Grape::Validations::Base def validate_param!(attr_name, params) - unless params[attr_name] =~ Gitlab::Regex.bulk_import_namespace_path_regex # rubocop: disable Style/GuardClause + unless params[attr_name] =~ Gitlab::Regex.bulk_import_source_full_path_regex # rubocop: disable Style/GuardClause raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], message: "must be a relative path and not include protocol, sub-domain, or domain information. " \ diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index 96e285f4eea..3922499f3a7 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -158,7 +158,7 @@ module Atlassian end def request_body_schema(response) - Oj.load(response.request.raw_body).deep_transform_values! {} + Oj.load(response.request.raw_body) rescue Oj::ParseError, EncodingError, Encoding::UndefinedConversionError 'Request body includes invalid JSON' end diff --git a/lib/atlassian/jira_issue_key_extractor.rb b/lib/atlassian/jira_issue_key_extractor.rb index f1b432787ac..968e8b0f82e 100644 --- a/lib/atlassian/jira_issue_key_extractor.rb +++ b/lib/atlassian/jira_issue_key_extractor.rb @@ -2,8 +2,8 @@ module Atlassian class JiraIssueKeyExtractor - def self.has_keys?(*text) - new(*text).issue_keys.any? + def self.has_keys?(...) + new(...).issue_keys.any? end def initialize(*text) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index cf19b4fa8ff..bd3832c7327 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -6,7 +6,7 @@ module Backup class Database < Task extend ::Gitlab::Utils::Override include Backup::Helper - attr_reader :force, :config + attr_reader :force IGNORED_ERRORS = [ # Ignore warnings @@ -18,98 +18,105 @@ module Backup ].freeze IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze - def initialize(database_name, progress, force:) + def initialize(progress, force:) super(progress) - @database_name = database_name - @config = base_model.connection_db_config.configuration_hash @force = force end override :dump - def dump(db_file_name, backup_id) - FileUtils.mkdir_p(File.dirname(db_file_name)) - FileUtils.rm_f(db_file_name) - compress_rd, compress_wr = IO.pipe - compress_pid = spawn(gzip_cmd, in: compress_rd, out: [db_file_name, 'w', 0600]) - compress_rd.close - - dump_pid = - case config[:adapter] - when "postgresql" then - progress.print "Dumping PostgreSQL database #{database} ... " - pg_env - pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. - pgsql_args << '--if-exists' - - if Gitlab.config.backup.pg_schema - pgsql_args << '-n' - pgsql_args << Gitlab.config.backup.pg_schema + def dump(destination_dir, backup_id) + FileUtils.mkdir_p(destination_dir) - Gitlab::Database::EXTRA_SCHEMAS.each do |schema| - pgsql_args << '-n' - pgsql_args << schema.to_s - end - end + snapshot_ids.each do |database_name, snapshot_id| + base_model = base_models_for_backup[database_name] + + config = base_model.connection_db_config.configuration_hash + + db_file_name = file_name(destination_dir, database_name) + FileUtils.rm_f(db_file_name) + + pg_database = config[:database] - Process.spawn('pg_dump', *pgsql_args, database, out: compress_wr) + progress.print "Dumping PostgreSQL database #{pg_database} ... " + pg_env(config) + pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. + pgsql_args << '--if-exists' + pgsql_args << "--snapshot=#{snapshot_ids[database_name]}" + + if Gitlab.config.backup.pg_schema + pgsql_args << '-n' + pgsql_args << Gitlab.config.backup.pg_schema + + Gitlab::Database::EXTRA_SCHEMAS.each do |schema| + pgsql_args << '-n' + pgsql_args << schema.to_s + end end - compress_wr.close - success = [compress_pid, dump_pid].all? do |pid| - Process.waitpid(pid) - $?.success? - end + success = Backup::Dump::Postgres.new.dump(pg_database, db_file_name, pgsql_args) + + base_model.connection.rollback_transaction - report_success(success) - progress.flush + raise DatabaseBackupError.new(config, db_file_name) unless success - raise DatabaseBackupError.new(config, db_file_name) unless success + report_success(success) + progress.flush + end + ensure + base_models_for_backup.each do |_database_name, base_model| + Gitlab::Database::TransactionTimeoutSettings.new(base_model.connection).restore_timeouts + end end override :restore - def restore(db_file_name) - unless File.exist?(db_file_name) - raise(Backup::Error, "Source database file does not exist #{db_file_name}") if main_database? + def restore(destination_dir) + base_models_for_backup.each do |database_name, base_model| + config = base_model.connection_db_config.configuration_hash - progress.puts "Source backup for the database #{@database_name} doesn't exist. Skipping the task" - return - end + db_file_name = file_name(destination_dir, database_name) + database = config[:database] - unless force - progress.puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) - sleep(5) - end + unless File.exist?(db_file_name) + raise(Backup::Error, "Source database file does not exist #{db_file_name}") if main_database?(database_name) - # Drop all tables Load the schema to ensure we don't have any newer tables - # hanging out from a failed upgrade - puts_time 'Cleaning the database ... '.color(:blue) - Rake::Task['gitlab:db:drop_tables'].invoke - puts_time 'done'.color(:green) - - decompress_rd, decompress_wr = IO.pipe - decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) - decompress_wr.close - - status, @errors = - case config[:adapter] - when "postgresql" then - progress.print "Restoring PostgreSQL database #{database} ... " - pg_env - execute_and_track_errors(pg_restore_cmd, decompress_rd) + progress.puts "Source backup for the database #{database_name} doesn't exist. Skipping the task" + return false end - decompress_rd.close - Process.waitpid(decompress_pid) - success = $?.success? && status.success? + unless force + progress.puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) + sleep(5) + end - if @errors.present? - progress.print "------ BEGIN ERRORS -----\n".color(:yellow) - progress.print @errors.join.color(:yellow) - progress.print "------ END ERRORS -------\n".color(:yellow) - end + # Drop all tables Load the schema to ensure we don't have any newer tables + # hanging out from a failed upgrade + drop_tables(database_name) - report_success(success) - raise Backup::Error, 'Restore failed' unless success + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close + + status, @errors = + case config[:adapter] + when "postgresql" then + progress.print "Restoring PostgreSQL database #{database} ... " + pg_env(config) + execute_and_track_errors(pg_restore_cmd(database), decompress_rd) + end + decompress_rd.close + + Process.waitpid(decompress_pid) + success = $?.success? && status.success? + + if @errors.present? + progress.print "------ BEGIN ERRORS -----\n".color(:yellow) + progress.print @errors.join.color(:yellow) + progress.print "------ END ERRORS -------\n".color(:yellow) + end + + report_success(success) + raise Backup::Error, 'Restore failed' unless success + end end override :pre_restore_warning @@ -144,16 +151,22 @@ module Backup protected - def database - @config[:database] + def base_models_for_backup + @base_models_for_backup ||= Gitlab::Database.database_base_models_with_gitlab_shared end - def base_model - Gitlab::Database.database_base_models[@database_name] + def main_database?(database_name) + database_name.to_sym == :main end - def main_database? - @database_name == :main + def file_name(base_dir, database_name) + prefix = if database_name.to_sym != :main + "#{database_name}_" + else + '' + end + + File.join(base_dir, "#{prefix}database.sql.gz") end def ignore_error?(line) @@ -189,7 +202,7 @@ module Backup end end - def pg_env + def pg_env(config) args = { username: 'PGUSER', host: 'PGHOST', @@ -223,8 +236,32 @@ module Backup private - def pg_restore_cmd + def drop_tables(database_name) + if Rake::Task.task_defined? "gitlab:db:drop_tables:#{database_name}" + puts_time 'Cleaning the database ... '.color(:blue) + Rake::Task["gitlab:db:drop_tables:#{database_name}"].invoke + puts_time 'done'.color(:green) + elsif Gitlab::Database.database_base_models.one? + # In single database, we do not have rake tasks per database + puts_time 'Cleaning the database ... '.color(:blue) + Rake::Task["gitlab:db:drop_tables"].invoke + puts_time 'done'.color(:green) + end + end + + def pg_restore_cmd(database) ['psql', database] end + + def snapshot_ids + @snapshot_ids ||= base_models_for_backup.each_with_object({}) do |(database_name, base_model), snapshot_ids| + Gitlab::Database::TransactionTimeoutSettings.new(base_model.connection).disable_timeouts + + base_model.connection.begin_transaction(isolation: :repeatable_read) + + snapshot_ids[database_name] = + base_model.connection.execute("SELECT pg_export_snapshot() as snapshot_id;").first['snapshot_id'] + end + end end end diff --git a/lib/backup/dump/postgres.rb b/lib/backup/dump/postgres.rb new file mode 100644 index 00000000000..1a5128b5a6b --- /dev/null +++ b/lib/backup/dump/postgres.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +module Backup + module Dump + class Postgres + include Backup::Helper + + FILE_PERMISSION = 0o600 + + def dump(database_name, output_file, pgsql_args) + compress_rd, compress_wr = IO.pipe + compress_pid = spawn(gzip_cmd, in: compress_rd, out: [output_file, 'w', FILE_PERMISSION]) + compress_rd.close + + dump_pid = Process.spawn('pg_dump', *pgsql_args, database_name, out: compress_wr) + compress_wr.close + + [compress_pid, dump_pid].all? do |pid| + Process.waitpid(pid) + $?.success? + end + end + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f8424f6250e..ba4a26ba714 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -22,7 +22,6 @@ module Backup :destination_optional, # `true` if the destination might not exist on a successful backup. :cleanup_path, # Path to remove after a successful backup. Uses `destination_path` when not specified. :task, - :task_group, keyword_init: true ) do def enabled? @@ -121,20 +120,11 @@ module Backup def build_definitions # rubocop:disable Metrics/AbcSize { - 'main_db' => TaskDefinition.new( - human_name: _('main_database'), - destination_path: 'db/database.sql.gz', + 'db' => TaskDefinition.new( + human_name: _('database'), + destination_path: 'db', cleanup_path: 'db', - task: build_db_task(:main), - task_group: 'db' - ), - 'ci_db' => TaskDefinition.new( - human_name: _('ci_database'), - destination_path: 'db/ci_database.sql.gz', - cleanup_path: 'db', - task: build_db_task(:ci), - enabled: Gitlab::Database.has_config?(:ci), - task_group: 'db' + task: build_db_task ), 'repositories' => TaskDefinition.new( human_name: _('repositories'), @@ -186,16 +176,15 @@ module Backup }.freeze end - def build_db_task(database_name) - return unless Gitlab::Database.has_config?(database_name) # It will be disabled for a single db setup - + def build_db_task force = Gitlab::Utils.to_boolean(ENV['force'], default: false) - Database.new(database_name, progress, force: force) + + Database.new(progress, force: force) end def build_repositories_task - max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence - max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence + max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence&.to_i + max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence&.to_i strategy = Backup::GitalyBackup.new(progress, incremental: incremental?, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) Repositories.new(progress, @@ -483,7 +472,7 @@ module Backup end def skipped?(item) - skipped.include?(item) || skipped.include?(definitions[item]&.task_group) + skipped.include?(item) end def skipped diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb index e5cf20d00df..8681173b1f4 100644 --- a/lib/banzai/filter/blockquote_fence_filter.rb +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -6,14 +6,14 @@ module Banzai REGEX = %r{ #{::Gitlab::Regex.markdown_code_or_html_blocks} | - (?=(?<=^\n|\A)>>>\ *\n.*\n>>>\ *(?=\n$|\z))(?: + (?=(?<=^\n|\A)\ *>>>\ *\n.*\n\ *>>>\ *(?=\n$|\z))(?: # Blockquote: # >>> # Anything, including code and HTML blocks # >>> - (?<=^\n|\A)>>>\ *\n - (?<quote> + (?<=^\n|\A)(?<indent>\ *)>>>\ *\n + (?<blockquote> (?: # Any character that doesn't introduce a code or HTML block (?! @@ -30,7 +30,7 @@ module Banzai \g<html> )+? ) - \n>>>\ *(?=\n$|\z) + \n\ *>>>\ *(?=\n$|\z) ) }mx.freeze @@ -41,10 +41,11 @@ module Banzai def call @text.gsub(REGEX) do - if $~[:quote] + if $~[:blockquote] # keep the same number of source lines/positions by replacing the # fence lines with newlines - "\n" + $~[:quote].gsub(/^/, "> ").gsub(/^> $/, ">") + "\n" + indent = $~[:indent] + "\n" + $~[:blockquote].gsub(/^#{Regexp.quote(indent)}/, "#{indent}> ").gsub(/^> $/, ">") + "\n" else $~[0] end diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb index 8c0bd62f80a..4d37fba33aa 100644 --- a/lib/banzai/filter/markdown_post_escape_filter.rb +++ b/lib/banzai/filter/markdown_post_escape_filter.rb @@ -7,11 +7,11 @@ module Banzai LITERAL_KEYWORD = MarkdownPreEscapeFilter::LITERAL_KEYWORD LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-(.*?)-#{LITERAL_KEYWORD}}.freeze NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze - SPAN_REGEX = %r{<span>(.*?)</span>}.freeze + SPAN_REGEX = %r{<span data-escaped-char>(.*?)</span>}.freeze - XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath('a').freeze - XPATH_LANG_TAG = Gitlab::Utils::Nokogiri.css_to_xpath('pre').freeze - XPATH_CODE_SPAN = Gitlab::Utils::Nokogiri.css_to_xpath('code > span').freeze + XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath('a').freeze + XPATH_LANG_TAG = Gitlab::Utils::Nokogiri.css_to_xpath('pre').freeze + XPATH_ESCAPED_CHAR = Gitlab::Utils::Nokogiri.css_to_xpath('span[data-escaped-char]').freeze def call return doc unless result[:escaped_literals] @@ -22,7 +22,7 @@ module Banzai @doc = parse_html(new_html) remove_spans_in_certain_attributes - remove_spans_in_code + remove_unnecessary_escapes doc end @@ -57,7 +57,7 @@ module Banzai escaped_item = Banzai::Filter::MarkdownPreEscapeFilter::ESCAPABLE_CHARS.find { |item| item[:token] == last_match_token } escaped_char = escaped_item ? escaped_item[:char] : ::Regexp.last_match(1) - "<span>#{escaped_char}</span>" + "<span data-escaped-char>#{escaped_char}</span>" end html @@ -75,14 +75,55 @@ module Banzai end end - # Any `<span>` that makes it into a `<code>` element is from the math processing, - # convert back to the escaped character, such as `\$` - def remove_spans_in_code - doc.xpath(XPATH_CODE_SPAN).each do |node| - escaped_item = Banzai::Filter::MarkdownPreEscapeFilter::ESCAPABLE_CHARS.find { |item| item[:char] == node.content && item[:latex] } + def remove_unnecessary_escapes + doc.xpath(XPATH_ESCAPED_CHAR).each do |node| + escaped_item = Banzai::Filter::MarkdownPreEscapeFilter::ESCAPABLE_CHARS.find { |item| item[:char] == node.content } + + next unless escaped_item + + if node.parent.name == 'code' + # For any `data-escaped-char` that makes it into a `<code>` element, + # convert back to the escaped character, such as `\$`. Usually this would + # only happen for dollar math + content = +escaped_item[:escaped] + elsif escaped_item[:latex] && !escaped_item[:reference] + # Character only used in latex, since it's outside of a code block we can + # transform into the regular character + content = +escaped_item[:char] + else + # Escaped reference character, so leave as is. This is so that our normal + # reference processing can be short-circuited by escaping the reference, + # like \@username + next + end + + merge_adjacent_text_nodes(node, content) + end + end + + def text_node?(node) + node.is_a?(Nokogiri::XML::Text) + end - node.replace(escaped_item[:escaped]) if escaped_item + # Merge directly adjacent text nodes and replace existing node with + # the merged content. For example, the document could be + # #(Text "~c_bug"), #(Element:0x57724 { name = "span" }, children = [ #(Text "_")] })] + # Our reference processing requires a single string of text to match against. So even if it was + # #(Text "~c_bug"), #(Text "_") + # it wouldn't match. Merging together will give + # #(Text "~c_bug_") + def merge_adjacent_text_nodes(node, content) + if text_node?(node.previous) + content.prepend(node.previous.content) + node.previous.remove end + + if text_node?(node.next) + content.concat(node.next.content) + node.next.remove + end + + node.replace(content) end end end diff --git a/lib/banzai/filter/playable_link_filter.rb b/lib/banzai/filter/playable_link_filter.rb index 5b290ff08f6..6030935f65d 100644 --- a/lib/banzai/filter/playable_link_filter.rb +++ b/lib/banzai/filter/playable_link_filter.rb @@ -24,7 +24,7 @@ module Banzai raise NotImplementedError end - def extra_element_attrs + def extra_element_attrs(element) {} end @@ -43,7 +43,7 @@ module Banzai controls: true, 'data-setup': '{}', 'data-title': element['title'] || element['alt'] - }.merge!(extra_element_attrs) + }.merge!(extra_element_attrs(element)) if element['data-canonical-src'] media_element_attrs['data-canonical-src'] = element['data-canonical-src'] diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index de9b846efe9..b119c2ffccf 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -20,6 +20,7 @@ module Banzai # Allow the 'data-sourcepos' from CommonMark on all elements allowlist[:attributes][:all].push('data-sourcepos') + allowlist[:attributes][:all].push('data-escaped-char') # Remove any `style` properties not required for table alignment allowlist[:transformers].push(self.class.remove_unsafe_table_style) diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 489b4d21300..3da6ce5c90c 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -72,8 +72,9 @@ module Banzai pre_node.set_attribute(LANG_PARAMS_ATTR, escape_once(lang_params)) if lang_params.present? pre_node.set_attribute('v-pre', 'true') pre_node.remove_attribute('data-meta') + copy_code_btn = "<copy-code></copy-code>" unless language == 'suggestion' - highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code">#{pre_node.to_html}<copy-code></copy-code></div>) + highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code">#{pre_node.to_html}#{copy_code_btn}</div>) # Extracted to a method to measure it replace_pre_element(pre_node, highlighted) diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index 3789a215dd8..c7a01f5583c 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -7,15 +7,21 @@ module Banzai private def media_type - "video" + 'video' end def safe_media_ext Gitlab::FileTypeDetection::SAFE_VIDEO_EXT end - def extra_element_attrs - { width: "400", preload: "metadata" } + def extra_element_attrs(element) + attrs = { preload: 'metadata' } + + attrs[:height] = element[:height] if element[:height] + attrs[:width] = element[:width] if element[:width] + attrs[:width] = '400' unless attrs[:width] || attrs[:height] + + attrs end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 73065571849..6bd9e65f431 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -20,9 +20,9 @@ module Banzai Filter::MathFilter, Filter::ColorFilter, Filter::MermaidFilter, + Filter::AttributesFilter, Filter::VideoLinkFilter, Filter::AudioLinkFilter, - Filter::AttributesFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, *metrics_filters, diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb index a9f908a4247..e1e78f52801 100644 --- a/lib/bulk_imports/clients/graphql.rb +++ b/lib/bulk_imports/clients/graphql.rb @@ -16,8 +16,15 @@ module BulkImports }.to_json ) + unless response.success? + raise ::BulkImports::NetworkError.new( + "Unsuccessful response #{response.code} from #{response.request.path.path}", + response: response + ) + end + ::Gitlab::Json.parse(response.body) - rescue *Gitlab::HTTP::HTTP_ERRORS => e + rescue *Gitlab::HTTP::HTTP_ERRORS, JSON::ParserError => e raise ::BulkImports::NetworkError, e end end @@ -34,10 +41,10 @@ module BulkImports @compatible_instance_version = false end - def execute(*args) + def execute(...) validate_instance_version! - client.execute(*args) + client.execute(...) end def options(extra = {}) diff --git a/lib/bulk_imports/error.rb b/lib/bulk_imports/error.rb index 38f26028276..4699d5eab5f 100644 --- a/lib/bulk_imports/error.rb +++ b/lib/bulk_imports/error.rb @@ -3,7 +3,8 @@ module BulkImports class Error < StandardError def self.unsupported_gitlab_version - self.new("Unsupported GitLab Version. Minimum Supported Gitlab Version #{BulkImport::MIN_MAJOR_VERSION}.") + self.new("Unsupported GitLab version. Source instance must run GitLab version #{BulkImport::MIN_MAJOR_VERSION} " \ + "or later.") end def self.scope_validation_failure @@ -14,5 +15,9 @@ module BulkImports def self.invalid_url self.new("Import aborted as it was not possible to connect to the provided GitLab instance URL.") end + + def self.destination_full_path_validation_failure(full_path) + self.new("Import aborted as '#{full_path}' already exists. Change the destination and try again.") + end end end diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb index 4784baf225c..01d2d9776ec 100644 --- a/lib/bulk_imports/groups/graphql/get_projects_query.rb +++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb @@ -22,6 +22,7 @@ module BulkImports nodes { id name + path full_path: fullPath } } diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb index 026b2e55713..7d9d8120e6c 100644 --- a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb @@ -13,7 +13,7 @@ module BulkImports { source_type: :project_entity, source_full_path: data['full_path'], - destination_name: data['name'], + destination_name: data['path'], destination_namespace: context.entity.group.full_path, parent_id: context.entity.id, source_xid: GlobalID.parse(data['id']).model_id diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb index 83b442458dc..19993629ff5 100644 --- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -4,6 +4,8 @@ module BulkImports module Groups module Transformers class GroupAttributesTransformer + include BulkImports::VisibilityLevel + # rubocop: disable Style/IfUnlessModifier def transform(context, data) import_entity = context.entity @@ -49,10 +51,10 @@ module BulkImports end if data.has_key?('visibility') - params['visibility_level'] = Gitlab::VisibilityLevel.string_options[data['visibility']] + params['visibility_level'] = visibility_level(import_entity, namespace, data['visibility']) end - params + params.with_indifferent_access end # rubocop: enable Style/IfUnlessModifier diff --git a/lib/bulk_imports/projects/pipelines/references_pipeline.rb b/lib/bulk_imports/projects/pipelines/references_pipeline.rb index 9c76f96c7be..8f44f3ffe6a 100644 --- a/lib/bulk_imports/projects/pipelines/references_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/references_pipeline.rb @@ -12,6 +12,8 @@ module BulkImports data = Enumerator.new do |enum| add_matching_objects(portable.issues, enum) add_matching_objects(portable.merge_requests, enum) + add_notes(portable.issues, enum) + add_notes(portable.merge_requests, enum) end BulkImports::Pipeline::ExtractedData.new(data: data) @@ -39,9 +41,16 @@ module BulkImports collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| batch.each do |object| enum << object if object_has_reference?(object) + end + end + end + def add_notes(collection, enum) + collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| + batch.each do |object| object.notes.each_batch(of: BATCH_SIZE) do |notes_batch| notes_batch.each do |note| + note.refresh_markdown_cache! enum << note if object_has_reference?(note) end end diff --git a/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb index 205c3185f72..c5ed9d42e44 100644 --- a/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb +++ b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb @@ -4,21 +4,23 @@ module BulkImports module Projects module Transformers class ProjectAttributesTransformer + include BulkImports::VisibilityLevel + PROJECT_IMPORT_TYPE = 'gitlab_project_migration' def transform(context, data) project = {} entity = context.entity - visibility = data.delete('visibility') + namespace = Namespace.find_by_full_path(entity.destination_namespace) project[:name] = entity.destination_slug project[:path] = entity.destination_slug.parameterize project[:created_at] = data['created_at'] project[:import_type] = PROJECT_IMPORT_TYPE - project[:visibility_level] = Gitlab::VisibilityLevel.string_options[visibility] if visibility.present? - project[:namespace_id] = Namespace.find_by_full_path(entity.destination_namespace)&.id if entity.destination_namespace.present? + project[:visibility_level] = visibility_level(entity, namespace, data['visibility']) + project[:namespace_id] = namespace.id if namespace - project + project.with_indifferent_access end end end diff --git a/lib/bulk_imports/visibility_level.rb b/lib/bulk_imports/visibility_level.rb new file mode 100644 index 00000000000..6b0af15dd7b --- /dev/null +++ b/lib/bulk_imports/visibility_level.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module BulkImports + module VisibilityLevel + private + + def visibility_level(entity, namespace, visibility_string) + requested = requested_visibility_level(entity, visibility_string) + max_allowed = max_allowed_visibility_level(namespace) + + return requested if max_allowed >= requested + + max_allowed + end + + def requested_visibility_level(entity, visibility_string) + Gitlab::VisibilityLevel.string_options[visibility_string] || entity.default_visibility_level + end + + def max_allowed_visibility_level(namespace) + return Gitlab::VisibilityLevel.allowed_levels.max if namespace.blank? + + Gitlab::VisibilityLevel.closest_allowed_level(namespace.visibility_level) + end + end +end diff --git a/lib/csv_builder.rb b/lib/csv_builder.rb index f270f7984da..a54c355396d 100644 --- a/lib/csv_builder.rb +++ b/lib/csv_builder.rb @@ -23,15 +23,17 @@ class CsvBuilder # # * +collection+ - The data collection to be used # * +header_to_hash_value+ - A hash of 'Column Heading' => 'value_method'. + # * +associations_to_preload+ - An array of records to preload with a batch of records. # # The value method will be called once for each object in the collection, to # determine the value for that row. It can either be the name of a method on # the object, or a lamda to call passing in the object. - def initialize(collection, header_to_value_hash) + def initialize(collection, header_to_value_hash, associations_to_preload = []) @header_to_value_hash = header_to_value_hash @collection = collection @truncated = false @rows_written = 0 + @associations_to_preload = associations_to_preload end # Renders the csv to a string @@ -75,7 +77,13 @@ class CsvBuilder protected def each(&block) - @collection.find_each(&block) # rubocop: disable CodeReuse/ActiveRecord + if @associations_to_preload.present? && @collection.respond_to?(:each_batch) + @collection.each_batch(order_hint: :created_at) do |relation| + relation.preload(@associations_to_preload).order(:id).each(&block) # rubocop:disable CodeReuse/ActiveRecord + end + else + @collection.find_each(&block) # rubocop: disable CodeReuse/ActiveRecord + end end private diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb index f22996df0a5..dba1aad639c 100644 --- a/lib/extracts_ref.rb +++ b/lib/extracts_ref.rb @@ -68,7 +68,7 @@ module ExtractsRef return unless @ref.present? - @commit = if ref_type && Feature.enabled?(:use_ref_type_parameter, @repo.project) + @commit = if ref_type @fully_qualified_ref = %(refs/#{ref_type}/#{@ref}) @repo.commit(@fully_qualified_ref) else diff --git a/lib/feature.rb b/lib/feature.rb index d012639d489..17c26796ea1 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -206,6 +206,7 @@ module Feature # to register Flipper groups. # See https://docs.gitlab.com/ee/development/feature_flags/index.html def register_feature_groups + Flipper.register(:gitlab_team_members) { |actor| FeatureGroups::GitlabTeamMembers.enabled?(actor.thing) } end def register_definitions @@ -423,4 +424,5 @@ module Feature end end +Feature.prepend_mod Feature::ActiveSupportCacheStoreAdapter.prepend_mod_with('Feature::ActiveSupportCacheStoreAdapter') diff --git a/lib/feature_groups/gitlab_team_members.rb b/lib/feature_groups/gitlab_team_members.rb new file mode 100644 index 00000000000..7f4c597fddd --- /dev/null +++ b/lib/feature_groups/gitlab_team_members.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module FeatureGroups + class GitlabTeamMembers + GITLAB_COM_GROUP_ID = 6543 + + class << self + def enabled?(thing) + return false unless Gitlab.com? + + team_member?(thing) + end + + private + + def team_member?(thing) + thing.is_a?(::User) && gitlab_com_member_ids.include?(thing.id) + end + + def gitlab_com + @gitlab_com ||= ::Group.find(GITLAB_COM_GROUP_ID) + end + + def gitlab_com_member_ids + Rails.cache.fetch("gitlab_team_members", expires_in: 1.hour) do + gitlab_com.members.pluck_user_ids.to_set + end + end + end + end +end diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index ac3210b4e98..8ac5f2f6b54 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -4,7 +4,7 @@ class FileSizeValidator < ActiveModel::EachValidator MESSAGES = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze CHECKS = { is: :==, minimum: :>=, maximum: :<= }.freeze - DEFAULT_TOKENIZER = -> (value) { value.split(//) }.freeze + DEFAULT_TOKENIZER = -> (value) { value.split("") }.freeze RESERVED_OPTIONS = [:minimum, :maximum, :within, :is, :tokenizer, :too_short, :too_long].freeze def initialize(options) diff --git a/lib/generators/gitlab/partitioning/foreign_keys_generator.rb b/lib/generators/gitlab/partitioning/foreign_keys_generator.rb new file mode 100644 index 00000000000..e013188c753 --- /dev/null +++ b/lib/generators/gitlab/partitioning/foreign_keys_generator.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +require 'rails/generators' +require 'rails/generators/active_record/migration' + +module Gitlab + module Partitioning + class ForeignKeysGenerator < Rails::Generators::Base + include ActiveRecord::Generators::Migration + + desc 'This generator creates the migrations needed for updating the foreign keys when partitioning the tables' + + source_root File.expand_path('templates', __dir__) + + class_option :target, type: :string, required: true, desc: 'Target table name' + class_option :source, type: :string, required: true, desc: 'Source table name' + class_option :partitioning_column, type: :string, default: :partition_id, + desc: 'The column that is used for partitioning' + class_option :database, type: :string, default: :ci, + desc: 'Database name connection' + + def create_fk_index_migration + migration_template( + '../templates/foreign_key_index.rb.template', + fk_index_file_name) + end + + def create_fk_definition_migration + migration_template( + '../templates/foreign_key_definition.rb.template', + fk_definition_file_name) + end + + def create_fk_validation_migration + migration_template( + '../templates/foreign_key_validation.rb.template', + fk_validation_file_name) + end + + def remove_old_fk_migration + migration_template( + '../templates/foreign_key_removal.rb.template', + fk_removal_file_name) + end + + private + + def fk_index_file_name + post_migration_file_path( + "add_fk_index_to_#{source_table_name}_on_#{partitioning_column}_and_#{foreign_key_column}.rb") + end + + def fk_definition_file_name + post_migration_file_path( + "add_fk_to_#{source_table_name}_on_#{partitioning_column}_and_#{foreign_key_column}.rb") + end + + def fk_validation_file_name + post_migration_file_path( + "validate_fk_on_#{source_table_name}_#{partitioning_column}_and_#{foreign_key_column}.rb") + end + + def fk_removal_file_name + post_migration_file_path( + "remove_fk_to_#{target_table_name}_#{source_table_name}_on_#{foreign_key_column}.rb") + end + + def post_migration_file_path(name) + File.join(db_migrate_path, name) + end + + def db_migrate_path + super.sub('migrate', 'post_migrate') + end + + def source_table_name + options[:source] + end + + def target_table_name + options[:target] + end + + def partitioning_column + options[:partitioning_column] + end + + def foreign_keys_candidates + connection + .foreign_keys(source_table_name) + .select { |fk| fk.to_table == target_table_name } + .reject { |fk| fk.name.end_with?('_p') } + end + + def fk_candidate + @fk_candidate ||= select_foreign_key + end + + def foreign_key_name + fk_candidate.name + end + + def partitioned_foreign_key_name + "#{foreign_key_name}_p" + end + + def foreign_key_column + fk_candidate.column + end + + def fk_on_delete_option + fk_candidate.on_delete + end + + def fk_target_column + fk_candidate.primary_key + end + + def connection + Gitlab::Database + .database_base_models + .fetch(options[:database]) + .connection + end + + def select_foreign_key + case foreign_keys_candidates.size + when 0 + raise Thor::InvocationError, "No FK found between #{source_table_name} and #{target_table_name}" + when 1 + foreign_keys_candidates.first + else + select_fk_from_user_input + end + end + + def select_fk_from_user_input + options = (0...foreign_keys_candidates.size).to_a.map(&:to_s) + + say "There are multiple FKs between #{source_table_name} and #{target_table_name}:" + foreign_keys_candidates.each.with_index do |fk, index| + say "\t#{index} : #{fk}" + end + + input = ask("Please select one:", limited_to: options, default: '0') + + foreign_keys_candidates.fetch(input.to_i) + end + end + end +end diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template new file mode 100644 index 00000000000..f2f9acea923 --- /dev/null +++ b/lib/generators/gitlab/partitioning/templates/foreign_key_definition.rb.template @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + disable_ddl_transaction! + + SOURCE_TABLE_NAME = :<%= source_table_name %> + TARGET_TABLE_NAME = :<%= target_table_name %> + COLUMN = :<%= foreign_key_column %> + TARGET_COLUMN = :<%= fk_target_column %> + FK_NAME = :<%= partitioned_foreign_key_name %> + PARTITION_COLUMN = :<%= partitioning_column %> + + def up + add_concurrent_foreign_key( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + column: [PARTITION_COLUMN, COLUMN], + target_column: [PARTITION_COLUMN, TARGET_COLUMN], + validate: false, + reverse_lock_order: true, + on_update: :cascade, + on_delete: :<%= fk_on_delete_option %>, + name: FK_NAME + ) + end + + def down + with_lock_retries do + remove_foreign_key_if_exists( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + name: FK_NAME, + reverse_lock_order: true + ) + end + end +end diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template new file mode 100644 index 00000000000..4896d931333 --- /dev/null +++ b/lib/generators/gitlab/partitioning/templates/foreign_key_index.rb.template @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + disable_ddl_transaction! + + INDEX_NAME = :index_<%= source_table_name -%>_on_<%= partitioning_column -%>_<%= foreign_key_column %> + TABLE_NAME = :<%= source_table_name %> + COLUMNS = [:<%= partitioning_column -%>, :<%= foreign_key_column -%>] + + def up + add_concurrent_index(TABLE_NAME, COLUMNS, name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME) + end +end diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template new file mode 100644 index 00000000000..b4e881074ad --- /dev/null +++ b/lib/generators/gitlab/partitioning/templates/foreign_key_removal.rb.template @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + disable_ddl_transaction! + + SOURCE_TABLE_NAME = :<%= source_table_name %> + TARGET_TABLE_NAME = :<%= target_table_name %> + COLUMN = :<%= foreign_key_column %> + TARGET_COLUMN = :<%= fk_target_column %> + FK_NAME = :<%= foreign_key_name %> + + def up + with_lock_retries do + remove_foreign_key_if_exists( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + name: FK_NAME, + reverse_lock_order: true + ) + end + end + + def down + add_concurrent_foreign_key( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + column: COLUMN, + target_column: TARGET_COLUMN, + validate: true, + reverse_lock_order: true, + on_delete: :<%= fk_on_delete_option %>, + name: FK_NAME + ) + end +end diff --git a/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template b/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template new file mode 100644 index 00000000000..bad7d17a51b --- /dev/null +++ b/lib/generators/gitlab/partitioning/templates/foreign_key_validation.rb.template @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Database::Migration.current_version %>] + disable_ddl_transaction! + + TABLE_NAME = :<%= source_table_name %> + FK_NAME = :<%= partitioned_foreign_key_name %> + COLUMNS = [:<%= partitioning_column -%>, :<%= foreign_key_column -%>] + + def up + validate_foreign_key(TABLE_NAME, COLUMNS, name: FK_NAME) + end + + def down + # no-op + end +end diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb index 4518c8a862c..879247d0174 100644 --- a/lib/gitlab/allowable.rb +++ b/lib/gitlab/allowable.rb @@ -2,8 +2,8 @@ module Gitlab module Allowable - def can?(*args) - Ability.allowed?(*args) + def can?(...) + Ability.allowed?(...) end end end diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb index dd8149aba94..41f94e79f91 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb @@ -24,7 +24,7 @@ module Gitlab def initialize(stage:, params: {}) @stage = stage @params = params - @root_ancestor = stage.parent.root_ancestor + @root_ancestor = stage.namespace.root_ancestor @stage_event_model = MODEL_CLASSES.fetch(stage.subject_class.to_s) end @@ -90,7 +90,7 @@ module Gitlab end def filter_by_stage_parent(query) - query.by_project_id(stage.parent_id) + query.by_project_id(stage.namespace.project.id) end def base_query diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb index 3abf380d461..8b40a8c2b26 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb @@ -5,7 +5,7 @@ module Gitlab module CycleAnalytics module Aggregated # Arguments: - # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::Stage + # stage - an instance of CycleAnalytics::Stage # params: # current_user: an instance of User # from: DateTime diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb index c7987d63153..ca8b4a3a890 100644 --- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -57,9 +57,14 @@ module Gitlab end def add_parent_model_params!(finder_params) - raise(ArgumentError, "unknown parent_class: #{parent_class}") unless parent_class.eql?(Project) - - finder_params[:project_id] = stage.parent_id + case stage.parent + when Namespaces::ProjectNamespace + finder_params[:project_id] = stage.parent.project.id + when Project + finder_params[:project_id] = stage.parent_id + else + raise(ArgumentError, "unknown parent_class: #{parent_class}") + end end def add_time_range_params!(finder_params, from, to) diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index 0db027b9861..ffafafda4ac 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics # Arguments: - # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::Stage + # stage - an instance of CycleAnalytics::Stage # params: # current_user: an instance of User # from: DateTime diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 43683ae174e..0f1380125fa 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -6,7 +6,7 @@ # Example: # # params = Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_issue_stage -# Analytics::CycleAnalytics::ProjectStage.new(params) +# Analytics::CycleAnalytics::Stage.new(params) module Gitlab module Analytics module CycleAnalytics diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index d058782ae87..2df3680db5f 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -196,7 +196,7 @@ module Gitlab return unless value_stream strong_memoize(:stage) do - ::Analytics::CycleAnalytics::StageFinder.new(parent: project || group, stage_id: stage_id).execute if stage_id + ::Analytics::CycleAnalytics::StageFinder.new(parent: project&.project_namespace || group, stage_id: stage_id).execute if stage_id end end end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb index afada055928..3f8025c408a 100644 --- a/lib/gitlab/api_authentication/token_resolver.rb +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -165,7 +165,9 @@ module Gitlab end def with_deploy_token(raw, &block) - raise ::Gitlab::Auth::UnauthorizedError if Gitlab::ExternalAuthorization.enabled? + unless Gitlab::ExternalAuthorization.allow_deploy_tokens_and_deploy_keys? + raise ::Gitlab::Auth::UnauthorizedError + end token = ::DeployToken.active.find_by_token(raw.password) return unless token diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index a788586ebec..466538df56e 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -55,7 +55,8 @@ module Gitlab phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, namespace_exists: { threshold: 20, interval: 1.minute }, fetch_google_ip_list: { threshold: 10, interval: 1.minute }, - jobs_index: { threshold: 600, interval: 1.minute } + jobs_index: { threshold: 600, interval: 1.minute }, + bulk_import: { threshold: 6, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c97ef5a10ef..06bdb2c1ddc 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -32,7 +32,7 @@ module Gitlab # Scopes used for GitLab as admin SUDO_SCOPE = :sudo ADMIN_MODE_SCOPE = :admin_mode - ADMIN_SCOPES = [SUDO_SCOPE].freeze + ADMIN_SCOPES = [SUDO_SCOPE, ADMIN_MODE_SCOPE].freeze # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [API_SCOPE].freeze @@ -366,7 +366,10 @@ module Gitlab def available_scopes_for(current_user) scopes = non_admin_available_scopes - scopes += ADMIN_SCOPES if current_user.admin? + + if current_user.admin? # rubocop: disable Cop/UserAdmin + scopes += Feature.enabled?(:admin_mode_for_api) ? ADMIN_SCOPES : [SUDO_SCOPE] + end scopes end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 16bee187c87..c69462b12de 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -121,6 +121,7 @@ module Gitlab # It is also used by GraphQL/API requests. # And to allow accessing /archive programatically as it was a big pain point # for users https://gitlab.com/gitlab-org/gitlab/-/issues/28978. + # Used for release downloading as well def find_user_from_web_access_token(request_format, scopes: [:api]) return unless access_token && valid_web_access_format?(request_format) @@ -147,7 +148,7 @@ module Gitlab # deploy tokens are accepted with deploy token headers and basic auth headers def deploy_token_from_request return unless route_authentication_setting[:deploy_token_allowed] - return if Gitlab::ExternalAuthorization.enabled? + return unless Gitlab::ExternalAuthorization.allow_deploy_tokens_and_deploy_keys? token = current_request.env[DEPLOY_TOKEN_HEADER].presence || parsed_oauth_token @@ -301,6 +302,8 @@ module Gitlab api_request? when :archive archive_request? + when :download + download_request? end end @@ -352,6 +355,10 @@ module Gitlab current_request.path.include?('/-/archive/') end + def download_request? + current_request.path.include?('/downloads/') + end + def blob_request? current_request.path.include?('/raw/') end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 9aedc131e92..0201f1f8725 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -33,8 +33,8 @@ module Gitlab users_search(options) end - def user(*args) - users(*args).first + def user(...) + users(...).first end def dn_matches_filter?(dn, filter) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 242390c3e89..01e126ec2f5 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -258,7 +258,7 @@ module Gitlab metadata = gl_user.build_user_synced_attributes_metadata if sync_profile_from_provider? - UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| + UserSyncedAttributesMetadata.syncable_attributes.each do |key| if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) gl_user.public_send("#{key}=".to_sym, auth_hash.public_send(key)) # rubocop:disable GitlabSecurity/PublicSend metadata.set_attribute_synced(key, true) diff --git a/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb b/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb new file mode 100644 index 00000000000..46e2d5cb930 --- /dev/null +++ b/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Iterates through the namespaces table and attempts to set the + # opposite of the value of the column "emails_disabled" to a new + # column in namespace_settings called emails_enabled + class AddNamespacesEmailsEnabledColumnData < BatchedMigrationJob + feature_category :database + operation_name :add_namespaces_emails_enabled_column_data + + # Targeted table + class NamespaceSetting < ApplicationRecord + self.table_name = 'namespace_settings' + end + + def perform + each_sub_batch do |sub_batch| + plucked_list = sub_batch.where('NOT emails_disabled IS NULL').pluck(:id, :emails_disabled) + next if plucked_list.empty? + + ApplicationRecord.connection.execute <<~SQL + UPDATE namespace_settings + SET emails_enabled= NOT subquery.emails_enabled + FROM (SELECT * FROM (#{Arel::Nodes::ValuesList.new(plucked_list).to_sql}) AS updates(namespace_id, emails_enabled)) AS subquery + WHERE namespace_settings.namespace_id=subquery.namespace_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb b/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb new file mode 100644 index 00000000000..a0ce5d22597 --- /dev/null +++ b/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Iterates through the Projects table and attempts to set the + # opposite of the value of the column "emails_disabled" to a new + # column in project_settings called emails_enabled + class AddProjectsEmailsEnabledColumnData < BatchedMigrationJob + feature_category :database + operation_name :add_projects_emails_enabled_column_data + + # Targeted table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch do |sub_batch| + plucked_list = sub_batch.where('NOT emails_disabled IS NULL').pluck(:id, :emails_disabled) + next if plucked_list.empty? + + ApplicationRecord.connection.execute <<~SQL + UPDATE project_settings + SET emails_enabled=NOT subquery.emails_enabled + FROM (SELECT * FROM (#{Arel::Nodes::ValuesList.new(plucked_list).to_sql}) AS updates(project_id, emails_enabled)) AS subquery + WHERE project_settings.project_id=subquery.project_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb deleted file mode 100644 index 669e5338dd1..00000000000 --- a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2, -# it's enough at the moment to simply notice if the url is from `atlassian.net` -module Gitlab - module BackgroundMigration - # Backfill the deployment_type in jira_tracker_data table - class BackfillJiraTrackerDeploymentType2 - # Migration only version of jira_tracker_data table - class JiraTrackerDataTemp < ApplicationRecord - self.table_name = 'jira_tracker_data' - - def self.encryption_options - { - key: Settings.attr_encrypted_db_key_base_32, - encode: true, - mode: :per_attribute_iv, - algorithm: 'aes-256-gcm' - } - end - - attr_encrypted :url, encryption_options - attr_encrypted :api_url, encryption_options - - enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment - end - - # Migration only version of services table - class JiraServiceTemp < ApplicationRecord - self.table_name = 'services' - self.inheritance_column = :_type_disabled - end - - def perform(start_id, stop_id) - @server_ids = [] - @cloud_ids = [] - - JiraTrackerDataTemp - .where(id: start_id..stop_id, deployment_type: 0) - .each do |jira_tracker_data| - collect_deployment_type(jira_tracker_data) - end - - unless cloud_ids.empty? - JiraTrackerDataTemp.where(id: cloud_ids) - .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud]) - end - - unless server_ids.empty? - JiraTrackerDataTemp.where(id: server_ids) - .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server]) - end - - mark_jobs_as_succeeded(start_id, stop_id) - end - - private - - attr_reader :server_ids, :cloud_ids - - def client_url(jira_tracker_data) - jira_tracker_data.api_url.presence || jira_tracker_data.url.presence - end - - def server_type(url) - url.downcase.include?('.atlassian.net') ? :cloud : :server - end - - def collect_deployment_type(jira_tracker_data) - url = client_url(jira_tracker_data) - return unless url - - case server_type(url) - when :cloud - cloud_ids << jira_tracker_data.id - else - server_ids << jira_tracker_data.id - end - end - - def mark_jobs_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name.demodulize, arguments) - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb new file mode 100644 index 00000000000..b6e22e481fa --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migration to make sure that all the prevously saved tokens have their encrypted values in the db. + class EncryptCiTriggerToken < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :continuous_integration + scope_to ->(relation) { relation.where(encrypted_token: nil) } + operation_name :update + # Class that is imitating Ci::Trigger + class CiTrigger < ::Ci::ApplicationRecord + ALGORITHM = 'aes-256-gcm' + + self.table_name = 'ci_triggers' + + attr_encrypted :encrypted_token_tmp, + attribute: :encrypted_token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_vi: false + + before_save :copy_token_to_encrypted_token + + def copy_token_to_encrypted_token + self.encrypted_token_tmp = token + end + end + + def perform + each_sub_batch do |sub_batch| + sub_batch.each do |trigger| + Gitlab::BackgroundMigration::EncryptCiTriggerToken::CiTrigger.find(trigger.id).save! + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb index 3843356af69..c9582da2a51 100644 --- a/lib/gitlab/background_migration/encrypt_integration_properties.rb +++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb @@ -31,7 +31,7 @@ module Gitlab def encrypt_properties data = ::Gitlab::Json.parse(properties) iv = generate_iv(ALGORITHM) - ep = self.class.encrypt(:encrypted_properties_tmp, data, { iv: iv }) + ep = self.class.attr_encrypt(:encrypted_properties_tmp, data, { iv: iv }) [ep, iv] end diff --git a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb new file mode 100644 index 00000000000..4b6bb12c91b --- /dev/null +++ b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A migration that will: + # * get all project statistics where packages_size is non zero + # * check the coherence with the related package files + # * fix non coherent packages_size values + class FixIncoherentPackagesSizeOnProjectStatistics < BatchedMigrationJob + MIGRATOR = 'FixIncoherentPackagesSizeOnProjectStatistics' + + feature_category :package_registry + + operation_name :fix_incorrect_packages_size + + def perform + each_sub_batch do |sub_batch| + fix_packages_size(sub_batch) + end + end + + private + + def fix_packages_size(project_statistics) + statistics_table = FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics.arel_table + from = [ + statistics_table, + FixIncoherentPackagesSizeOnProjectStatistics::PackageFile.sum_query.arel.lateral.as('size_sum') + ] + size_sum_table = ::Arel::Table.new(:size_sum) + + project_statistics.select(:id, :project_id, :packages_size, size_sum_table[:total]) + .from(from) + .where.not(statistics_table[:packages_size].eq(size_sum_table[:total])) + .each do |stat| + increment = stat[:total].to_i - stat[:packages_size] - buffered_update(stat) + next if increment == 0 + + ::Gitlab::BackgroundMigration::Logger.info( + migrator: MIGRATOR, + project_id: stat[:project_id], + old_size: stat[:packages_size], + new_size: stat[:total].to_i + ) + + stat.becomes(FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics) # rubocop:disable Cop/AvoidBecomes + .increment(increment) + end + end + + def buffered_update(stat) + key = "project:{#{stat[:project_id]}}:counters:ProjectStatistics:#{stat[:id]}:packages_size" + + Gitlab::Redis::SharedState.with do |redis| + redis.get(key).to_i + end + end + + # rubocop:disable Style/Documentation + class ProjectStatistics < ::ApplicationRecord + self.table_name = 'project_statistics' + + def increment(amount) + FixIncoherentPackagesSizeOnProjectStatistics::BufferedCounter.new(self).increment(amount) + end + end + + class Package < ::ApplicationRecord + self.table_name = 'packages_packages' + + has_many :package_files, + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' # rubocop:disable Layout/LineLength + end + + class PackageFile < ::ApplicationRecord + self.table_name = 'packages_package_files' + + belongs_to :package, + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' # rubocop:disable Layout/LineLength + + def self.sum_query + packages = FixIncoherentPackagesSizeOnProjectStatistics::Package.arel_table + stats = FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics.arel_table + + joins(:package) + .where(packages[:project_id].eq(stats[:project_id])) + .where.not(size: nil) + .select('SUM(packages_package_files.size) as total') + end + end + + class BufferedCounter + WORKER_DELAY = 10.minutes + + def initialize(stat) + @stat = stat + end + + def key + "project:{#{@stat.project_id}}:counters:ProjectStatistics:#{@stat.id}:packages_size" + end + + def increment(amount) + Gitlab::Redis::SharedState.with do |redis| + redis.incrby(key, amount) + end + + FlushCounterIncrementsWorker.perform_in( + WORKER_DELAY, + 'ProjectStatistics', + @stat.id, + :packages_size + ) + end + end + # rubocop:enable Style/Documentation + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb index 914ababa5c2..db3f98bc2ba 100644 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -68,7 +68,7 @@ module Gitlab def valid_json?(metadata) Oj.load(metadata) true - rescue Oj::ParseError, EncodingError, JSON::ParserError, Encoding::UndefinedConversionError + rescue Oj::ParseError, EncodingError, JSON::ParserError, JSON::GeneratorError, Encoding::UndefinedConversionError false end diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb new file mode 100644 index 00000000000..592ef3220ff --- /dev/null +++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify `projects.creator_id` column of projects who creator + # does not exist in `users` table anymore. + class NullifyCreatorIdColumnOfOrphanedProjects < BatchedMigrationJob + scope_to ->(relation) do + relation.where.not(creator_id: nil) + .joins('LEFT OUTER JOIN users ON users.id = projects.creator_id') + .where(users: { id: nil }) + end + + operation_name :update_all + feature_category :projects + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(creator_id: nil) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/rebalance_partition_id.rb b/lib/gitlab/background_migration/rebalance_partition_id.rb new file mode 100644 index 00000000000..7000ae5a856 --- /dev/null +++ b/lib/gitlab/background_migration/rebalance_partition_id.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This rebalances partition_id to fix invalid records in production + class RebalancePartitionId < BatchedMigrationJob + INVALID_PARTITION_ID = 101 + VALID_PARTITION_ID = 100 + + scope_to ->(relation) { relation.where(partition_id: INVALID_PARTITION_ID) } + operation_name :update_all + feature_category :continuous_integration + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(partition_id: VALID_PARTITION_ID) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/sanitize_confidential_todos.rb b/lib/gitlab/background_migration/sanitize_confidential_todos.rb index 2df0b8a4d93..8215e92cbeb 100644 --- a/lib/gitlab/background_migration/sanitize_confidential_todos.rb +++ b/lib/gitlab/background_migration/sanitize_confidential_todos.rb @@ -10,43 +10,14 @@ module Gitlab # to extract all related logic. # Details in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908#note_952459215 class SanitizeConfidentialTodos < BatchedMigrationJob - scope_to ->(relation) { relation.where(confidential: true) } - operation_name :delete_invalid_todos feature_category :database def perform - each_sub_batch do |sub_batch| - delete_ids = invalid_todo_ids(sub_batch) - - Todo.where(id: delete_ids).delete_all if delete_ids.present? - end - end - - private - - def invalid_todo_ids(notes_batch) - todos = Todo.where(note_id: notes_batch.select(:id)).includes(:note, :user) - - todos.each_with_object([]) do |todo, ids| - ids << todo.id if invalid_todo?(todo) - end - end - - def invalid_todo?(todo) - return false unless todo.note - return false if Ability.allowed?(todo.user, :read_todo, todo) - - logger.info( - message: "#{self.class.name} deleting invalid todo", - attributes: todo.attributes - ) - - true - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build + # no-op: this BG migration is left here only for compatibility reasons, + # but it's not scheduled from any migration anymore. + # It was a temporary migration which used not-isolated code. + # https://gitlab.com/gitlab-org/gitlab/-/issues/382557 end end end diff --git a/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb b/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb new file mode 100644 index 00000000000..24080a06c46 --- /dev/null +++ b/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class ThirdRecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +# we just want to re-enqueue the previous BackfillEpicCacheCounts migration, +# because it's a EE-only migation and it's a module, we just prepend new +# RecountEpicCacheCounts with existing batched migration module (which is same in both cases) +Gitlab::BackgroundMigration::ThirdRecountEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 49b8ab760f3..3dafe7c8962 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -92,15 +92,15 @@ module Gitlab def import_issues return unless repo.issues_enabled? - # If a user creates an issue while the import is in progress, this can lead to an import failure. - # The workaround is to allocate IIDs before starting the importer. - allocate_issues_internal_id!(project, client) - create_labels issue_type_id = ::WorkItems::Type.default_issue_type.id - client.issues(repo).each do |issue| + client.issues(repo).each_with_index do |issue, index| + # If a user creates an issue while the import is in progress, this can lead to an import failure. + # The workaround is to allocate IIDs before starting the importer. + allocate_issues_internal_id!(project, client) if index == 0 + import_issue(issue, issue_type_id) end end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 242979da367..ea9b79c12fd 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -55,7 +55,7 @@ module Gitlab handle_errors metrics.track_finished_import - log_info(stage: "complete") + log_info(import_stage: "complete") Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) true @@ -139,16 +139,16 @@ module Gitlab end def import_repository - log_info(stage: 'import_repository', message: 'starting import') + log_info(import_stage: 'import_repository', message: 'starting import') project.repository.import_repository(project.import_url) project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap) - log_info(stage: 'import_repository', message: 'finished import') + log_info(import_stage: 'import_repository', message: 'finished import') rescue ::Gitlab::Git::CommandError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'import_repository', message: 'failed import', error: e.message + import_stage: 'import_repository', message: 'failed import', error: e.message ) # Expire cache to prevent scenarios such as: @@ -179,10 +179,10 @@ module Gitlab def import_pull_requests page = 0 - log_info(stage: 'import_pull_requests', message: "starting") + log_info(import_stage: 'import_pull_requests', message: "starting") loop do - log_debug(stage: 'import_pull_requests', message: "importing page #{page} and batch-size #{BATCH_SIZE} from #{page * BATCH_SIZE} to #{(page + 1) * BATCH_SIZE}") + log_debug(import_stage: 'import_pull_requests', message: "importing page #{page} and batch-size #{BATCH_SIZE} from #{page * BATCH_SIZE} to #{(page + 1) * BATCH_SIZE}") pull_requests = client.pull_requests(project_key, repository_slug, page_offset: page, limit: BATCH_SIZE).to_a @@ -196,21 +196,21 @@ module Gitlab pull_requests.each do |pull_request| if already_imported?(pull_request) - log_info(stage: 'import_pull_requests', message: 'already imported', iid: pull_request.iid) + log_info(import_stage: 'import_pull_requests', message: 'already imported', iid: pull_request.iid) else import_bitbucket_pull_request(pull_request) end rescue StandardError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'import_pull_requests', iid: pull_request.iid, error: e.message + import_stage: 'import_pull_requests', iid: pull_request.iid, error: e.message ) backtrace = Gitlab::BacktraceCleaner.clean_backtrace(e.backtrace) errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } end - log_debug(stage: 'import_pull_requests', message: "finished page #{page} and batch-size #{BATCH_SIZE}") + log_debug(import_stage: 'import_pull_requests', message: "finished page #{page} and batch-size #{BATCH_SIZE}") page += 1 end end @@ -235,7 +235,7 @@ module Gitlab rescue BitbucketServer::Connection::ConnectionError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'delete_temp_branches', branch: branch.name, error: e.message + import_stage: 'delete_temp_branches', branch: branch.name, error: e.message ) @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message } @@ -243,7 +243,7 @@ module Gitlab end def import_bitbucket_pull_request(pull_request) - log_info(stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) + log_info(import_stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) description = '' description += author_line(pull_request) @@ -274,12 +274,12 @@ module Gitlab metrics.merge_requests_counter.increment end - log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) + log_info(import_stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) mark_as_imported(pull_request) end def import_pull_request_comments(pull_request, merge_request) - log_info(stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid) + log_info(import_stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid) comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?) @@ -291,7 +291,7 @@ module Gitlab import_inline_comments(inline_comments.map(&:comment), merge_request) import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) - log_info(stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid, + log_info(import_stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid, merge_event_found: merge_event.present?, inline_comments_count: inline_comments.count, standalone_pr_comments: pr_comments.count) @@ -299,7 +299,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def import_merge_event(merge_request, merge_event) - log_info(stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) + log_info(import_stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) committer = merge_event.committer_email @@ -309,12 +309,12 @@ module Gitlab metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) metric.update(merged_by_id: user_id, merged_at: timestamp) - log_info(stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) + log_info(import_stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) end # rubocop: enable CodeReuse/ActiveRecord def import_inline_comments(inline_comments, merge_request) - log_info(stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) + log_info(import_stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) inline_comments.each do |comment| position = build_position(merge_request, comment) @@ -329,7 +329,7 @@ module Gitlab end end - log_info(stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) + log_info(import_stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) end def create_diff_note(merge_request, comment, position, discussion_id = nil) @@ -344,7 +344,7 @@ module Gitlab return note end - log_info(stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) + log_info(import_stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) # Bitbucket Server supports the ability to comment on any line, not just the # line in the diff. If we can't add the note as a DiffNote, fallback to creating @@ -353,7 +353,7 @@ module Gitlab rescue StandardError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'create_diff_note', comment_id: comment.id, error: e.message + import_stage: 'create_diff_note', comment_id: comment.id, error: e.message ) errors << { type: :pull_request, id: comment.id, errors: e.message } @@ -394,7 +394,7 @@ module Gitlab rescue StandardError => e Gitlab::ErrorTracking.log_exception( e, - stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message + import_stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message ) errors << { type: :pull_request, comment_id: comment.id, errors: e.message } diff --git a/lib/gitlab/cache/helpers.rb b/lib/gitlab/cache/helpers.rb index 024fa48c066..0fc0b1504af 100644 --- a/lib/gitlab/cache/helpers.rb +++ b/lib/gitlab/cache/helpers.rb @@ -45,7 +45,14 @@ module Gitlab def contextual_cache_key(presenter, object, context) return object.cache_key if context.nil? - [presenter.class.name, object.cache_key, context.call(object)].flatten.join(":") + [presenter_class_name(presenter), object.cache_key, context.call(object)].flatten.join(":") + end + + def presenter_class_name(presenter) + return presenter.class.name if presenter.is_a?(BaseSerializer) + return presenter.name if presenter.is_a?(Class) && presenter < Grape::Entity + + raise ArgumentError, "presenter #{presenter} is not supported" end # Used for fetching or rendering a single object diff --git a/lib/gitlab/cache/metadata.rb b/lib/gitlab/cache/metadata.rb new file mode 100644 index 00000000000..d6c89b5b2c3 --- /dev/null +++ b/lib/gitlab/cache/metadata.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + # Value object for cache metadata + class Metadata + VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze + DEFAULT_BACKING_RESOURCE = :unknown + + def initialize( + cache_identifier:, + feature_category:, + caller_id: Gitlab::ApplicationContext.current_context_attribute(:caller_id), + backing_resource: DEFAULT_BACKING_RESOURCE + ) + @cache_identifier = cache_identifier + @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) + @caller_id = caller_id + @backing_resource = fetch_backing_resource!(backing_resource) + end + + attr_reader :caller_id, :cache_identifier, :feature_category, :backing_resource + + private + + def fetch_backing_resource!(resource) + return resource if VALID_BACKING_RESOURCES.include?(resource) + + raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env? + + DEFAULT_BACKING_RESOURCE + end + end + end +end diff --git a/lib/gitlab/cache/metrics.rb b/lib/gitlab/cache/metrics.rb index 0143052beb1..00d4e6e4d4e 100644 --- a/lib/gitlab/cache/metrics.rb +++ b/lib/gitlab/cache/metrics.rb @@ -5,19 +5,9 @@ module Gitlab module Cache class Metrics DEFAULT_BUCKETS = [0, 1, 5].freeze - VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze - DEFAULT_BACKING_RESOURCE = :unknown - def initialize( - caller_id:, - cache_identifier:, - feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT, - backing_resource: DEFAULT_BACKING_RESOURCE - ) - @caller_id = caller_id - @cache_identifier = cache_identifier - @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) - @backing_resource = fetch_backing_resource!(backing_resource) + def initialize(cache_metadata) + @cache_metadata = cache_metadata end # Increase cache hit counter @@ -51,7 +41,7 @@ module Gitlab private - attr_reader :caller_id, :cache_identifier, :feature_category, :backing_resource + attr_reader :cache_metadata def counter @counter ||= Gitlab::Metrics.counter(:redis_hit_miss_operations_total, "Hit/miss Redis cache counter") @@ -68,20 +58,12 @@ module Gitlab def labels @labels ||= { - caller_id: caller_id, - cache_identifier: cache_identifier, - feature_category: feature_category, - backing_resource: backing_resource + caller_id: cache_metadata.caller_id, + cache_identifier: cache_metadata.cache_identifier, + feature_category: cache_metadata.feature_category, + backing_resource: cache_metadata.backing_resource } end - - def fetch_backing_resource!(resource) - return resource if VALID_BACKING_RESOURCES.include?(resource) - - raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env? - - DEFAULT_BACKING_RESOURCE - end end end end diff --git a/lib/gitlab/chat/responder.rb b/lib/gitlab/chat/responder.rb index 53a625e9d43..478be5bd350 100644 --- a/lib/gitlab/chat/responder.rb +++ b/lib/gitlab/chat/responder.rb @@ -11,10 +11,21 @@ module Gitlab # # build - A `Ci::Build` that executed a chat command. def self.responder_for(build) - integration = build.pipeline.chat_data&.chat_name&.integration + if Feature.enabled?(:use_response_url_for_chat_responder) + response_url = build.pipeline.chat_data&.response_url + return unless response_url - if (responder = integration.try(:chat_responder)) - responder.new(build) + if response_url.start_with?('https://hooks.slack.com/') + Gitlab::Chat::Responder::Slack.new(build) + else + Gitlab::Chat::Responder::Mattermost.new(build) + end + else + integration = build.pipeline.chat_data&.chat_name&.integration + + if (responder = integration.try(:chat_responder)) + responder.new(build) + end end end end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index abe2f272ca7..21fc2980cdc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -26,7 +26,7 @@ module Gitlab # Without forcing the encoding to UTF-8 and then replacing # invalid UTF-8 sequences we can get an error when serializing # the Hash to JSON. - # Encoding::UndefinedConversionError: + # Encoding::UndefinedConversionError (or possibly JSON::GeneratorError in json 2.6.1+): # "\xE2" from ASCII-8BIT to UTF-8 { text: encode_utf8_no_detect(text) }.tap do |result| result[:style] = style.to_s if style.set? diff --git a/lib/gitlab/ci/artifacts/logger.rb b/lib/gitlab/ci/artifacts/logger.rb index 628f4129df4..63064118232 100644 --- a/lib/gitlab/ci/artifacts/logger.rb +++ b/lib/gitlab/ci/artifacts/logger.rb @@ -29,17 +29,19 @@ module Gitlab ) end - def self.log_created(artifact) - payload = Gitlab::ApplicationContext.current.merge( - message: 'Artifact created', - job_artifact_id: artifact.id, - size: artifact.size, - type: artifact.file_type, - build_id: artifact.job_id, - project_id: artifact.project_id - ) + def self.log_created(job_artifacts) + Array(job_artifacts).each do |artifact| + payload = Gitlab::ApplicationContext.current.merge( + message: 'Artifact created', + job_artifact_id: artifact.id, + size: artifact.size, + file_type: artifact.file_type, + build_id: artifact.job_id, + project_id: artifact.project_id + ) - Gitlab::AppLogger.info(payload) + Gitlab::AppLogger.info(payload) + end end def self.log_deleted(job_artifacts, method) @@ -49,7 +51,7 @@ module Gitlab job_artifact_id: artifact.id, expire_at: artifact.expire_at, size: artifact.size, - type: artifact.file_type, + file_type: artifact.file_type, build_id: artifact.job_id, project_id: artifact.project_id, method: method diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb index 4950a7616c8..453c293f6cd 100644 --- a/lib/gitlab/ci/build/auto_retry.rb +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -47,7 +47,9 @@ class Gitlab::Ci::Build::AutoRetry end def options_retry_when - options_retry.fetch(:when, ['always']) + default = ['always'] + + options_retry.fetch(:when, default) || default end def retry_on_reason_or_always? diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index 1025e1cc2d7..81efbdb297b 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -9,24 +9,19 @@ module Gitlab attr_reader :attributes - def initialize(pipeline, attributes = {}, build = nil) + def initialize(pipeline, attributes = {}) super(pipeline) - @build = build @attributes = attributes end def variables - build.scoped_variables + stub_build.scoped_variables end strong_memoize_attr :variables private - def build - @build || stub_build - end - def stub_build # This is a temporary piece of technical debt to allow us access # to the CI variables to evaluate rules before we persist a Build diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb new file mode 100644 index 00000000000..010ce57d2a0 --- /dev/null +++ b/lib/gitlab/ci/components/instance_path.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Components + class InstancePath + include Gitlab::Utils::StrongMemoize + + def self.match?(address) + address.include?('@') && address.start_with?(Settings.gitlab_ci['component_fqdn']) + end + + attr_reader :host + + def initialize(address:, content_filename:) + @full_path, @version = address.to_s.split('@', 2) + @content_filename = content_filename + @host = Settings.gitlab_ci['component_fqdn'] + end + + def fetch_content!(current_user:) + return unless project + return unless sha + + raise Gitlab::Access::AccessDeniedError unless Ability.allowed?(current_user, :download_code, project) + + project.repository.blob_data_at(sha, project_file_path) + end + + def project + find_project_by_component_path(instance_path) + end + strong_memoize_attr :project + + def project_file_path + return unless project + + component_dir = instance_path.delete_prefix(project.full_path) + File.join(component_dir, @content_filename).delete_prefix('/') + end + + # TODO: Add support when version is a released tag and "~latest" moving target + def sha + return unless project + + project.commit(version)&.id + end + strong_memoize_attr :sha + + private + + attr_reader :version, :path + + def instance_path + @full_path.delete_prefix(host) + end + + # Given a path like "my-org/sub-group/the-project/path/to/component" + # find the project "my-org/sub-group/the-project" by looking at all possible paths. + def find_project_by_component_path(path) + possible_paths = [path] + + while index = path.rindex('/') # find index of last `/` in a path + possible_paths << (path = path[0..index - 1]) + end + + # remove shortest path as it is group + possible_paths.pop + + ::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 142f0b8dfd8..585e671ce42 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -117,7 +117,8 @@ module Gitlab def expand_config(config) build_config(config) - rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError, + Gitlab::Config::Loader::MultiDocYaml::DataTooLargeError => e track_and_raise_for_dev_exception(e) raise Config::ConfigError, e.message diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index 368d8f07f8d..baab7ebdb79 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[local file remote template artifact job project ref rules].freeze + ALLOWED_KEYS = %i[local file remote template component artifact job project ref rules].freeze validations do validates :config, hash_or_string: true @@ -36,8 +36,8 @@ module Gitlab end entry :rules, ::Gitlab::Ci::Config::Entry::Include::Rules, - description: 'List of evaluable Rules to determine file inclusion.', - inherit: false + description: 'List of evaluable Rules to determine file inclusion.', + inherit: false attributes :rules diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 138e79db331..6eef279d3de 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -10,6 +10,7 @@ module Gitlab TimeoutError = Class.new(StandardError) MAX_INCLUDES = 100 + NEW_MAX_INCLUDES = 150 # Update to MAX_INCLUDES when FF ci_includes_count_duplicates is removed include ::Gitlab::Utils::StrongMemoize @@ -27,10 +28,10 @@ module Gitlab @user = user @parent_pipeline = parent_pipeline @variables = variables || Ci::Variables::Collection.new - @expandset = Set.new + @expandset = Feature.enabled?(:ci_includes_count_duplicates, project) ? [] : Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - @max_includes = MAX_INCLUDES + @max_includes = Feature.enabled?(:ci_includes_count_duplicates, project) ? NEW_MAX_INCLUDES : MAX_INCLUDES yield self if block_given? end diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 140cbfac5c1..0b90d240a15 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -20,8 +20,6 @@ module Gitlab def content strong_memoize(:content) do - next unless artifact_job - Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) rescue Gitlab::Ci::ArtifactFileReader::Error => error errors.push(error.message) @@ -36,8 +34,6 @@ module Gitlab ) end - private - def validate_context! context.logger.instrument(:config_file_artifact_validate_context) do if !creating_child_pipeline? @@ -54,10 +50,10 @@ module Gitlab errors.push("File `#{masked_location}` is empty!") unless content.present? end + private + def artifact_job strong_memoize(:artifact_job) do - next unless creating_child_pipeline? - context.parent_pipeline.find_job_with_archive_artifacts(job_name) end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 7899fe0ff73..84f34f2584b 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -46,13 +46,6 @@ module Gitlab expanded_content_hash end - def validate! - validate_location! - validate_context! if valid? - fetch_and_validate_content! if valid? - load_and_validate_expanded_hash! if valid? - end - def metadata { context_project: context.project&.full_path, @@ -68,22 +61,16 @@ module Gitlab [params, context.project&.full_path, context.sha].hash end - protected - - def expanded_content_hash - return unless content_hash - - strong_memoize(:expanded_content_yaml) do - expand_includes(content_hash) + def load_and_validate_expanded_hash! + context.logger.instrument(:config_file_fetch_content_hash) do + content_hash # calling the method loads then memoizes the result end - end - def content_hash - strong_memoize(:content_yaml) do - ::Gitlab::Ci::Config::Yaml.load!(content) + context.logger.instrument(:config_file_expand_content_includes) do + expanded_content_hash # calling the method expands then memoizes the result end - rescue Gitlab::Config::Loader::FormatError - nil + + validate_hash! end def validate_location! @@ -98,34 +85,28 @@ module Gitlab raise NotImplementedError, 'subclass must implement validate_context' end - def fetch_and_validate_content! - context.logger.instrument(:config_file_fetch_content) do - content # calling the method fetches then memoizes the result - end - - return if errors.any? - - context.logger.instrument(:config_file_validate_content) do - validate_content! + def validate_content! + if content.blank? + errors.push("Included file `#{masked_location}` is empty or does not exist!") end end - def load_and_validate_expanded_hash! - context.logger.instrument(:config_file_fetch_content_hash) do - content_hash # calling the method loads then memoizes the result - end + protected - context.logger.instrument(:config_file_expand_content_includes) do - expanded_content_hash # calling the method expands then memoizes the result - end + def expanded_content_hash + return unless content_hash - validate_hash! + strong_memoize(:expanded_content_hash) do + expand_includes(content_hash) + end end - def validate_content! - if content.blank? - errors.push("Included file `#{masked_location}` is empty or does not exist!") + def content_hash + strong_memoize(:content_hash) do + ::Gitlab::Ci::Config::Yaml.load!(content) end + rescue Gitlab::Config::Loader::FormatError + nil end def validate_hash! diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb new file mode 100644 index 00000000000..33e7724bf9b --- /dev/null +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Component < Base + extend ::Gitlab::Utils::Override + include Gitlab::Utils::StrongMemoize + + def initialize(params, context) + @location = params[:component] + super + end + + def matching? + super && ::Feature.enabled?(:ci_include_components, context.project) + end + + def content + return unless component_result.success? + + component_result.payload.fetch(:content) + end + strong_memoize_attr :content + + def metadata + super.merge( + type: :component, + location: masked_location, + blob: masked_blob, + raw: nil, + extra: {} + ) + end + + def validate_location! + return unless invalid_location_type? + + errors.push("Included file `#{masked_location}` needs to be a string") + end + + def validate_context! + return if context.project&.repository + + errors.push('Unable to use components outside of a project context') + end + + def validate_content! + return if content.present? + + errors.push(component_result.message) + end + + private + + attr_reader :path, :version + + def component_result + ::Ci::Components::FetchService.new( + address: location, + current_user: context.user + ).execute + end + strong_memoize_attr :component_result + + override :expand_context_attrs + def expand_context_attrs + { + project: component_path.project, + sha: component_path.sha, + user: context.user, + variables: context.variables + } + end + + def masked_blob + return unless component_path + + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_blob_url( + component_path.project, + ::File.join(component_path.sha, component_path.project_file_path)) + ) + end + strong_memoize_attr :masked_blob + + def component_path + return unless component_result.success? + + component_result.payload.fetch(:path) + end + strong_memoize_attr :component_path + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 0912a732158..bb1c304d02b 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -10,7 +10,8 @@ module Gitlab include Gitlab::Utils::StrongMemoize def initialize(params, context) - @location = params[:local] + # `Repository#blobs_at` does not support files with the `/` prefix. + @location = Gitlab::Utils.remove_leading_slashes(params[:local]) super end @@ -29,8 +30,6 @@ module Gitlab ) end - private - def validate_context! return if context.project&.repository @@ -45,14 +44,19 @@ module Gitlab end end + private + def fetch_local_content - context.logger.instrument(:config_file_fetch_local_content) do - context.project.repository.blob_data_at(context.sha, location) + BatchLoader.for([context.sha, location]) + .batch(key: context.project) do |locations, loader, args| + context.logger.instrument(:config_file_fetch_local_content) do + args[:key].repository.blobs_at(locations).each do |blob| + loader.call([blob.commit_id, blob.path], blob.data) + end + end + rescue GRPC::InvalidArgument + # no-op end - rescue GRPC::InvalidArgument - errors.push("Sha #{context.sha} is not valid!") - - nil end override :expand_context_attrs @@ -67,6 +71,8 @@ module Gitlab end def masked_blob + return unless valid? + strong_memoize(:masked_blob) do context.mask_variables_from( Gitlab::Routing.url_helpers.project_blob_url(context.project, ::File.join(context.sha, location)) @@ -75,7 +81,7 @@ module Gitlab end def masked_raw - return unless context.project + return unless valid? strong_memoize(:masked_raw) do context.mask_variables_from( diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 553cbd819ad..f8d4cb27710 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -12,7 +12,9 @@ module Gitlab attr_reader :project_name, :ref_name def initialize(params, context) - @location = params[:file] + # `Repository#blobs_at` does not support files with the `/` prefix. + @location = Gitlab::Utils.remove_leading_slashes(params[:file]) + @project_name = get_project_name(params[:project]) @ref_name = params[:ref] || 'HEAD' @@ -37,8 +39,6 @@ module Gitlab ) end - private - def validate_context! if !can_access_local_content? errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") @@ -55,6 +55,8 @@ module Gitlab end end + private + def project strong_memoize(:project) do ::Project.find_by_full_path(project_name) @@ -70,19 +72,19 @@ module Gitlab end def fetch_local_content - return unless can_access_local_content? - return unless sha - - context.logger.instrument(:config_file_fetch_project_content) do - project.repository.blob_data_at(sha, location) + BatchLoader.for([sha, location]) + .batch(key: project) do |locations, loader, args| + context.logger.instrument(:config_file_fetch_project_content) do + args[:key].repository.blobs_at(locations).each do |blob| + loader.call([blob.commit_id, blob.path], blob.data) + end + end + rescue GRPC::NotFound, GRPC::Internal + # no-op end - rescue GRPC::NotFound, GRPC::Internal - nil end def sha - return unless project - strong_memoize(:sha) do project.commit(ref_name).try(:sha) end @@ -112,7 +114,7 @@ module Gitlab end def masked_blob - return unless project + return unless valid? strong_memoize(:masked_blob) do context.mask_variables_from( @@ -122,7 +124,7 @@ module Gitlab end def masked_raw - return unless project + return unless valid? strong_memoize(:masked_raw) do context.mask_variables_from( diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index ed37357dc53..bc8cebb8c3e 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -28,8 +28,6 @@ module Gitlab ) end - private - def validate_context! # no-op end @@ -42,6 +40,8 @@ module Gitlab end end + private + def fetch_remote_content begin response = context.logger.instrument(:config_file_fetch_remote_content) do diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 53236cb317b..093b945da4b 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -31,8 +31,6 @@ module Gitlab ) end - private - def validate_context! # no-op end @@ -45,6 +43,8 @@ module Gitlab end end + private + def template_name return unless template_name_valid? diff --git a/lib/gitlab/ci/config/external/mapper/base.rb b/lib/gitlab/ci/config/external/mapper/base.rb index d2f56d0b8f6..d898ddb356c 100644 --- a/lib/gitlab/ci/config/external/mapper/base.rb +++ b/lib/gitlab/ci/config/external/mapper/base.rb @@ -11,9 +11,9 @@ module Gitlab @context = context end - def process(*args) + def process(...) context.logger.instrument(mapper_instrumentation_key) do - process_without_instrumentation(*args) + process_without_instrumentation(...) end end diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb index 85e19ff1ced..e59eaa6d324 100644 --- a/lib/gitlab/ci/config/external/mapper/matcher.rb +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -10,6 +10,7 @@ module Gitlab FILE_CLASSES = [ External::File::Local, External::File::Project, + External::File::Component, External::File::Remote, External::File::Template, External::File::Artifact @@ -29,11 +30,11 @@ module Gitlab matching.first elsif matching.empty? raise Mapper::AmbigiousSpecificationError, - "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ - "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ + "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" else raise Mapper::AmbigiousSpecificationError, - "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" end end end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 6d6f227b940..2982b0efb6c 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -10,13 +10,29 @@ module Gitlab private def process_without_instrumentation(files) - files.select do |file| + files.each do |file| + verify_execution_time! + + file.validate_location! + file.validate_context! if file.valid? + file.content if file.valid? + end + + # We do not combine the loops because we need to load the content of all files before continuing + # to call `BatchLoader` for all locations. + files.each do |file| # rubocop:disable Style/CombinableLoops + # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150 verify_max_includes! verify_execution_time! - file.validate! + file.validate_content! if file.valid? + file.load_and_validate_expanded_hash! if file.valid? - context.expandset.add(file) + if context.expandset.is_a?(Array) # To be removed when FF 'ci_includes_count_duplicates' is removed + context.expandset << file + else + context.expandset.add(file) + end end end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index de833619c8d..94ef0afe7f9 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -5,12 +5,21 @@ module Gitlab class Config module Yaml AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze + MAX_DOCUMENTS = 2 class << self def load!(content) ensure_custom_tags - Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + if ::Feature.enabled?(:ci_multi_doc_yaml) + Gitlab::Config::Loader::MultiDocYaml.new( + content, + max_documents: MAX_DOCUMENTS, + additional_permitted_classes: AVAILABLE_TAGS + ).load!.first + else + Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + end end private diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb index 45787077c91..5ecab033109 100644 --- a/lib/gitlab/ci/config/yaml/tags/reference.rb +++ b/lib/gitlab/ci/config/yaml/tags/reference.rb @@ -16,7 +16,7 @@ module Gitlab def valid? data[:seq].is_a?(Array) && !data[:seq].empty? && - data[:seq].all? { |identifier| identifier.is_a?(String) } + data[:seq].all?(String) end private diff --git a/lib/gitlab/ci/interpolation/access.rb b/lib/gitlab/ci/interpolation/access.rb new file mode 100644 index 00000000000..42598458902 --- /dev/null +++ b/lib/gitlab/ci/interpolation/access.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + class Access + attr_reader :content, :errors + + MAX_ACCESS_OBJECTS = 5 + MAX_ACCESS_BYTESIZE = 1024 + + def initialize(access, ctx) + @content = access + @ctx = ctx + @errors = [] + + if objects.count <= 1 # rubocop:disable Style/IfUnlessModifier + @errors.push('invalid interpolation access pattern') + end + + if access.bytesize > MAX_ACCESS_BYTESIZE # rubocop:disable Style/IfUnlessModifier + @errors.push('maximum interpolation expression size exceeded') + end + + evaluate! if valid? + end + + def valid? + errors.none? + end + + def objects + @objects ||= @content.split('.', MAX_ACCESS_OBJECTS) + end + + def value + raise ArgumentError, 'access path invalid' unless valid? + + @value + end + + private + + def evaluate! + raise ArgumentError, 'access path invalid' unless valid? + + @value ||= objects.inject(@ctx) do |memo, value| + memo.fetch(value.to_sym) + end + rescue KeyError => e + @errors.push(e) + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/block.rb b/lib/gitlab/ci/interpolation/block.rb new file mode 100644 index 00000000000..389cbf378a2 --- /dev/null +++ b/lib/gitlab/ci/interpolation/block.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + class Block + PREFIX = '$[[' + PATTERN = /(?<block>\$\[\[\s*(?<access>.*?)\s*\]\])/.freeze + + attr_reader :block, :data, :ctx + + def initialize(block, data, ctx) + @block = block + @ctx = ctx + @data = data + + @access = Interpolation::Access.new(@data, ctx) + end + + def valid? + errors.none? + end + + def errors + @access.errors + end + + def content + @access.content + end + + def value + raise ArgumentError, 'block invalid' unless valid? + + @access.value + end + + def self.match(data) + return data unless data.is_a?(String) && data.include?(PREFIX) + + data.gsub(PATTERN) do + yield ::Regexp.last_match(1), ::Regexp.last_match(2) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/config.rb b/lib/gitlab/ci/interpolation/config.rb new file mode 100644 index 00000000000..32f58521139 --- /dev/null +++ b/lib/gitlab/ci/interpolation/config.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + ## + # Interpolation::Config represents a configuration artifact that we want to perform interpolation on. + # + class Config + include Gitlab::Utils::StrongMemoize + ## + # Total number of hash nodes traversed. For example, loading a YAML below would result in a hash having 12 nodes + # instead of 9, because hash values are being counted before we recursively traverse them. + # + # test: + # spec: + # env: $[[ inputs.env ]] + # + # $[[ inputs.key ]]: + # name: $[[ inputs.key ]] + # script: my-value + # + # According to our benchmarks performed when developing this code, the worst-case scenario of processing + # a hash with 500_000 nodes takes around 1 second and consumes around 225 megabytes of memory. + # + # The typical scenario, using just a few interpolations takes 250ms and consumes around 20 megabytes of memory. + # + # Given the above the 500_000 nodes should be an upper limit, provided that the are additional safeguard + # present in other parts of the code (example: maximum number of interpolation blocks found). Typical size of a + # YAML configuration with 500k nodes might be around 10 megabytes, which is an order of magnitude higher than + # the 1MB limit for loading YAML on GitLab.com + # + MAX_NODES = 500_000 + MAX_NODE_SIZE = 1024 * 1024 # 1MB + + TooManyNodesError = Class.new(StandardError) + NodeTooLargeError = Class.new(StandardError) + + Visitor = Class.new do + def initialize + @visited = 0 + end + + def visit! + @visited += 1 + + raise Config::TooManyNodesError if @visited > Config::MAX_NODES + end + end + + attr_reader :errors + + def initialize(hash) + @config = hash + @errors = [] + end + + def to_h + @config + end + + ## + # The replace! method will yield a block and replace a each of the hash config nodes with a return value of the + # block. + # + # It returns `nil` if there were errors found during the process. + # + def replace!(&block) + recursive_replace(@config, Visitor.new, &block) + rescue TooManyNodesError + @errors.push('config too large') + nil + rescue NodeTooLargeError + @errors.push('config node too large') + nil + end + strong_memoize_attr :replace! + + def self.fabricate(config) + case config + when Hash + new(config) + when Interpolation::Config + config + else + raise ArgumentError, 'unknown interpolation config' + end + end + + private + + def recursive_replace(config, visitor, &block) + visitor.visit! + + case config + when Hash + {}.tap do |new_hash| + config.each_pair do |key, value| + new_key = recursive_replace(key, visitor, &block) + new_value = recursive_replace(value, visitor, &block) + + if new_key != key + new_hash[new_key] = new_value + else + new_hash[key] = new_value + end + end + end + when Array + config.map { |value| recursive_replace(value, visitor, &block) } + when Symbol + recursive_replace(config.to_s, visitor, &block) + when String + raise NodeTooLargeError if config.bytesize > MAX_NODE_SIZE + + yield config + else + config + end + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/context.rb b/lib/gitlab/ci/interpolation/context.rb new file mode 100644 index 00000000000..ce7a86a3c9b --- /dev/null +++ b/lib/gitlab/ci/interpolation/context.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + ## + # Interpolation::Context is a class that represents the data that can be used when performing string interpolation + # on a CI configuration. + # + class Context + ContextTooComplexError = Class.new(StandardError) + NotSymbolizedContextError = Class.new(StandardError) + + MAX_DEPTH = 3 + + def initialize(hash) + @context = hash + + raise ContextTooComplexError if depth > MAX_DEPTH + end + + def valid? + errors.none? + end + + ## + # This method is here because `Context` will be responsible for validating specs, inputs and defaults. + # + def errors + [] + end + + def depth + deep_depth(@context) + end + + def fetch(field) + @context.fetch(field) + end + + def to_h + @context.to_h + end + + private + + def deep_depth(context, depth = 0) + values = context.values.map do |value| + if value.is_a?(Hash) + deep_depth(value, depth + 1) + else + depth + 1 + end + end + + values.max + end + + def self.fabricate(context) + case context + when Hash + new(context) + when Interpolation::Context + context + else + raise ArgumentError, 'unknown interpolation context' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/template.rb b/lib/gitlab/ci/interpolation/template.rb new file mode 100644 index 00000000000..0211279f266 --- /dev/null +++ b/lib/gitlab/ci/interpolation/template.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Interpolation + class Template + include Gitlab::Utils::StrongMemoize + + attr_reader :blocks, :ctx + + TooManyBlocksError = Class.new(StandardError) + InvalidBlockError = Class.new(StandardError) + + MAX_BLOCKS = 10_000 + + def initialize(config, ctx) + @config = Interpolation::Config.fabricate(config) + @ctx = Interpolation::Context.fabricate(ctx) + @errors = [] + @blocks = {} + + interpolate! if valid? + end + + def valid? + errors.none? + end + + def errors + @errors + @config.errors + @ctx.errors + @blocks.values.flat_map(&:errors) + end + + def size + @blocks.size + end + + def interpolated + @result if valid? + end + + private + + def interpolate! + @result = @config.replace! do |data| + Interpolation::Block.match(data) do |block, data| + evaluate_block(block, data) + end + end + rescue TooManyBlocksError + @errors.push('too many interpolation blocks') + rescue InvalidBlockError + @errors.push('interpolation interrupted by errors') + end + strong_memoize_attr :interpolate! + + def evaluate_block(block, data) + block = (@blocks[block] ||= Interpolation::Block.new(block, data, ctx)) + + raise TooManyBlocksError if @blocks.count > MAX_BLOCKS + raise InvalidBlockError unless block.valid? + + block.value + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/instrumentation.rb b/lib/gitlab/ci/parsers/instrumentation.rb index ab4a923d9aa..5e97b22ecdf 100644 --- a/lib/gitlab/ci/parsers/instrumentation.rb +++ b/lib/gitlab/ci/parsers/instrumentation.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentation BUCKETS = [0.25, 1, 5, 10].freeze - def parse!(*args) + def parse!(...) parser_result = nil duration = Benchmark.realtime do diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 67817c9f832..1b9afc92d6b 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -282,7 +282,12 @@ module Gitlab return data['name'] if data['name'].present? identifier = identifiers.find(&:cve?) || identifiers.find(&:cwe?) || identifiers.first - "#{identifier.name} in #{location&.fingerprint_path}" + + if location&.fingerprint_path + "#{identifier.name} in #{location.fingerprint_path}" + else + identifier.name.to_s + end end def calculate_uuid_v5(primary_identifier, location_fingerprint) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 53c8a7ac122..dcaaefee98f 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -5,59 +5,13 @@ module Gitlab module Pipeline module Chain class CancelPendingPipelines < Chain::Base - include Chain::Helpers - - BATCH_SIZE = 25 - - # rubocop: disable CodeReuse/ActiveRecord def perform! - return if pipeline.parent_pipeline? # skip if child pipeline - return unless project.auto_cancel_pending_pipelines? - - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| - cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| - auto_cancel_interruptible_pipelines(cancelables_batch.ids) - end - end + ::Ci::CancelRedundantPipelinesWorker.perform_async(pipeline.id) end - # rubocop: enable CodeReuse/ActiveRecord def break? false end - - private - - def auto_cancelable_pipelines - project.all_pipelines - .created_after(1.week.ago) - .ci_and_parent_sources - .for_ref(pipeline.ref) - .where_not_sha(project.commit(pipeline.ref).try(:id)) - .alive_or_scheduled - .id_not_in(pipeline.id) - end - - def auto_cancel_interruptible_pipelines(pipeline_ids) - ::Ci::Pipeline - .id_in(pipeline_ids) - .with_only_interruptible_builds - .each do |cancelable_pipeline| - Gitlab::AppLogger.info( - class: self.class.name, - message: "Pipeline #{pipeline.id} auto-canceling pipeline #{cancelable_pipeline.id}", - canceled_pipeline_id: cancelable_pipeline.id, - canceled_by_pipeline_id: pipeline.id, - canceled_by_pipeline_source: pipeline.source - ) - - # cascade_to_children not needed because we iterate through descendants here - cancelable_pipeline.cancel_running( - auto_canceled_by_pipeline_id: pipeline.id, - cascade_to_children: false - ) - end - end end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb deleted file mode 100644 index 99e438ddbae..00000000000 --- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - class CreateDeployments < Chain::Base - def perform! - create_deployments! if Feature.disabled?(:move_create_deployments_to_worker, pipeline.project) - end - - def break? - false - end - - private - - def create_deployments! - pipeline.stages.map(&:statuses).flatten.map(&method(:create_deployment)) - end - - def create_deployment(build) - ::Deployments::CreateForBuildService.new.execute(build) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb index b17ae77d445..b886aa22ba3 100644 --- a/lib/gitlab/ci/pipeline/chain/metrics.rb +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -6,15 +6,27 @@ module Gitlab module Chain class Metrics < Chain::Base def perform! - counter.increment(source: @pipeline.source) + increment_pipeline_created_counter + create_snowplow_event_for_pipeline_name end def break? false end - def counter - ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter + def increment_pipeline_created_counter + ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter.increment(source: @pipeline.source) + end + + def create_snowplow_event_for_pipeline_name + return unless @pipeline.pipeline_metadata&.name + + Gitlab::Tracking.event( + self.class.name, + 'create_pipeline_with_name', + project: @pipeline.project, + user: @pipeline.user, + namespace: @pipeline.project.namespace) end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 684b58474ad..484e18c6979 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -9,13 +9,12 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(context, attributes, stages_for_needs_lookup, stage) + def initialize(context, attributes, stages_for_needs_lookup) @context = context @pipeline = context.pipeline @seed_attributes = attributes @stages_for_needs_lookup = stages_for_needs_lookup.compact @needs_attributes = dig(:needs_attributes) - @stage = stage @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true } @@ -34,8 +33,6 @@ module Gitlab .new(attributes.delete(:cache), @pipeline) calculate_yaml_variables! - - @processable = initialize_processable end def name @@ -66,20 +63,13 @@ module Gitlab end strong_memoize_attr :errors - # TODO: Method used only in specs. Replace with `to_resource.attributes` when - # the feature flag ci_reuse_build_in_seed_context is removed. - # Then remove this method. def attributes - if reuse_build_in_seed_context? - initial_attributes.deep_merge(evaluated_attributes) - else - @seed_attributes - .deep_merge(pipeline_attributes) - .deep_merge(rules_attributes) - .deep_merge(allow_failure_criteria_attributes) - .deep_merge(@cache.cache_attributes) - .deep_merge(runner_tags) - end + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(rules_attributes) + .deep_merge(allow_failure_criteria_attributes) + .deep_merge(@cache.cache_attributes) + .deep_merge(runner_tags) end def bridge? @@ -91,17 +81,10 @@ module Gitlab def to_resource logger.instrument(:pipeline_seed_build_to_resource) do - if reuse_build_in_seed_context? - # The `options` attribute need to be entirely reassigned because they may - # be overridden by evaluated_attributes. - # We also don't want to reassign all the `initial_attributes` since those - # can affect performance. We only want to assign what's changed. - assignable_attributes = initial_attributes.slice(:options) - .deep_merge(evaluated_attributes) - processable.assign_attributes(assignable_attributes) - processable + if bridge? + ::Ci::Bridge.new(attributes) else - legacy_initialize_processable + ::Ci::Build.new(attributes) end end end @@ -109,18 +92,8 @@ module Gitlab private - attr_reader :processable - delegate :logger, to: :@context - def legacy_initialize_processable - if bridge? - ::Ci::Bridge.new(attributes) - else - ::Ci::Build.new(attributes) - end - end - def initialize_processable return unless reuse_build_in_seed_context? @@ -131,19 +104,6 @@ module Gitlab end end - def initial_attributes - @seed_attributes - .deep_merge(pipeline_attributes) - .deep_merge(ci_stage: @stage) - .deep_merge(@cache.cache_attributes) - end - - def evaluated_attributes - rules_attributes - .deep_merge(allow_failure_criteria_attributes) - .deep_merge(runner_tags) - end - def all_of_only? @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end @@ -223,11 +183,7 @@ module Gitlab strong_memoize_attr :rules_errors def evaluate_context - if reuse_build_in_seed_context? - Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes, processable) - else - Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) - end + Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) end strong_memoize_attr :evaluate_context @@ -257,11 +213,6 @@ module Gitlab from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance ) end - - def reuse_build_in_seed_context? - Feature.enabled?(:ci_reuse_build_in_seed_context, @pipeline.project) - end - strong_memoize_attr :reuse_build_in_seed_context? end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index c3e94529634..42970d8f5ac 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,25 +10,25 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - attr_reader :attributes - - def initialize(context, stage_attributes, previous_stages) - pipeline = context.pipeline - @attributes = { - name: stage_attributes.fetch(:name), - position: stage_attributes.fetch(:index), - pipeline: pipeline, - project: pipeline.project, - partition_id: pipeline.partition_id - } - - @stage = ::Ci::Stage.new(@attributes) - - @builds = stage_attributes.fetch(:builds).map do |build_attributes| - Seed::Build.new(context, build_attributes, previous_stages + [self], @stage) + def initialize(context, attributes, previous_stages) + @context = context + @pipeline = context.pipeline + @attributes = attributes + @previous_stages = previous_stages + + @builds = attributes.fetch(:builds).map do |attributes| + Seed::Build.new(context, attributes, previous_stages + [self]) end end + def attributes + { name: @attributes.fetch(:name), + position: @attributes.fetch(:index), + pipeline: @pipeline, + project: @pipeline.project, + partition_id: @pipeline.partition_id } + end + def seeds @builds.select(&:included?) end @@ -49,8 +49,9 @@ module Gitlab end def to_resource - @stage.statuses = seeds.map(&:to_resource) - @stage + ::Ci::Stage.new(attributes).tap do |stage| + stage.statuses = seeds.map(&:to_resource) + end end strong_memoize_attr :to_resource end diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb index 3196bf3fc6d..aba2d2e8b19 100644 --- a/lib/gitlab/ci/reports/codequality_reports.rb +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -42,6 +42,18 @@ module Gitlab rescue StandardError => _ false end + + def code_quality_report_summary + report_degradations = @degradations.presence + return if report_degradations.nil? + + summary = ::Gitlab::Ci::Reports::CodequalityReports::SEVERITY_PRIORITIES.keys.index_with(0) + report_degradations.each_value do |degradation| + summary[degradation[:severity]] += 1 + end + summary['count'] = summary.values.sum + summary + end end end end diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 5188304f4ed..08307580987 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -31,8 +31,10 @@ module Gitlab end def supported_purl_type? + # the purl type is not required as per the spec: https://cyclonedx.org/docs/1.4/json/#components_items_purl return true unless purl + # however, if the purl type is provided, it _must be valid_ ::Enums::Sbom.purl_types.include?(purl.type.to_sym) end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index bcda2fec5ba..1bf015a0aa0 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -47,6 +47,9 @@ module Gitlab kubernetes: { human_readable_name: "Kubernetes", installation_instructions_url: "https://docs.gitlab.com/runner/install/kubernetes.html" + }, + aws: { + human_readable_name: "AWS" } }.freeze diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index 03130addd6a..19a8dc446d4 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -42,7 +42,7 @@ module Gitlab # Consider update if there's a newer release within the currently deployed GitLab version add_available_runner_release(runner_version, suggestions) - suggestions[runner_version] = :not_available if suggestions.empty? + suggestions[runner_version] = :unavailable if suggestions.empty? suggestions end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index d66d4b20bba..8e9eda560a7 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -6,7 +6,7 @@ module Gitlab module Bridge module Common def label - subject.description + subject.description.presence || super end def has_details? diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index 4d5a94a3beb..fbe2e2282a0 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -6,11 +6,13 @@ module Gitlab module Bridge class Factory < Status::Factory def self.extended_statuses - [[Status::Bridge::Failed], + [[Status::Bridge::Retryable], + [Status::Bridge::Failed], [Status::Bridge::Manual], [Status::Bridge::WaitingForResource], [Status::Bridge::Play], - [Status::Bridge::Action]] + [Status::Bridge::Action], + [Status::Bridge::Retried]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/bridge/retried.rb b/lib/gitlab/ci/status/bridge/retried.rb new file mode 100644 index 00000000000..101974f10f2 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/retried.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Retried < Status::Build::Retried + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/retryable.rb b/lib/gitlab/ci/status/bridge/retryable.rb new file mode 100644 index 00000000000..203a0b8d5ca --- /dev/null +++ b/lib/gitlab/ci/status/bridge/retryable.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Retryable < Status::Build::Retryable + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index c66b8ca5654..55e510166b4 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -22,7 +22,7 @@ module Gitlab end def action_button_title - _('Trigger this manual action') + _('Run job') end def action_path diff --git a/lib/gitlab/ci/status/waiting_for_resource.rb b/lib/gitlab/ci/status/waiting_for_resource.rb index 2026148f752..9ced0aadb88 100644 --- a/lib/gitlab/ci/status/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/waiting_for_resource.rb @@ -17,7 +17,7 @@ module Gitlab end def favicon - 'favicon_pending' + 'favicon_status_pending' end def group diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index fc1f4f0cce8..40f5109851b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.21.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.28.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index fc1f4f0cce8..40f5109851b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.21.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.28.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 6884a9556b4..aa2356f6a34 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 222f534387a..eb8e5de5b56 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -15,6 +15,7 @@ variables: DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 3 + DS_SCHEMA_MODEL: 15 dependency_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml index 67057e916a8..655ac6ee712 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -15,6 +15,7 @@ variables: DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 3 + DS_SCHEMA_MODEL: 15 dependency_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index dc7e5f445d2..372b782c0a0 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 9e15b07f5d1..feba2efcf22 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml new file mode 100644 index 00000000000..4beaaae9d93 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml @@ -0,0 +1,23 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml + +include: + - template: Terraform/Module-Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml + - template: Jobs/SAST-IaC.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml + +stages: + - validate + - build + - test + - deploy + +fmt: + extends: .terraform-module:fmt + +# See the included job template at `Terraform/Module-Base.gitlab-ci.yml` to learn about supported variables. +deploy: + extends: .terraform-module:deploy + rules: + - if: $CI_COMMIT_TAG diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 0b6c10293fc..f3f736e96c4 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -24,6 +24,9 @@ validate: build: extends: .terraform:build + environment: + name: $TF_STATE_NAME + action: prepare deploy: extends: .terraform:deploy @@ -31,3 +34,4 @@ deploy: - build environment: name: $TF_STATE_NAME + action: start diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 9c967d48de1..bc23a7c2a95 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -69,6 +69,7 @@ cache: - gitlab-terraform apply resource_group: ${TF_STATE_NAME} rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $TF_AUTO_DEPLOY == "true" - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: manual diff --git a/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml new file mode 100644 index 00000000000..e73e6194760 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml @@ -0,0 +1,39 @@ +# Terraform/Module-Base +# +# The purpose of this template is to provide flexibility to the user so +# they are able to only include the jobs that they find interesting. +# +# Therefore, this template is not supposed to run any jobs. The idea is to only +# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs +# +# There is a more opinionated template which we suggest the users to abide, +# which is the lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml + +# These variables may be overridden by the pipeline including it to control how the Terraform module is being deployed. +variables: + TERRAFORM_MODULE_DIR: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project. + TERRAFORM_MODULE_NAME: ${CI_PROJECT_NAME} # The name of your Terraform module, must not have any spaces or underscores (will be translated to hyphens). + TERRAFORM_MODULE_SYSTEM: local # The system or provider your Terraform module targets (ex. local, aws, google). + TERRAFORM_MODULE_VERSION: ${CI_COMMIT_TAG} # The version - it's recommended to follow SemVer for Terraform Module Versioning. + +.terraform-module:fmt: + stage: validate + image: $CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest + variables: + TF_ROOT: $TERRAFORM_MODULE_DIR + script: + - gitlab-terraform fmt + allow_failure: true + +.terraform-module:deploy: + stage: deploy + image: $CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest + script: + - TERRAFORM_MODULE_NAME=$(echo "${TERRAFORM_MODULE_NAME}" | tr " _" -) # module-name must not have spaces or underscores, so translate them to hyphens + # Builds the Terraform module artifact: a gzipped tar archive with the contents from `$TERRAFORM_MODULE_DIR` without a `.git` directory. + - tar -vczf /tmp/${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz -C ${TERRAFORM_MODULE_DIR} --exclude=./.git . + # Uploads the Terraform module artifact to the GitLab Terraform Module Registry, see + # docs/user/packages/terraform_module_registry/index.html#publish-a-terraform-module + - 'curl --fail-with-body --location --header "JOB-TOKEN: ${CI_JOB_TOKEN}" + --upload-file /tmp/${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz + ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/terraform/modules/${TERRAFORM_MODULE_NAME}/${TERRAFORM_MODULE_SYSTEM}/${TERRAFORM_MODULE_VERSION}/file' diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 8e18d57b724..89d681c418d 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -8,6 +8,7 @@ module Gitlab def initialize(pipeline) @pipeline = pipeline + @pipeline_variables_builder = Builder::Pipeline.new(pipeline) @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) @group_variables_builder = Builder::Group.new(project&.group) @@ -18,7 +19,7 @@ module Gitlab Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables(job)) variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) + variables.concat(pipeline_variables_builder.predefined_variables) variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner variables.concat(kubernetes_variables(environment: environment, job: job)) variables.concat(job.yaml_variables) @@ -38,7 +39,7 @@ module Gitlab break variables unless project variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) + variables.concat(pipeline_variables_builder.predefined_variables) variables.concat(secret_instance_variables) variables.concat(secret_group_variables(environment: nil)) variables.concat(secret_project_variables(environment: nil)) @@ -117,6 +118,7 @@ module Gitlab private attr_reader :pipeline + attr_reader :pipeline_variables_builder attr_reader :instance_variables_builder attr_reader :project_variables_builder attr_reader :group_variables_builder diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb new file mode 100644 index 00000000000..96d6f1673b9 --- /dev/null +++ b/lib/gitlab/ci/variables/builder/pipeline.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + class Pipeline + include Gitlab::Utils::StrongMemoize + + def initialize(pipeline) + @pipeline = pipeline + end + + def predefined_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s) + variables.append(key: 'CI_PIPELINE_SOURCE', value: pipeline.source.to_s) + variables.append(key: 'CI_PIPELINE_CREATED_AT', value: pipeline.created_at&.iso8601) + + variables.concat(predefined_commit_variables) if pipeline.sha.present? + variables.concat(predefined_commit_tag_variables) if pipeline.tag? + variables.concat(predefined_merge_request_variables) if pipeline.merge_request? + + if pipeline.open_merge_requests_refs.any? + variables.append(key: 'CI_OPEN_MERGE_REQUESTS', value: pipeline.open_merge_requests_refs.join(',')) + end + + variables.append(key: 'CI_GITLAB_FIPS_MODE', value: 'true') if Gitlab::FIPS.enabled? + + variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if pipeline.has_kubernetes_active? + variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') if pipeline.freeze_period? + + if pipeline.external_pull_request_event? && pipeline.external_pull_request + variables.concat(pipeline.external_pull_request.predefined_variables) + end + end + end + + private + + attr_reader :pipeline + + def predefined_commit_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + next variables unless pipeline.sha.present? + + variables.append(key: 'CI_COMMIT_SHA', value: pipeline.sha) + variables.append(key: 'CI_COMMIT_SHORT_SHA', value: pipeline.short_sha) + variables.append(key: 'CI_COMMIT_BEFORE_SHA', value: pipeline.before_sha) + variables.append(key: 'CI_COMMIT_REF_NAME', value: pipeline.source_ref) + variables.append(key: 'CI_COMMIT_REF_SLUG', value: pipeline.source_ref_slug) + variables.append(key: 'CI_COMMIT_BRANCH', value: pipeline.ref) if pipeline.branch? + variables.append(key: 'CI_COMMIT_MESSAGE', value: pipeline.git_commit_message.to_s) + variables.append(key: 'CI_COMMIT_TITLE', value: pipeline.git_commit_full_title.to_s) + variables.append(key: 'CI_COMMIT_DESCRIPTION', value: pipeline.git_commit_description.to_s) + variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!pipeline.protected_ref?).to_s) + variables.append(key: 'CI_COMMIT_TIMESTAMP', value: pipeline.git_commit_timestamp.to_s) + variables.append(key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text.to_s) + + variables.concat(legacy_predefined_commit_variables) + end + end + strong_memoize_attr :predefined_commit_variables + + def legacy_predefined_commit_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_BUILD_REF', value: pipeline.sha) + variables.append(key: 'CI_BUILD_BEFORE_SHA', value: pipeline.before_sha) + variables.append(key: 'CI_BUILD_REF_NAME', value: pipeline.source_ref) + variables.append(key: 'CI_BUILD_REF_SLUG', value: pipeline.source_ref_slug) + end + end + strong_memoize_attr :legacy_predefined_commit_variables + + def predefined_commit_tag_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + git_tag = pipeline.project.repository.find_tag(pipeline.ref) + + next variables unless git_tag + + variables.append(key: 'CI_COMMIT_TAG', value: pipeline.ref) + variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message) + + variables.concat(legacy_predefined_commit_tag_variables) + end + end + strong_memoize_attr :predefined_commit_tag_variables + + def legacy_predefined_commit_tag_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_BUILD_TAG', value: pipeline.ref) + end + end + strong_memoize_attr :legacy_predefined_commit_tag_variables + + def predefined_merge_request_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: pipeline.merge_request_event_type.to_s) + variables.append(key: 'CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', value: pipeline.source_sha.to_s) + variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_SHA', value: pipeline.target_sha.to_s) + + if merge_request_diff.present? + variables.append(key: 'CI_MERGE_REQUEST_DIFF_ID', value: merge_request_diff.id.to_s) + variables.append(key: 'CI_MERGE_REQUEST_DIFF_BASE_SHA', value: merge_request_diff.base_commit_sha) + end + + variables.concat(pipeline.merge_request.predefined_variables) + end + end + strong_memoize_attr :predefined_merge_request_variables + + def merge_request_diff + pipeline.merge_request_diff + end + strong_memoize_attr :merge_request_diff + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 9e6a3d86e92..fad2260d818 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -98,7 +98,7 @@ module Gitlab private def validate_array_of_hashes(value) - value.is_a?(Array) && value.all? { |obj| obj.is_a?(Hash) } + value.is_a?(Array) && value.all?(Hash) end end diff --git a/lib/gitlab/config/loader/multi_doc_yaml.rb b/lib/gitlab/config/loader/multi_doc_yaml.rb new file mode 100644 index 00000000000..346adc79896 --- /dev/null +++ b/lib/gitlab/config/loader/multi_doc_yaml.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Loader + class MultiDocYaml + TooManyDocumentsError = Class.new(Loader::FormatError) + DataTooLargeError = Class.new(Loader::FormatError) + NotHashError = Class.new(Loader::FormatError) + + MULTI_DOC_DIVIDER = /^---$/.freeze + + def initialize(config, max_documents:, additional_permitted_classes: []) + @max_documents = max_documents + @safe_config = load_config(config, additional_permitted_classes) + end + + def load! + raise TooManyDocumentsError, 'The parsed YAML has too many documents' if too_many_documents? + raise DataTooLargeError, 'The parsed YAML is too big' if too_big? + raise NotHashError, 'Invalid configuration format' unless all_hashes? + + safe_config.map(&:deep_symbolize_keys) + end + + private + + attr_reader :safe_config, :max_documents + + def load_config(config, additional_permitted_classes) + config.split(MULTI_DOC_DIVIDER).filter_map do |document| + YAML.safe_load(document, + permitted_classes: [Symbol, *additional_permitted_classes], + permitted_symbols: [], + aliases: true + ) + end + rescue Psych::Exception => e + raise Loader::FormatError, e.message + end + + def all_hashes? + safe_config.all?(Hash) + end + + def too_many_documents? + safe_config.count > max_documents + end + + def too_big? + !deep_sizes.all?(&:valid?) + end + + def deep_sizes + safe_config.map do |config| + Gitlab::Utils::DeepSize.new(config, + max_size: Gitlab::CurrentSettings.current_application_settings.max_yaml_size_bytes, + max_depth: Gitlab::CurrentSettings.current_application_settings.max_yaml_depth) + end + end + end + end + end +end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 8b1298d0561..ceca206b084 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -26,7 +26,7 @@ module Gitlab 'manifest_src' => "'self'", 'media_src' => "'self' data: http: https:", 'script_src' => ContentSecurityPolicy::Directives.script_src, - 'style_src' => "'self' 'unsafe-inline'", + 'style_src' => ContentSecurityPolicy::Directives.style_src, 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", 'object_src' => "'none'", 'report_uri' => nil @@ -43,6 +43,7 @@ module Gitlab allow_websocket_connections(directives) allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? + allow_zuora(directives) if Gitlab.com? # Support for Sentry setup via configuration files will be removed in 16.0 # in favor of Gitlab::CurrentSettings. allow_legacy_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn @@ -128,6 +129,14 @@ module Gitlab append_to_directive(directives, 'frame_src', cdn_host) end + def self.zuora_host + "https://*.zuora.com/apps/PublicHostedPageLite.do" + end + + def self.allow_zuora(directives) + append_to_directive(directives, 'frame_src', zuora_host) + end + def self.append_to_directive(directives, directive, text) directives[directive] = "#{directives[directive]} #{text}".strip end diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb index 4ad420f9e2f..e293e5653c7 100644 --- a/lib/gitlab/content_security_policy/directives.rb +++ b/lib/gitlab/content_security_policy/directives.rb @@ -18,6 +18,10 @@ module Gitlab def self.script_src "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com" end + + def self.style_src + "'self' 'unsafe-inline'" + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 40e2e637114..756d0afa7e4 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,7 +2,7 @@ module Gitlab module Database - DATABASE_NAMES = %w[main ci].freeze + DATABASE_NAMES = %w[main ci main_clusterwide].freeze MAIN_DATABASE_NAME = 'main' CI_DATABASE_NAME = 'ci' @@ -34,10 +34,8 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab-foss/issues/61974 MAX_TEXT_SIZE_LIMIT = 1_000_000 - # Minimum schema version from which migrations are supported # Migrations before this version may have been removed - MIN_SCHEMA_VERSION = 20190506135400 - MIN_SCHEMA_GITLAB_VERSION = '11.11.0' + MIN_SCHEMA_GITLAB_VERSION = '15.0' # Schema we store dynamically managed partitions in (e.g. for time partitioning) DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic @@ -51,6 +49,8 @@ module Gitlab PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym # rubocop:disable Database/MultipleDatabases + FULLY_QUALIFIED_IDENTIFIER = /^\w+\.\w+$/ + def self.database_base_models @database_base_models ||= { # Note that we use ActiveRecord::Base here and not ApplicationRecord. @@ -59,6 +59,7 @@ module Gitlab # that inherit from ActiveRecord::Base; not just our own models that # inherit from ApplicationRecord. main: ::ActiveRecord::Base, + main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil }.compact.with_indifferent_access.freeze end @@ -74,6 +75,7 @@ module Gitlab # that inher from ActiveRecord::Base; not just our own models that # inherit from ApplicationRecord. main: ::ActiveRecord::Base, + main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil }.compact.with_indifferent_access.freeze end @@ -91,6 +93,7 @@ module Gitlab # that inher from ActiveRecord::Base; not just our own models that # inherit from ApplicationRecord. main: ::ActiveRecord::Base, + main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil }.compact.with_indifferent_access.freeze end @@ -102,7 +105,8 @@ module Gitlab gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main gitlab_shared: database_base_models_with_gitlab_shared.values, # all models gitlab_internal: database_base_models.values, # all models - gitlab_pm: [self.database_base_models.fetch(:main)] # package metadata models + gitlab_pm: [self.database_base_models.fetch(:main)], # package metadata models + gitlab_main_clusterwide: [self.database_base_models[:main_clusterwide] || self.database_base_models.fetch(:main)] }.with_indifferent_access.freeze end @@ -125,7 +129,9 @@ module Gitlab end def self.has_config?(database_name) - Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s) + ActiveRecord::Base.configurations + .configs_for(env_name: Rails.env, name: database_name.to_s, include_replicas: true) + .present? end class PgUser < ApplicationRecord @@ -303,6 +309,14 @@ module Gitlab !read_only? end + # Determines minimum viable migration version, determined by the timestamp + # of the earliest migration file. + def self.read_minimum_migration_version + Dir.open( + Rails.root.join('db/migrate') + ).filter_map { |f| /\A\d{14}/.match(f)&.to_s }.map(&:to_i).min + end + # Monkeypatch rails with upgraded database observability def self.install_transaction_metrics_patches! ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics) diff --git a/lib/gitlab/database/indexing_exclusive_lease_guard.rb b/lib/gitlab/database/async_ddl_exclusive_lease_guard.rb index fb45de347e6..5742e96c9b3 100644 --- a/lib/gitlab/database/indexing_exclusive_lease_guard.rb +++ b/lib/gitlab/database/async_ddl_exclusive_lease_guard.rb @@ -2,16 +2,16 @@ module Gitlab module Database - module IndexingExclusiveLeaseGuard + module AsyncDdlExclusiveLeaseGuard extend ActiveSupport::Concern include ExclusiveLeaseGuard def lease_key - @lease_key ||= "gitlab/database/indexing/actions/#{database_config_name}" + @lease_key ||= "gitlab/database/asyncddl/actions/#{database_config_name}" end def database_config_name - Gitlab::Database.db_config_name(connection) + connection_db_config.name end end end diff --git a/lib/gitlab/database/async_foreign_keys.rb b/lib/gitlab/database/async_foreign_keys.rb new file mode 100644 index 00000000000..115ae9ba2e8 --- /dev/null +++ b/lib/gitlab/database/async_foreign_keys.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncForeignKeys + DEFAULT_ENTRIES_PER_INVOCATION = 2 + + def self.validate_pending_entries!(how_many: DEFAULT_ENTRIES_PER_INVOCATION) + PostgresAsyncForeignKeyValidation.ordered.limit(how_many).each do |record| + ForeignKeyValidator.new(record).perform + end + end + end + end +end diff --git a/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb b/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb new file mode 100644 index 00000000000..5958c56a45a --- /dev/null +++ b/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncForeignKeys + class ForeignKeyValidator + include AsyncDdlExclusiveLeaseGuard + + TIMEOUT_PER_ACTION = 1.day + STATEMENT_TIMEOUT = 12.hours + + def initialize(async_validation) + @async_validation = async_validation + end + + def perform + try_obtain_lease do + if foreign_key_exists? + log_index_info("Starting to validate foreign key") + validate_foreign_with_error_handling + log_index_info("Finished validating foreign key") + else + log_index_info(skip_log_message) + async_validation.destroy! + end + end + end + + private + + attr_reader :async_validation + + delegate :connection, :name, :table_name, :connection_db_config, to: :async_validation + + def foreign_key_exists? + relation = if table_name =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + Gitlab::Database::PostgresForeignKey.by_constrained_table_identifier(table_name) + else + Gitlab::Database::PostgresForeignKey.by_constrained_table_name(table_name) + end + + relation.by_name(name).exists? + end + + def validate_foreign_with_error_handling + validate_foreign_key + async_validation.destroy! + rescue StandardError => error + async_validation.handle_exception!(error) + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::AppLogger.error(message: error.message, **logging_options) + end + + def validate_foreign_key + set_statement_timeout do + connection.execute(<<~SQL.squish) + ALTER TABLE #{connection.quote_table_name(table_name)} + VALIDATE CONSTRAINT #{connection.quote_column_name(name)}; + SQL + end + end + + def set_statement_timeout + connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT)) + yield + ensure + connection.execute('RESET statement_timeout') + end + + def lease_timeout + TIMEOUT_PER_ACTION + end + + def log_index_info(message) + Gitlab::AppLogger.info(message: message, **logging_options) + end + + def skip_log_message + "Skipping #{name} validation since it does not exist. " \ + "The queuing entry will be deleted" + end + + def logging_options + { + fk_name: name, + table_name: table_name, + class: self.class.name.to_s + } + end + end + end + end +end diff --git a/lib/gitlab/database/async_foreign_keys/migration_helpers.rb b/lib/gitlab/database/async_foreign_keys/migration_helpers.rb new file mode 100644 index 00000000000..b8b9fc6d156 --- /dev/null +++ b/lib/gitlab/database/async_foreign_keys/migration_helpers.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncForeignKeys + module MigrationHelpers + # Prepares a foreign key for asynchronous validation. + # + # Stores the FK information in the postgres_async_foreign_key_validations + # table to be executed later. + # + def prepare_async_foreign_key_validation(table_name, column_name = nil, name: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_fk_validation_available? + + fk_name = name || concurrent_foreign_key_name(table_name, column_name) + + unless foreign_key_exists?(table_name, name: fk_name) + raise missing_schema_object_message(table_name, "foreign key", fk_name) + end + + async_validation = PostgresAsyncForeignKeyValidation + .find_or_create_by!(name: fk_name, table_name: table_name) + + Gitlab::AppLogger.info( + message: 'Prepared FK for async validation', + table_name: async_validation.table_name, + fk_name: async_validation.name) + + async_validation + end + + def unprepare_async_foreign_key_validation(table_name, column_name = nil, name: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_fk_validation_available? + + fk_name = name || concurrent_foreign_key_name(table_name, column_name) + + PostgresAsyncForeignKeyValidation.find_by(name: fk_name).try(&:destroy) + end + + def prepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_fk_validation_available? + + Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition| + prepare_async_foreign_key_validation(partition.identifier, column_name, name: name) + end + end + + def unprepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_fk_validation_available? + + Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition| + unprepare_async_foreign_key_validation(partition.identifier, column_name, name: name) + end + end + + private + + def async_fk_validation_available? + connection.table_exists?(:postgres_async_foreign_key_validations) + end + end + end + end +end diff --git a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb new file mode 100644 index 00000000000..de69a3d496f --- /dev/null +++ b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncForeignKeys + class PostgresAsyncForeignKeyValidation < SharedModel + include QueueErrorHandlingConcern + + self.table_name = 'postgres_async_foreign_key_validations' + + MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH + MAX_LAST_ERROR_LENGTH = 10_000 + + validates :name, presence: true, uniqueness: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + + scope :ordered, -> { order(attempts: :asc, id: :asc) } + end + end + end +end diff --git a/lib/gitlab/database/async_indexes.rb b/lib/gitlab/database/async_indexes.rb index 6f301a66803..581c7e7ff94 100644 --- a/lib/gitlab/database/async_indexes.rb +++ b/lib/gitlab/database/async_indexes.rb @@ -16,6 +16,15 @@ module Gitlab IndexDestructor.new(async_index).perform end end + + def self.execute_pending_actions!(how_many: DEFAULT_INDEXES_PER_INVOCATION) + queue_ids = PostgresAsyncIndex.ordered.limit(how_many).pluck(:id) + removal_actions = PostgresAsyncIndex.where(id: queue_ids).to_drop.ordered + creation_actions = PostgresAsyncIndex.where(id: queue_ids).to_create.ordered + + removal_actions.each { |async_index| IndexDestructor.new(async_index).perform } + creation_actions.each { |async_index| IndexCreator.new(async_index).perform } + end end end end diff --git a/lib/gitlab/database/async_indexes/index_base.rb b/lib/gitlab/database/async_indexes/index_base.rb new file mode 100644 index 00000000000..bde75e12295 --- /dev/null +++ b/lib/gitlab/database/async_indexes/index_base.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncIndexes + class IndexBase + include AsyncDdlExclusiveLeaseGuard + extend ::Gitlab::Utils::Override + + TIMEOUT_PER_ACTION = 1.day + + def initialize(async_index) + @async_index = async_index + end + + def perform + try_obtain_lease do + if preconditions_met? + log_index_info("Starting async index #{action_type}") + execute_action_with_error_handling + log_index_info("Finished async index #{action_type}") + else + log_index_info(skip_log_message) + async_index.destroy! + end + end + end + + private + + attr_reader :async_index + + delegate :connection, :connection_db_config, to: :async_index + + def preconditions_met? + raise NotImplementedError, 'must implement preconditions_met?' + end + + def action_type + raise NotImplementedError, 'must implement action_type' + end + + def execute_action_with_error_handling + around_execution { execute_action } + rescue StandardError => error + async_index.handle_exception!(error) + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::AppLogger.error(message: error.message, **logging_options) + end + + def around_execution + yield + end + + def execute_action + connection.execute(async_index.definition) + async_index.destroy! + end + + def index_exists? + connection.indexes(async_index.table_name).any? do |index| + index.name == async_index.name + end + end + + def lease_timeout + TIMEOUT_PER_ACTION + end + + def log_index_info(message) + Gitlab::AppLogger.info(message: message, **logging_options) + end + + def skip_log_message + "Skipping index #{action_type} since preconditions are not met. " \ + "The queuing entry will be deleted" + end + + def logging_options + { + table_name: async_index.table_name, + index_name: async_index.name, + class: self.class.name.to_s + } + end + end + end + end +end diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb index 3ae2bb7b3e5..c5f4c5f30ad 100644 --- a/lib/gitlab/database/async_indexes/index_creator.rb +++ b/lib/gitlab/database/async_indexes/index_creator.rb @@ -3,48 +3,24 @@ module Gitlab module Database module AsyncIndexes - class IndexCreator - include IndexingExclusiveLeaseGuard - - TIMEOUT_PER_ACTION = 1.day + class IndexCreator < AsyncIndexes::IndexBase STATEMENT_TIMEOUT = 20.hours - def initialize(async_index) - @async_index = async_index - end - - def perform - try_obtain_lease do - if index_exists? - log_index_info('Skipping index creation as the index exists') - else - log_index_info('Creating async index') - - set_statement_timeout do - connection.execute(async_index.definition) - end - - log_index_info('Finished creating async index') - end - - async_index.destroy - end - end - private - attr_reader :async_index - - def index_exists? - connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name } + override :preconditions_met? + def preconditions_met? + !index_exists? end - def connection - @connection ||= async_index.connection + override :action_type + def action_type + 'creation' end - def lease_timeout - TIMEOUT_PER_ACTION + override :around_execution + def around_execution(&block) + set_statement_timeout(&block) end def set_statement_timeout @@ -53,10 +29,6 @@ module Gitlab ensure connection.execute('RESET statement_timeout') end - - def log_index_info(message) - Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name) - end end end end diff --git a/lib/gitlab/database/async_indexes/index_destructor.rb b/lib/gitlab/database/async_indexes/index_destructor.rb index 66955df9d04..5596e099cb6 100644 --- a/lib/gitlab/database/async_indexes/index_destructor.rb +++ b/lib/gitlab/database/async_indexes/index_destructor.rb @@ -3,58 +3,29 @@ module Gitlab module Database module AsyncIndexes - class IndexDestructor - include IndexingExclusiveLeaseGuard - - TIMEOUT_PER_ACTION = 1.day - - def initialize(async_index) - @async_index = async_index - end - - def perform - try_obtain_lease do - if !index_exists? - log_index_info('Skipping dropping as the index does not exist') - else - log_index_info('Dropping async index') - - retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - connection: connection, - timing_configuration: Gitlab::Database::Reindexing::REMOVE_INDEX_RETRY_CONFIG, - klass: self.class, - logger: Gitlab::AppLogger - ) - - retries.run(raise_on_exhaustion: false) do - connection.execute(async_index.definition) - end - - log_index_info('Finished dropping async index') - end - - async_index.destroy - end - end - + class IndexDestructor < AsyncIndexes::IndexBase private - attr_reader :async_index - - def index_exists? - connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name } + override :preconditions_met? + def preconditions_met? + index_exists? end - def connection - @connection ||= async_index.connection + override :action_type + def action_type + 'removal' end - def lease_timeout - TIMEOUT_PER_ACTION - end + override :around_execution + def around_execution(&block) + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + connection: connection, + timing_configuration: Gitlab::Database::Reindexing::REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: Gitlab::AppLogger + ) - def log_index_info(message) - Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name) + retries.run(raise_on_exhaustion: false, &block) end end end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index c8f6761534c..f459c43e0ee 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -22,7 +22,7 @@ module Gitlab return unless async_index_creation_available? PostgresAsyncIndex.find_by(name: index_name).try do |async_index| - async_index.destroy + async_index.destroy! end end diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb index dc932482d40..9f5f39613ed 100644 --- a/lib/gitlab/database/async_indexes/postgres_async_index.rb +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -4,6 +4,8 @@ module Gitlab module Database module AsyncIndexes class PostgresAsyncIndex < SharedModel + include QueueErrorHandlingConcern + self.table_name = 'postgres_async_indexes' MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH @@ -15,6 +17,7 @@ module Gitlab scope :to_create, -> { where("definition ILIKE 'CREATE%'") } scope :to_drop, -> { where("definition ILIKE 'DROP%'") } + scope :ordered, -> { order(attempts: :asc, id: :asc) } def to_s definition diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index 4b4a9b38fd8..36dbb157b0d 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -43,15 +43,7 @@ module Gitlab end def update! - if without_prepared_statement? - # A workaround for https://github.com/rails/rails/issues/24893 - # When prepared statements are prevented (such as when using the - # query counter or in omnibus by default), we cannot call - # `exec_update`, since that will discard the bindings. - connection.send(:exec_no_cache, sql, log_name, params) # rubocop: disable GitlabSecurity/PublicSend - else - connection.exec_update(sql, log_name, params) - end + connection.exec_update(sql, log_name, params) end def self.column_definitions(model, columns) @@ -93,14 +85,6 @@ module Gitlab end end - # A workaround for https://github.com/rails/rails/issues/24893 - # We need to detect if prepared statements have been disabled. - def without_prepared_statement? - strong_memoize(:without_prepared_statement) do - connection.send(:without_prepared_statement?, [1]) # rubocop: disable GitlabSecurity/PublicSend - end - end - def query_attribute(column, key, values) value = values[column.name] key[column.name] = value if key.try(:id) # optimistic update diff --git a/lib/gitlab/database/connection_timer.rb b/lib/gitlab/database/connection_timer.rb index f9b893ffd0f..4eb214e74f4 100644 --- a/lib/gitlab/database/connection_timer.rb +++ b/lib/gitlab/database/connection_timer.rb @@ -27,7 +27,7 @@ module Gitlab end def current_clock_value - Concurrent.monotonic_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) end end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index f0343f9d8b5..622e310ead3 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -97,11 +97,11 @@ module Gitlab if current_session.use_primary? && !current_session.use_replicas_for_read_queries? @load_balancer.read_write do |connection| - connection.send(...) + connection.public_send(...) end else @load_balancer.read do |connection| - connection.send(...) + connection.public_send(...) end end end @@ -117,7 +117,7 @@ module Gitlab end @load_balancer.read_write do |connection| - connection.send(...) + connection.public_send(...) end end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index cb3a378ad64..23476e1f5e9 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -105,11 +105,9 @@ module Gitlab def read_write connection = nil transaction_open = nil - attempts = 3 - if prevent_load_balancer_retries_in_transaction? - attempts = 1 if pool.connection.transaction_open? - end + # Retry only once when in a transaction (see https://gitlab.com/gitlab-org/gitlab/-/issues/220242) + attempts = pool.connection.transaction_open? ? 1 : 3 # In the event of a failover the primary may be briefly unavailable. # Instead of immediately grinding to a halt we'll retry the operation @@ -348,10 +346,6 @@ module Gitlab row = ar_connection.select_all(sql).first row['location'] if row end - - def prevent_load_balancer_retries_in_transaction? - Gitlab::Utils.to_boolean(ENV['PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION'], default: false) - end end end end diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index 8e5dc98e96e..f5cb83e398a 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -121,19 +121,19 @@ module Gitlab end def unstick(namespace, id) - Gitlab::Redis::SharedState.with do |redis| + with_redis do |redis| redis.del(redis_key_for(namespace, id)) end end def set_write_location_for(namespace, id, location) - Gitlab::Redis::SharedState.with do |redis| + with_redis do |redis| redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION) end end def last_write_location_for(namespace, id) - Gitlab::Redis::SharedState.with do |redis| + with_redis do |redis| redis.get(redis_key_for(namespace, id)) end end @@ -143,6 +143,12 @@ module Gitlab "database-load-balancing/write-location/#{name}/#{namespace}/#{id}" end + + private + + def with_redis(&block) + Gitlab::Redis::DbLoadBalancing.with(&block) + end end end end diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 2e08e1ffb42..83884e89d6e 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -10,18 +10,6 @@ module Gitlab # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us EXPECTED_TRIGGER_RECORD_COUNT = 3 - def self.tables_to_lock(connection) - Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| - yield table_name, schema_name - end - - Gitlab::Database::SharedModel.using_connection(connection) do - Postgresql::DetachedPartition.find_each do |detached_partition| - yield detached_partition.fully_qualified_table_name, detached_partition.table_schema - end - end - end - def initialize(table_name:, connection:, database_name:, with_retries: true, logger: nil, dry_run: false) @table_name = table_name @connection = connection diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e41107370ec..9c1cb8e352c 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -14,6 +14,7 @@ module Gitlab include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers + include AsyncForeignKeys::MigrationHelpers def define_batchable_model(table_name, connection: self.connection) super(table_name, connection: connection) diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 25e75a10bb3..60df3370046 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -200,11 +200,14 @@ module Gitlab end end + # rubocop: disable Style/ArgumentsForwarding + # Reason: the default argument will not apply if we just forward via `...` def migrate_in(*args, coordinator: coordinator_for_tracking_database) with_migration_context do coordinator.perform_in(*args) end end + # rubocop: enable Style/ArgumentsForwarding def delete_queued_jobs(class_name) coordinator_for_tracking_database.steal(class_name) do |job| diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb index 8975c04e33a..840add8783d 100644 --- a/lib/gitlab/database/migrations/base_background_runner.rb +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -38,13 +38,15 @@ module Gitlab def run_jobs_for_migration(migration_name:, jobs:, run_until:) per_background_migration_result_dir = File.join(@result_dir, migration_name) - instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) + instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir, + observer_classes: observers) + batch_names = (1..).each.lazy.map { |i| "batch_#{i}" } jobs.each do |j| break if run_until <= Time.current - meta = migration_meta(j) + meta = { job_meta: job_meta(j) } instrumentation.observe(version: nil, name: batch_names.next, @@ -55,9 +57,13 @@ module Gitlab end end - def migration_meta(_job) + def job_meta(_job) {} end + + def observers + ::Gitlab::Database::Migrations::Observers.all_observers + end end end end diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index 80388c4dbbb..cd048beac96 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -4,16 +4,12 @@ module Gitlab module Database module Migrations - Observation = Struct.new( - :version, - :name, - :walltime, - :success, - :total_database_size_change, - :meta, - :query_statistics, - keyword_init: true - ) + Observation = Struct.new(:version, :name, :walltime, :success, :total_database_size_change, + :meta, :query_statistics, keyword_init: true) do + def to_json(...) + as_json.except('meta').to_json(...) + end + end end end end diff --git a/lib/gitlab/database/migrations/observers/batch_details.rb b/lib/gitlab/database/migrations/observers/batch_details.rb new file mode 100644 index 00000000000..0f8cdcf3cd6 --- /dev/null +++ b/lib/gitlab/database/migrations/observers/batch_details.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class BatchDetails < MigrationObserver + FILE_NAME = 'batch-details.json' + + def before + @started_at = get_time + end + + def after + @finished_at = get_time + end + + def record + File.open(path, 'wb') { |file| file.write(file_contents.to_json) } + end + + private + + attr_reader :started_at, :finished_at + + def file_contents + { time_spent: time_spent }.merge(job_meta) + end + + def get_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def time_spent + @time_spent ||= (finished_at - started_at).round(2) + end + + def path + File.join(output_dir, FILE_NAME) + end + + def job_meta + meta = observation.meta + + return {} unless meta + + meta[:job_meta].to_h + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index c123d01f327..01fdba22c19 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -6,6 +6,8 @@ module Gitlab class TestBatchedBackgroundRunner < BaseBackgroundRunner include Gitlab::Database::DynamicModelHelpers + MIGRATION_DETAILS_FILE_NAME = 'details.json' + def initialize(result_dir:, connection:, from_id:) super(result_dir: result_dir, connection: connection) @connection = connection @@ -51,6 +53,7 @@ module Gitlab migration.column_name, batch_min_value: batch_start, batch_size: migration.batch_size, + job_class: migration.job_class, job_arguments: migration.job_arguments ) @@ -64,7 +67,11 @@ module Gitlab end end - [migration.job_class_name, jobs_to_sample] + job_class_name = migration.job_class_name + + export_migration_details(job_class_name, migration.slice(:interval, :total_tuple_count, :max_batch_size)) + + [job_class_name, jobs_to_sample] end end end @@ -112,11 +119,25 @@ module Gitlab Gitlab::Database::SharedModel.using_connection(connection, &block) end - def migration_meta(job) + def job_meta(job) set_shared_model_connection do - job.batched_migration.slice(:max_batch_size, :total_tuple_count, :interval) + job.slice(:min_value, :max_value, :batch_size, :sub_batch_size, :pause_ms) end end + + def export_migration_details(migration_name, attributes) + directory = result_dir.join(migration_name) + + FileUtils.mkdir_p(directory) unless Dir.exist?(directory) + + File.write(directory.join(MIGRATION_DETAILS_FILE_NAME), attributes.to_json) + end + + def observers + ::Gitlab::Database::Migrations::Observers.all_observers + [ + ::Gitlab::Database::Migrations::Observers::BatchDetails + ] + end end end end diff --git a/lib/gitlab/database/postgres_constraint.rb b/lib/gitlab/database/postgres_constraint.rb index fa590914332..fa3870cb9c7 100644 --- a/lib/gitlab/database/postgres_constraint.rb +++ b/lib/gitlab/database/postgres_constraint.rb @@ -4,7 +4,6 @@ module Gitlab module Database # Backed by the postgres_constraints view class PostgresConstraint < SharedModel - IDENTIFIER_REGEX = /^\w+\.\w+$/.freeze self.primary_key = :oid scope :check_constraints, -> { where(constraint_type: 'c') } @@ -18,7 +17,7 @@ module Gitlab scope :valid, -> { where(constraint_valid: true) } scope :by_table_identifier, ->(identifier) do - unless identifier =~ IDENTIFIER_REGEX + unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" end diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index d3ede45fe86..04ef574a451 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -5,17 +5,23 @@ module Gitlab class PostgresForeignKey < SharedModel self.primary_key = :oid - # These values come from the possible confdeltype values in pg_constraint - enum on_delete_action: { + # These values come from the possible confdeltype / confupdtype values in pg_constraint + ACTION_TYPES = { restrict: 'r', cascade: 'c', nullify: 'n', set_default: 'd', no_action: 'a' - } + }.freeze + + enum on_delete_action: ACTION_TYPES, _prefix: :on_delete + + enum on_update_action: ACTION_TYPES, _prefix: :on_update scope :by_referenced_table_identifier, ->(identifier) do - raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" + end where(referenced_table_identifier: identifier) end @@ -23,7 +29,9 @@ module Gitlab scope :by_referenced_table_name, ->(name) { where(referenced_table_name: name) } scope :by_constrained_table_identifier, ->(identifier) do - raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" + end where(constrained_table_identifier: identifier) end @@ -43,6 +51,12 @@ module Gitlab where(on_delete_action: on_delete) end + + scope :by_on_update_action, ->(on_update) do + raise ArgumentError, "Invalid on_update action #{on_update}" unless on_update_actions.key?(on_update) + + where(on_update_action: on_update) + end end end end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 4a9d8728c83..50009cadf5d 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -14,7 +14,9 @@ module Gitlab has_many :queued_reindexing_actions, class_name: 'Gitlab::Database::Reindexing::QueuedAction', foreign_key: :index_identifier scope :by_identifier, ->(identifier) do - raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" + end find(identifier) end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index e4f70ee1745..36dc6818157 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -8,7 +8,9 @@ module Gitlab belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' scope :for_identifier, ->(identifier) do - raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" + end where(primary_key => identifier) end diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb index 3bd342f940f..fead7379e43 100644 --- a/lib/gitlab/database/postgres_partitioned_table.rb +++ b/lib/gitlab/database/postgres_partitioned_table.rb @@ -10,7 +10,9 @@ module Gitlab has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier' scope :by_identifier, ->(identifier) do - raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" + end find(identifier) end @@ -19,6 +21,13 @@ module Gitlab find_by("identifier = concat(current_schema(), '.', ?)", name) end + def self.each_partition(table_name, &block) + find_by_name_in_current_schema(table_name) + .postgres_partitions + .order(:name) + .each(&block) + end + def dynamic? DYNAMIC_PARTITION_STRATEGIES.include?(strategy) end diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index c51282c9a55..4ae3622479f 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -9,7 +9,18 @@ module Gitlab DMLNotAllowedError = Class.new(UnsupportedSchemaError) DMLAccessDeniedError = Class.new(UnsupportedSchemaError) - IGNORED_SCHEMAS = %i[gitlab_shared gitlab_internal].freeze + # Re-map schemas observed schemas to a single cluster mode + # - symbol: + # The mapped schema indicates that it contains all data in a single-cluster mode + # - nil: + # Inidicates that changes made to this schema are ignored and always allowed + SCHEMA_MAPPING = { + gitlab_shared: nil, + gitlab_internal: nil, + + # Pods specific changes + gitlab_main_clusterwide: :gitlab_main + }.freeze class << self def enabled? @@ -90,7 +101,13 @@ module Gitlab def dml_schemas(tables) extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) - extra_schemas.subtract(IGNORED_SCHEMAS) + + SCHEMA_MAPPING.each do |schema, mapped_schema| + next unless extra_schemas.delete?(schema) + + extra_schemas.add(mapped_schema) if mapped_schema + end + extra_schemas end diff --git a/lib/gitlab/database/queue_error_handling_concern.rb b/lib/gitlab/database/queue_error_handling_concern.rb new file mode 100644 index 00000000000..7d35426d029 --- /dev/null +++ b/lib/gitlab/database/queue_error_handling_concern.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueueErrorHandlingConcern + extend ActiveSupport::Concern + + MAX_LAST_ERROR_LENGTH = 10_000 + + included do + validates :last_error, length: { maximum: MAX_LAST_ERROR_LENGTH }, + if: ->(record) { record.respond_to?(:last_error) } + end + + def handle_exception!(error) + transaction do + increment!(:attempts) + update!(last_error: format_last_error(error)) + end + end + + private + + def format_last_error(error) + [error.message] + .concat(error.backtrace) + .join("\n") + .truncate(MAX_LAST_ERROR_LENGTH) + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index aba45fcc57b..78de7161a0f 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -28,6 +28,7 @@ module Gitlab # Hack: Before we do actual reindexing work, create async indexes Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) Gitlab::Database::AsyncIndexes.drop_pending_indexes! + Gitlab::Database::AsyncForeignKeys.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops) automatic_reindexing end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index eca118a4ff2..57e2e0c1beb 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -4,7 +4,7 @@ module Gitlab module Database module Reindexing class Coordinator - include IndexingExclusiveLeaseGuard + include AsyncDdlExclusiveLeaseGuard # Maximum lease time for the global Redis lease # This should be higher than the maximum time for any @@ -54,7 +54,7 @@ module Gitlab private - delegate :connection, to: :index + delegate :connection, :connection_db_config, to: :index def with_notifications(action) notifier.notify_start(action) diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb new file mode 100644 index 00000000000..dfc845f0b44 --- /dev/null +++ b/lib/gitlab/database/schema_validation/database.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Database + def initialize(connection) + @connection = connection + end + + def fetch_index_by_name(index_name) + index_map[index_name] + end + + def indexes + index_map.values + end + + private + + def index_map + @index_map ||= + fetch_indexes.transform_values! do |index_stmt| + Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) + end + end + + attr_reader :connection + + def fetch_indexes + sql = <<~SQL + SELECT indexname, indexdef + FROM pg_indexes + WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ('public', 'gitlab_partitions_static'); + SQL + + @fetch_indexes ||= connection.exec_query(sql).rows.to_h + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/index.rb b/lib/gitlab/database/schema_validation/index.rb new file mode 100644 index 00000000000..af0d5f31f4e --- /dev/null +++ b/lib/gitlab/database/schema_validation/index.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Index + def initialize(parsed_stmt) + @parsed_stmt = parsed_stmt + end + + def name + parsed_stmt.idxname + end + + def statement + @statement ||= PgQuery.deparse_stmt(parsed_stmt) + end + + private + + attr_reader :parsed_stmt + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/indexes.rb b/lib/gitlab/database/schema_validation/indexes.rb new file mode 100644 index 00000000000..b7c3705bde9 --- /dev/null +++ b/lib/gitlab/database/schema_validation/indexes.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Indexes + def initialize(structure_sql, database) + @structure_sql = structure_sql + @database = database + end + + def missing_indexes + structure_sql.indexes.map(&:name) - database.indexes.map(&:name) + end + + def extra_indexes + database.indexes.map(&:name) - structure_sql.indexes.map(&:name) + end + + def wrong_indexes + structure_sql.indexes.filter_map do |structure_sql_index| + database_index = database.fetch_index_by_name(structure_sql_index.name) + + next if database_index.nil? + next if database_index.statement == structure_sql_index.statement + + structure_sql_index.name + end + end + + private + + attr_reader :structure_sql, :database + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb new file mode 100644 index 00000000000..32c69a0e5e7 --- /dev/null +++ b/lib/gitlab/database/schema_validation/structure_sql.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class StructureSql + def initialize(structure_file_path) + @structure_file_path = structure_file_path + end + + def indexes + @indexes ||= index_statements.map do |index_statement| + index_statement.relation.schemaname = "public" if index_statement.relation.schemaname == '' + + Index.new(index_statement) + end + end + + private + + attr_reader :structure_file_path + + def index_statements + parsed_structure_file.tree.stmts.filter_map { |s| s.stmt.index_stmt } + end + + def parsed_structure_file + PgQuery.parse(File.read(structure_file_path)) + end + end + end + end +end diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index 877866b9b23..41c3a27bc5b 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -44,6 +44,11 @@ module Gitlab end end + # in case the connection has been switched with using_connection + def connection_pool + connection.pool + end + private def overriding_connection diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb new file mode 100644 index 00000000000..c417ce716e8 --- /dev/null +++ b/lib/gitlab/database/tables_locker.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class TablesLocker + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze + + def initialize(logger: nil, dry_run: false) + @logger = logger + @dry_run = dry_run + end + + def unlock_writes + Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| + tables_to_lock(connection) do |table_name, schema_name| + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 + next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE + + lock_writes_manager(table_name, connection, database_name).unlock_writes + end + end + end + + def lock_writes + Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name| + schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) + + tables_to_lock(connection) do |table_name, schema_name| + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 + next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE + + if schemas_for_connection.include?(schema_name) + lock_writes_manager(table_name, connection, database_name).unlock_writes + else + lock_writes_manager(table_name, connection, database_name).lock_writes + end + end + end + end + + private + + def tables_to_lock(connection, &block) + Gitlab::Database::GitlabSchema.tables_to_schema.each(&block) + + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + yield detached_partition.fully_qualified_table_name, detached_partition.table_schema + end + end + end + + def lock_writes_manager(table_name, connection, database_name) + Gitlab::Database::LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: database_name, + with_retries: true, + logger: @logger, + dry_run: @dry_run + ) + end + end + end +end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index daef0402742..a6430d1758b 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -71,17 +71,25 @@ module Gitlab @connection ||= Gitlab::Database.database_base_models[database_name].connection end + def remove_schema_name(table_with_schema) + ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table_with_schema) + .identifier + end + + def disable_locks_on_table(table) + sql_statement = "SELECT set_config('lock_writes.#{table}', 'false', false)" + logger&.info(sql_statement) + connection.execute(sql_statement) unless dry_run + end + def truncate_tables_in_batches(tables_sorted) truncated_tables = [] tables_sorted.flatten.each do |table| - table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils - .extract_schema_qualified_name(table) - .identifier + table_name_without_schema = remove_schema_name(table) - sql_statement = "SELECT set_config('lock_writes.#{table_name_without_schema}', 'false', false)" - logger&.info(sql_statement) - connection.execute(sql_statement) unless dry_run + disable_locks_on_table(table_name_without_schema) # Temporarily unlocking writes on the attached partitions of the table. # Because in some cases they might have been locked for writes as well, when they used to be @@ -89,13 +97,7 @@ module Gitlab Gitlab::Database::SharedModel.using_connection(connection) do table_partitions = Gitlab::Database::PostgresPartition.for_parent_table(table_name_without_schema) table_partitions.each do |table_partition| - partition_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils - .extract_schema_qualified_name(table_partition.identifier) - .identifier - - sql_statement = "SELECT set_config('lock_writes.#{partition_name_without_schema}', 'false', false)" - logger&.info(sql_statement) - connection.execute(sql_statement) unless dry_run + disable_locks_on_table(remove_schema_name(table_partition.identifier)) end end end diff --git a/lib/gitlab/database/transaction_timeout_settings.rb b/lib/gitlab/database/transaction_timeout_settings.rb new file mode 100644 index 00000000000..9485b8d4cbc --- /dev/null +++ b/lib/gitlab/database/transaction_timeout_settings.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class TransactionTimeoutSettings + SETTING = 'idle_in_transaction_session_timeout' + + def initialize(connection) + @connection = connection + end + + def disable_timeouts + @connection.execute("SET #{SETTING} = 0") + end + + def restore_timeouts + @connection.execute("RESET #{SETTING}") + end + end + end +end diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 1e29ae7761b..9796a5905e3 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -4,6 +4,85 @@ module Gitlab module DatabaseImporters module WorkItems module BaseTypeImporter + WIDGET_NAMES = { + assignees: 'Assignees', + labels: 'Labels', + description: 'Description', + hierarchy: 'Hierarchy', + start_and_due_date: 'Start and due date', + milestone: 'Milestone', + notes: 'Notes', + iteration: 'Iteration', + weight: 'Weight', + health_status: 'Health status', + progress: 'Progress', + status: 'Status', + requirement_legacy: 'Requirement legacy', + test_reports: 'Test reports' + }.freeze + + WIDGETS_FOR_TYPE = { + issue: [ + :assignees, + :labels, + :description, + :hierarchy, + :start_and_due_date, + :milestone, + :notes, + :iteration, + :weight, + :health_status + ], + incident: [ + :description, + :hierarchy, + :notes + ], + test_case: [ + :description, + :notes + ], + requirement: [ + :description, + :notes, + :status, + :requirement_legacy, + :test_reports + ], + task: [ + :assignees, + :labels, + :description, + :hierarchy, + :start_and_due_date, + :milestone, + :notes, + :iteration, + :weight + ], + objective: [ + :assignees, + :labels, + :description, + :hierarchy, + :milestone, + :notes, + :health_status, + :progress + ], + key_result: [ + :assignees, + :labels, + :description, + :hierarchy, + :start_and_due_date, + :notes, + :health_status, + :progress + ] + }.freeze + def self.upsert_types current_time = Time.current @@ -16,6 +95,29 @@ module Gitlab base_types, unique_by: :idx_work_item_types_on_namespace_id_and_name_null_namespace ) + + upsert_widgets + end + + def self.upsert_widgets + type_ids_by_name = ::WorkItems::Type.default.pluck(:name, :id).to_h # rubocop: disable CodeReuse/ActiveRecord + + widgets = WIDGETS_FOR_TYPE.flat_map do |type_sym, widget_syms| + type_name = ::WorkItems::Type::TYPE_NAMES[type_sym] + + widget_syms.map do |widget_sym| + { + work_item_type_id: type_ids_by_name[type_name], + name: WIDGET_NAMES[widget_sym], + widget_type: ::WorkItems::WidgetDefinition.widget_types[widget_sym] + } + end + end + + ::WorkItems::WidgetDefinition.upsert_all( + widgets, + unique_by: :index_work_item_widget_definitions_on_default_witype_and_name + ) end end end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 6d114de8ae8..203cee1fd5e 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -13,8 +13,8 @@ module Gitlab Gitlab::FileDetector.type_of(blob_name) == file_type end - def self.link(*args) - new(*args).link + def self.link(...) + new(...).link end attr_accessor :plain_text, :highlighted_text diff --git a/lib/gitlab/deploy_key_access.rb b/lib/gitlab/deploy_key_access.rb index ca16582d2b4..a582c978be7 100644 --- a/lib/gitlab/deploy_key_access.rb +++ b/lib/gitlab/deploy_key_access.rb @@ -17,11 +17,14 @@ module Gitlab attr_reader :deploy_key def protected_tag_accessible_to?(ref, action:) - assert_project! - - # a deploy key can always push a protected tag - # (which is not always the case when pushing to a protected branch) - true + if Feature.enabled?(:deploy_key_for_protected_tags, project) + super + else + assert_project! + # a deploy key can always push a protected tag + # (which is not always the case when pushing to a protected branch) + true + end end def can_collaborate?(_ref) diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb index cd075569d10..03d3f062372 100644 --- a/lib/gitlab/doctor/secrets.rb +++ b/lib/gitlab/doctor/secrets.rb @@ -16,7 +16,7 @@ module Gitlab models_with_attributes = Hash.new { |h, k| h[k] = [] } models_with_encrypted_attributes.each do |model| - models_with_attributes[model] += model.encrypted_attributes.keys + models_with_attributes[model] += model.attr_encrypted_attributes.keys end models_with_encrypted_tokens.each do |model| @@ -79,7 +79,7 @@ module Gitlab end def models_with_encrypted_attributes - all_models.select { |d| d.encrypted_attributes.present? } + all_models.select { |d| d.attr_encrypted_attributes.present? } end def models_with_encrypted_tokens diff --git a/lib/gitlab/email/common.rb b/lib/gitlab/email/common.rb index afee8d9cd3d..01316995c4d 100644 --- a/lib/gitlab/email/common.rb +++ b/lib/gitlab/email/common.rb @@ -54,6 +54,10 @@ module Gitlab # It's looking for each <...> references.scan(/(?!<)[^<>]+(?=>)/) end + + def encrypted_secrets + Settings.encrypted(config.encrypted_secret_file) + end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index e21a88c4e0d..c325112b673 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -62,7 +62,7 @@ module Gitlab def create_issue ::Issues::CreateService.new( - project: project, + container: project, current_user: author, params: { title: mail.subject, diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 06365296a76..076ba42daac 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -92,7 +92,7 @@ module Gitlab def create_issue! result = ::Issues::CreateService.new( - project: project, + container: project, current_user: User.support_bot, params: { title: mail.subject, diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb index 27ba5d2a314..10dbedbb464 100644 --- a/lib/gitlab/email/html_parser.rb +++ b/lib/gitlab/email/html_parser.rb @@ -18,9 +18,6 @@ module Gitlab end def filter_replies! - document.xpath('//blockquote').each(&:remove) - document.xpath('//table').each(&:remove) - # bogus links with no href are sometimes added by outlook, # and can result in Html2Text adding extra square brackets # to the text, so we unwrap them here. @@ -37,7 +34,11 @@ module Gitlab end def filtered_text - @filtered_text ||= Html2Text.convert(filtered_html) + @filtered_text ||= if Feature.enabled?(:service_desk_html_to_text_email_handler) + ::Gitlab::Email::HtmlToMarkdownParser.convert(filtered_html) + else + Html2Text.convert(filtered_html) + end end end end diff --git a/lib/gitlab/email/html_to_markdown_parser.rb b/lib/gitlab/email/html_to_markdown_parser.rb new file mode 100644 index 00000000000..42dd012308b --- /dev/null +++ b/lib/gitlab/email/html_to_markdown_parser.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'nokogiri' + +module Gitlab + module Email + class HtmlToMarkdownParser < Html2Text + ADDITIONAL_TAGS = %w[em strong img details].freeze + IMG_ATTRS = %w[alt src].freeze + + def self.convert(html) + html = fix_newlines(replace_entities(html)) + doc = Nokogiri::HTML(html) + + HtmlToMarkdownParser.new(doc).convert + end + + def iterate_over(node) + return super unless ADDITIONAL_TAGS.include?(node.name) + + if node.name == 'img' + node.keys.each { |key| node.remove_attribute(key) unless IMG_ATTRS.include?(key) } # rubocop:disable Style/HashEachMethods + end + + Kramdown::Document.new(node.to_html, input: 'html').to_commonmark + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb index 6b01c83b8e7..68f9a9a21c9 100644 --- a/lib/gitlab/email/message/in_product_marketing/create.rb +++ b/lib/gitlab/email/message/in_product_marketing/create.rb @@ -68,7 +68,7 @@ module Gitlab private def project_link - link(s_('InProductMarketing|create a project'), help_page_url('gitlab-basics/create-project')) + link(s_('InProductMarketing|create a project'), help_page_url('user/project/index')) end def repo_link @@ -76,7 +76,7 @@ module Gitlab end def github_link - link(s_('InProductMarketing|GitHub Enterprise projects to GitLab'), help_page_url('integration/github')) + link(s_('InProductMarketing|GitHub Enterprise projects to GitLab'), help_page_url('user/project/import/github')) end def bitbucket_link @@ -84,11 +84,11 @@ module Gitlab end def mirroring_link - link(s_('InProductMarketing|repository mirroring'), help_page_url('user/project/repository/repository_mirroring')) + link(s_('InProductMarketing|repository mirroring'), help_page_url('user/project/repository/mirror/index')) end def basics_link - link(s_('InProductMarketing|Git basics'), help_page_url('gitlab-basics/index')) + link(s_('InProductMarketing|Git basics'), help_page_url('topics/git/index')) end def import_link diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index b1fd35184ac..99240f2ad48 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -16,6 +16,7 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 UNICODE_REPLACEMENT_CHARACTER = "�" + BOM_UTF8 = "\xEF\xBB\xBF" def encode!(message) message = force_encode_utf8(message) @@ -147,6 +148,10 @@ module Gitlab filename.force_encoding("UTF-8") end + def strip_bom(message) + message.delete_prefix(BOM_UTF8) + end + private def force_encode_utf8(message) diff --git a/lib/gitlab/encrypted_incoming_email_command.rb b/lib/gitlab/encrypted_incoming_email_command.rb new file mode 100644 index 00000000000..a18382439d6 --- /dev/null +++ b/lib/gitlab/encrypted_incoming_email_command.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Output +module Gitlab + class EncryptedIncomingEmailCommand < EncryptedCommandBase + DISPLAY_NAME = "INCOMING_EMAIL" + EDIT_COMMAND_NAME = "gitlab:incoming_email:secret:edit" + + class << self + def encrypted_secrets + Gitlab::IncomingEmail.encrypted_secrets + end + + def encrypted_file_template + <<~YAML + # password: '123' + # user: 'gitlab-incoming@gmail.com' + YAML + end + end + end +end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/encrypted_service_desk_email_command.rb b/lib/gitlab/encrypted_service_desk_email_command.rb new file mode 100644 index 00000000000..ece6da7c1b3 --- /dev/null +++ b/lib/gitlab/encrypted_service_desk_email_command.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Output +module Gitlab + class EncryptedServiceDeskEmailCommand < EncryptedCommandBase + DISPLAY_NAME = "SERVICE_DESK_EMAIL" + EDIT_COMMAND_NAME = "gitlab:service_desk_email:secret:edit" + + class << self + def encrypted_secrets + Gitlab::ServiceDeskEmail.encrypted_secrets + end + + def encrypted_file_template + <<~YAML + # password: '123' + # user: 'gitlab-incoming@gmail.com' + YAML + end + end + end +end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb index b1a9603d3a5..86094727df5 100644 --- a/lib/gitlab/environment.rb +++ b/lib/gitlab/environment.rb @@ -5,5 +5,18 @@ module Gitlab def self.hostname @hostname ||= ENV['HOSTNAME'] || Socket.gethostname end + + # Check whether codebase is going through static verification + # in order to skip executing parts of the codebase + # + # @return [Boolean] Is the code going through static verification? + def self.static_verification? + static_verification = Gitlab::Utils.to_boolean(ENV['STATIC_VERIFICATION'], default: false) + env_production = ENV['RAILS_ENV'] == 'production' + + warn '[WARNING] Static Verification bypass is enabled in Production.' if static_verification && env_production + + static_verification + end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 876a1cbb183..2b00fe48951 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -57,6 +57,7 @@ module Gitlab config.before_send = method(:before_send_sentry) config.background_worker_threads = 0 config.send_default_pii = true + config.traces_sample_rate = 0.2 if Gitlab::Utils.to_boolean(ENV['ENABLE_SENTRY_PERFORMANCE_MONITORING']) yield config if block_given? end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index f6431483a15..7aabf699a59 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -65,13 +65,15 @@ module Gitlab status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429 - add_instrument_for_cache_hit(status_code, route, request) - Gitlab::ApplicationContext.push( feature_category: route.feature_category, - caller_id: route.caller_id + caller_id: route.caller_id, + remote_ip: request.remote_ip ) + request.env[Gitlab::Metrics::RequestsRackMiddleware::REQUEST_URGENCY_KEY] = route.urgency + add_instrument_for_cache_hit(status_code, route, request) + new_headers = { 'ETag' => etag, 'X-Gitlab-From-Cache' => 'true' @@ -102,7 +104,10 @@ module Gitlab format: request.format.ref, method: request.request_method, path: request.filtered_path, - status: status + status: status, + metadata: Gitlab::ApplicationContext.current, + request_urgency: route.urgency.name, + target_duration_s: route.urgency.duration } ActiveSupport::Notifications.instrument( diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 684afc6762a..754b54fda81 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -3,24 +3,34 @@ module Gitlab module EtagCaching module Router - Route = Struct.new(:router, :regexp, :name, :feature_category, :caller_id) do + Route = Struct.new(:router, :regexp, :name, :feature_category, :caller_id, :urgency, keyword_init: true) do delegate :match, to: :regexp delegate :cache_key, to: :router end module Helpers - def build_route(attrs) - EtagCaching::Router::Route.new(self, *attrs) + def build_graphql_route(regexp, name, feature_category) + EtagCaching::Router::Route.new( + router: self, + regexp: regexp, + name: name, + feature_category: feature_category, + # This information can be loaded from the graphql query, but is not + # included yet + # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/665 + caller_id: nil, + urgency: Gitlab::EndpointAttributes::DEFAULT_URGENCY + ) end - def build_rails_route(attrs) - regexp, name, controller, action_name = *attrs + def build_rails_route(regexp, name, controller, action_name) EtagCaching::Router::Route.new( - self, - regexp, - name, - controller.feature_category_for_action(action_name).to_s, - controller.endpoint_id_for_action(action_name).to_s + router: self, + regexp: regexp, + name: name, + feature_category: controller.feature_category_for_action(action_name).to_s, + caller_id: controller.endpoint_id_for_action(action_name).to_s, + urgency: controller.urgency_for_action(action_name) ) end end diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index 1f56670ee7f..7a0fb2ac269 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -23,7 +23,7 @@ module Gitlab 'on_demand_scans', 'dynamic_application_security_testing' ] - ].map(&method(:build_route)).freeze + ].map { |attrs| build_graphql_route(*attrs) }.freeze def self.match(request) return unless request.path_info == graphql_api_path diff --git a/lib/gitlab/etag_caching/router/rails.rb b/lib/gitlab/etag_caching/router/rails.rb index d80c003fe53..2924370f494 100644 --- a/lib/gitlab/etag_caching/router/rails.rb +++ b/lib/gitlab/etag_caching/router/rails.rb @@ -104,7 +104,7 @@ module Gitlab ::Projects::MergeRequests::ContentController, :cached_widget ] - ].map(&method(:build_rails_route)).freeze + ].map { |attrs| build_rails_route(*attrs) }.freeze # Overridden in EE to add more routes def self.all_routes diff --git a/lib/gitlab/external_authorization/config.rb b/lib/gitlab/external_authorization/config.rb index 8654a8c1e2e..e77318144ef 100644 --- a/lib/gitlab/external_authorization/config.rb +++ b/lib/gitlab/external_authorization/config.rb @@ -37,6 +37,12 @@ module Gitlab client_cert.present? && client_key.present? end + def allow_deploy_tokens_and_deploy_keys? + return true unless enabled? + + service_url.blank? && application_settings.allow_deploy_tokens_and_keys_with_external_authn? + end + private def application_settings diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 6225955a930..95f896a74e9 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -44,7 +44,15 @@ module Gitlab # Overridden in Gitlab::WikiFileFinder def search_paths(query) - repository.search_files_by_name(query, ref) + if Feature.enabled?(:code_basic_search_files_by_regexp, project) + return [] if query.blank? || ref.blank? + + escaped_query = RE2::Regexp.escape(query) + query_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)#{escaped_query}") + repository.search_files_by_regexp(query_regexp, ref) + else + repository.search_files_by_name(query, ref) + end end end end diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb index 9a41f04a4db..c0121311605 100644 --- a/lib/gitlab/git/raw_diff_change.rb +++ b/lib/gitlab/git/raw_diff_change.rb @@ -38,7 +38,7 @@ module Gitlab def extract_paths(file_path) case operation when :copied, :renamed - file_path.split(/\t/) + file_path.split("\t") when :deleted [file_path, nil] when :added diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 2f9cfe3e764..e054b6df98f 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -465,9 +465,9 @@ module Gitlab end # Returns the SHA of the most recent common ancestor of +from+ and +to+ - def merge_base(*commits) + def merge_base(...) wrapped_gitaly_errors do - gitaly_repository_client.find_merge_base(*commits) + gitaly_repository_client.find_merge_base(...) end end @@ -652,15 +652,23 @@ module Gitlab end end - def merge(user, source_sha, target_branch, message, &block) + def merge(user, source_sha:, target_branch:, message:, target_sha: nil, &block) wrapped_gitaly_errors do - gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) + gitaly_operation_client.user_merge_branch(user, + source_sha: source_sha, + target_branch: target_branch, + message: message, + target_sha: target_sha, + &block) end end - def ff_merge(user, source_sha, target_branch) + def ff_merge(user, source_sha:, target_branch:, target_sha: nil) wrapped_gitaly_errors do - gitaly_operation_client.user_ff_branch(user, source_sha, target_branch) + gitaly_operation_client.user_ff_branch(user, + source_sha: source_sha, + target_branch: target_branch, + target_sha: target_sha) end end @@ -720,9 +728,9 @@ module Gitlab raise DeleteBranchError, e end - def delete_refs(*ref_names) + def delete_refs(...) wrapped_gitaly_errors do - gitaly_delete_refs(*ref_names) + gitaly_delete_refs(...) end end @@ -847,9 +855,10 @@ module Gitlab end end - def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX]) + # peel_tags slows down the request by a factor of 3-4 + def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX], pointing_at_oids: [], peel_tags: false) wrapped_gitaly_errors do - gitaly_ref_client.list_refs(patterns) + gitaly_ref_client.list_refs(patterns, pointing_at_oids: pointing_at_oids, peel_tags: peel_tags) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 35b330fa089..9d19695363a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -367,7 +367,7 @@ module Gitlab end def deploy_key? - actor.is_a?(DeployKey) && !Gitlab::ExternalAuthorization.enabled? + actor.is_a?(DeployKey) && Gitlab::ExternalAuthorization.allow_deploy_tokens_and_deploy_keys? end def deploy_token @@ -375,7 +375,7 @@ module Gitlab end def deploy_token? - actor.is_a?(DeployToken) && !Gitlab::ExternalAuthorization.enabled? + actor.is_a?(DeployToken) && Gitlab::ExternalAuthorization.allow_deploy_tokens_and_deploy_keys? end def ci? diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 199257f767d..77d2ba315a8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -128,8 +128,8 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless %w(tcp unix tls).include?(URI(address).scheme) - raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" + unless %w(tcp unix tls dns).include?(URI(address).scheme) + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls' or 'dns'" end address diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index e5f8a255f7d..4df9d800ea6 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -6,6 +6,12 @@ module Gitlab include Gitlab::EncodingHelper include WithFeatureFlagActors + WHITESPACE_CHANGES = { + 'ignore_all_spaces' => Gitaly::CommitDiffRequest::WhitespaceChanges::WHITESPACE_CHANGES_IGNORE_ALL, + 'ignore_spaces' => Gitaly::CommitDiffRequest::WhitespaceChanges::WHITESPACE_CHANGES_IGNORE, + 'unspecified' => Gitaly::CommitDiffRequest::WhitespaceChanges::WHITESPACE_CHANGES_UNSPECIFIED + }.freeze + TREE_ENTRIES_DEFAULT_LIMIT = 100_000 def initialize(repository) @@ -538,6 +544,11 @@ module Gitlab def call_commit_diff(request_params, options = {}) request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + + if Feature.enabled?(:add_ignore_all_white_spaces) && (request_params[:ignore_whitespace_change]) + request_params[:whitespace_changes] = WHITESPACE_CHANGES['ignore_all_spaces'] + end + request_params[:enforce_limits] = options.fetch(:limits, true) request_params[:collapse_diffs] = !options.fetch(:expanded, true) request_params.merge!(Gitlab::Git::DiffCollection.limits(options)) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 66f70ed9dc6..313334737c0 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -40,11 +40,6 @@ module Gitlab ) response = gitaly_client_call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) - if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::PreReceiveError, pre_receive_error - elsif response.exists - raise Gitlab::Git::Repository::TagExistsError - end Gitlab::Git::Tag.new(@repository, response.tag) rescue GRPC::BadStatus => e @@ -79,10 +74,6 @@ module Gitlab response = gitaly_client_call(@repository.storage, :operation_service, :user_create_branch, request, timeout: GitalyClient.long_timeout) - if response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError, response.pre_receive_error - end - branch = response.branch return unless branch @@ -128,12 +119,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = gitaly_client_call(@repository.storage, :operation_service, - :user_delete_branch, request, timeout: GitalyClient.long_timeout) - - if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::PreReceiveError, pre_receive_error - end + gitaly_client_call(@repository.storage, :operation_service, + :user_delete_branch, request, timeout: GitalyClient.long_timeout) rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) @@ -165,7 +152,7 @@ module Gitlab response.commit_id end - def user_merge_branch(user, source_sha, target_branch, message) + def user_merge_branch(user, source_sha:, target_branch:, message:, target_sha: nil) request_enum = QueueEnumerator.new response_enum = gitaly_client_call( @repository.storage, @@ -181,6 +168,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, branch: encode_binary(target_branch), + expected_old_oid: target_sha, message: encode_binary(message), timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) @@ -197,7 +185,6 @@ module Gitlab raise Gitlab::Git::CommitError, 'failed to apply merge to branch' unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) - rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) @@ -220,12 +207,13 @@ module Gitlab request_enum.close end - def user_ff_branch(user, source_sha, target_branch) + def user_ff_branch(user, source_sha:, target_branch:, target_sha: nil) request = Gitaly::UserFFBranchRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: encode_binary(target_branch) + branch: encode_binary(target_branch), + expected_old_oid: target_sha ) response = gitaly_client_call( @@ -246,25 +234,54 @@ module Gitlab end def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false) - call_cherry_pick_or_revert(:cherry_pick, - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository, - dry_run: dry_run) + response = call_cherry_pick_or_revert(:cherry_pick, + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository, + dry_run: dry_run) + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :cherry_pick_conflict + raise Gitlab::Git::Repository::CreateTreeError, 'CONFLICT' + when :changes_already_applied + raise Gitlab::Git::Repository::CreateTreeError, 'EMPTY' + when :target_branch_diverged + raise Gitlab::Git::CommitError, 'branch diverged' + else + raise e + end end def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false) - call_cherry_pick_or_revert(:revert, - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository, - dry_run: dry_run) + response = call_cherry_pick_or_revert(:revert, + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository, + dry_run: dry_run) + + if response.pre_receive_error.presence + raise Gitlab::Git::PreReceiveError, response.pre_receive_error + elsif response.commit_error.presence + raise Gitlab::Git::CommitError, response.commit_error + elsif response.create_tree_error.presence + raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error_code + end + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: []) @@ -520,7 +537,7 @@ module Gitlab dry_run: dry_run ) - response = gitaly_client_call( + gitaly_client_call( @repository.storage, :operation_service, :"user_#{rpc}", @@ -528,37 +545,6 @@ module Gitlab remote_storage: start_repository.storage, timeout: GitalyClient.long_timeout ) - - handle_cherry_pick_or_revert_response(response) - rescue GRPC::BadStatus => e - detailed_error = GitalyClient.decode_detailed_error(e) - - case detailed_error&.error - when :access_check - access_check_error = detailed_error.access_check - # These messages were returned from internal/allowed API calls - raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) - when :cherry_pick_conflict - raise Gitlab::Git::Repository::CreateTreeError, 'CONFLICT' - when :changes_already_applied - raise Gitlab::Git::Repository::CreateTreeError, 'EMPTY' - when :target_branch_diverged - raise Gitlab::Git::CommitError, 'branch diverged' - else - raise e - end - end - - def handle_cherry_pick_or_revert_response(response) - if response.pre_receive_error.presence - raise Gitlab::Git::PreReceiveError, response.pre_receive_error - elsif response.commit_error.presence - raise Gitlab::Git::CommitError, response.commit_error - elsif response.create_tree_error.presence - raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error_code - end - - Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end # rubocop:disable Metrics/ParameterLists diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 74034c4e717..ac6491e8770 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -205,10 +205,13 @@ module Gitlab raise ArgumentError, ex end - def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX]) + # peel_tags slows down the request by a factor of 3-4 + def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX], pointing_at_oids: [], peel_tags: false) request = Gitaly::ListRefsRequest.new( repository: @gitaly_repo, - patterns: patterns + patterns: patterns, + pointing_at_oids: pointing_at_oids, + peel_tags: peel_tags ) response = gitaly_client_call(@storage, :ref_service, :list_refs, request, timeout: GitalyClient.fast_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 203854264ce..bcc03ca08c9 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -24,8 +24,20 @@ module Gitlab response.exists end - def optimize_repository - request = Gitaly::OptimizeRepositoryRequest.new(repository: @gitaly_repo) + # Optimize the repository. By default, this will perform heuristical housekeeping in the repository, which + # is the recommended approach and will only optimize what needs to be optimized. If `eager = true`, then + # Gitaly will instead be asked to perform eager housekeeping. As a consequence the housekeeping run will take a + # _lot_ longer. It is not recommended to use eager housekeeping in general, but only in situations where it is + # explicitly required. + def optimize_repository(eager: false) + strategy = if eager + Gitaly::OptimizeRepositoryRequest::Strategy::STRATEGY_EAGER + else + Gitaly::OptimizeRepositoryRequest::Strategy::STRATEGY_HEURISTICAL + end + + request = Gitaly::OptimizeRepositoryRequest.new(repository: @gitaly_repo, + strategy: strategy) gitaly_client_call(@storage, :repository_service, :optimize_repository, request, timeout: GitalyClient.long_timeout) end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index de66f310edf..b1e259fe940 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -113,6 +113,9 @@ module Gitlab state: ::MergeRequestReviewer.states['reviewed'], created_at: submitted_at ) + rescue ActiveRecord::RecordNotUnique + # multiple reviews from single person could make a SQL concurrency issue here + nil end # rubocop:disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 4090555c85e..e0a7e6479f5 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -16,6 +16,10 @@ module Gitlab @parallel = parallel @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % { project: project.id, collection: collection_method } + @job_waiter_cache_key = JOB_WAITER_CACHE_KEY % + { project: project.id, collection: collection_method } + @job_waiter_remaining_cache_key = JOB_WAITER_REMAINING_CACHE_KEY % + { project: project.id, collection: collection_method } end # In single endpoint there is no issue info to which associated related diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index 2424b3e8c1f..8e9d6d8dd50 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -16,8 +16,8 @@ module Gitlab PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' class << self - def format(*args) - new(*args).to_s + def format(...) + new(...).to_s end # Links like `https://domain.github.com/<namespace>/<project>/pull/<iid>` needs to be converted diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index 8873db24118..7ee64b2abac 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -7,6 +7,7 @@ module Gitlab OPERATIONS = %w[fetched imported].freeze PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}' PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}' + EMPTY_SUMMARY = OPERATIONS.index_with { |operation| {} } GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}' GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}' @@ -29,6 +30,18 @@ module Gitlab end def summary(project) + cached_summary = cashed_summary(project) + # Actual information about objects that have already been imported is stored + # in the Redis Cache until Redis key is expired. + # After import is completed we store this information in project's import_checksums + return cached_summary if cached_summary != EMPTY_SUMMARY || project.import_state.blank? + + project.import_state.in_progress? ? cached_summary : project.import_checksums + end + + private + + def cashed_summary(project) OPERATIONS.each_with_object({}) do |operation, result| result[operation] = {} @@ -42,8 +55,6 @@ module Gitlab end end - private - # Global counters are long lived, in Prometheus, # and it's used to report the health of the Github Importer # in the Grafana Dashboard diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 03aa02fb659..4b54a77983d 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -3,11 +3,18 @@ module Gitlab module GithubImport module ParallelScheduling - attr_reader :project, :client, :page_counter, :already_imported_cache_key + attr_reader :project, :client, :page_counter, :already_imported_cache_key, + :job_waiter_cache_key, :job_waiter_remaining_cache_key # The base cache key to use for tracking already imported objects. ALREADY_IMPORTED_CACHE_KEY = 'github-importer/already-imported/%{project}/%{collection}' + # The base cache key to use for storing job waiter key + JOB_WAITER_CACHE_KEY = + 'github-importer/job-waiter/%{project}/%{collection}' + # The base cache key to use for storing job waiter remaining jobs + JOB_WAITER_REMAINING_CACHE_KEY = + 'github-importer/job-waiter-remaining/%{project}/%{collection}' # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. @@ -19,6 +26,10 @@ module Gitlab @page_counter = PageCounter.new(project, collection_method) @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % { project: project.id, collection: collection_method } + @job_waiter_cache_key = JOB_WAITER_CACHE_KEY % + { project: project.id, collection: collection_method } + @job_waiter_remaining_cache_key = JOB_WAITER_REMAINING_CACHE_KEY % + { project: project.id, collection: collection_method } end def parallel? @@ -74,7 +85,27 @@ module Gitlab def parallel_import raise 'Batch settings must be defined for parallel import' if parallel_import_batch.blank? - spread_parallel_import + if Feature.enabled?(:improved_spread_parallel_import) + improved_spread_parallel_import + else + spread_parallel_import + end + end + + def improved_spread_parallel_import + enqueued_job_counter = 0 + + each_object_to_import do |object| + repr = object_representation(object) + + job_delay = calculate_job_delay(enqueued_job_counter) + sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash, job_waiter.key) + enqueued_job_counter += 1 + + job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key) + end + + job_waiter end def spread_parallel_import @@ -233,6 +264,22 @@ module Gitlab parallel: parallel? ) end + + def job_waiter + @job_waiter ||= begin + key = Gitlab::Cache::Import::Caching.read(job_waiter_cache_key) + key ||= Gitlab::Cache::Import::Caching.write(job_waiter_cache_key, JobWaiter.generate_key) + jobs_remaining = Gitlab::Cache::Import::Caching.read(job_waiter_remaining_cache_key).to_i || 0 + + JobWaiter.new(jobs_remaining, key) + end + end + + def calculate_job_delay(job_index) + multiplier = (job_index / parallel_import_batch[:size]) + + (multiplier * parallel_import_batch[:delay]) + 1.second + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index ceef072a710..c9766ee095a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -40,7 +40,6 @@ module Gitlab gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') - gon.select2_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/select2.css') gon.gridstack_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/gridstack.css') gon.test_env = Rails.env.test? gon.disable_animations = Gitlab.config.gitlab['disable_animations'] @@ -68,7 +67,7 @@ module Gitlab push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) push_frontend_feature_flag(:integration_slack_app_notifications) - push_frontend_feature_flag(:new_fonts, current_user) + push_frontend_feature_flag(:full_path_project_search, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 884fc85c4ec..983bdb9c0a2 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -67,8 +67,8 @@ module Gitlab self.class.authorization.ok?(object, current_user) end - def raise_resource_not_available_error!(*args) - self.class.raise_resource_not_available_error!(*args) + def raise_resource_not_available_error!(...) + self.class.raise_resource_not_available_error!(...) end end end diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb deleted file mode 100644 index 9b17962f9ec..00000000000 --- a/lib/gitlab/graphql/deprecation.rb +++ /dev/null @@ -1,149 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - class Deprecation - REASON_RENAMED = :renamed - REASON_ALPHA = :alpha - - REASONS = { - REASON_RENAMED => 'This was renamed.', - REASON_ALPHA => 'This feature is in Alpha. It can be changed or removed at any time.' - }.freeze - - include ActiveModel::Validations - - validates :milestone, presence: true, format: { with: /\A\d+\.\d+\z/, message: 'must be milestone-ish' } - validates :reason, presence: true - validates :reason, - format: { with: /.*[^.]\z/, message: 'must not end with a period' }, - if: :reason_is_string? - validate :milestone_is_string - validate :reason_known_or_string - - def self.parse(alpha: nil, deprecated: nil) - options = alpha || deprecated - return unless options - - if alpha - raise ArgumentError, '`alpha` and `deprecated` arguments cannot be passed at the same time' \ - if deprecated - - options[:reason] = :alpha - end - - new(**options) - end - - def initialize(reason: nil, milestone: nil, replacement: nil) - @reason = reason.presence - @milestone = milestone.presence - @replacement = replacement.presence - end - - def ==(other) - return false unless other.is_a?(self.class) - - [reason_text, milestone, replacement] == [:reason_text, :milestone, :replacement].map do |attr| - other.send(attr) # rubocop: disable GitlabSecurity/PublicSend - end - end - alias_method :eql, :== - - def markdown(context: :inline) - parts = [ - "#{changed_in_milestone(format: :markdown)}.", - reason_text, - replacement_markdown.then { |r| "Use: #{r}." if r } - ].compact - - case context - when :block - ['WARNING:', *parts].join("\n") - when :inline - parts.join(' ') - end - end - - def replacement_markdown - return unless replacement.present? - return "`#{replacement}`" unless replacement.include?('.') # only fully qualified references can be linked - - "[`#{replacement}`](##{replacement.downcase.tr('.', '')})" - end - - def edit_description(original_description) - @original_description = original_description - return unless original_description - - original_description + description_suffix - end - - def original_description - return unless @original_description - return @original_description if @original_description.ends_with?('.') - - "#{@original_description}." - end - - def deprecation_reason - [ - reason_text, - replacement && "Please use `#{replacement}`.", - "#{changed_in_milestone}." - ].compact.join(' ') - end - - def alpha? - reason == REASON_ALPHA - end - - private - - attr_reader :reason, :milestone, :replacement - - def milestone_is_string - return if milestone.is_a?(String) - - errors.add(:milestone, 'must be a string') - end - - def reason_known_or_string - return if REASONS.key?(reason) - return if reason_is_string? - - errors.add(:reason, 'must be a known reason or a string') - end - - def reason_is_string? - reason.is_a?(String) - end - - def reason_text - @reason_text ||= REASONS[reason] || "#{reason.to_s.strip}." - end - - def description_suffix - " #{changed_in_milestone}: #{reason_text}" - end - - # Returns 'Deprecated in <milestone>' for proper deprecations. - # Retruns 'Introduced in <milestone>' for :alpha deprecations. - # Formatted to markdown or plain format. - def changed_in_milestone(format: :plain) - verb = if alpha? - 'Introduced' - else - 'Deprecated' - end - - case format - when :plain - "#{verb} in #{milestone}" - when :markdown - "**#{verb}** in #{milestone}" - end - end - end - end -end diff --git a/lib/gitlab/graphql/deprecations.rb b/lib/gitlab/graphql/deprecations.rb new file mode 100644 index 00000000000..9cd8462f2e8 --- /dev/null +++ b/lib/gitlab/graphql/deprecations.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Concern for handling GraphQL deprecations. +# https://docs.gitlab.com/ee/development/api_graphql_styleguide.html#deprecating-schema-items +module Gitlab + module Graphql + module Deprecations + extend ActiveSupport::Concern + + included do + attr_accessor :deprecation + end + + def visible?(ctx) + super && ctx[:remove_deprecated] == true ? deprecation.nil? : true + end + + private + + # Set deprecation, mutate the arguments + def init_gitlab_deprecation(kwargs) + if kwargs[:deprecation_reason].present? + raise ArgumentError, <<~ERROR + Use `deprecated` property instead of `deprecation_reason`. See + #{Rails.application.routes.url_helpers.help_page_url('development/api_graphql_styleguide', anchor: 'deprecating-schema-items')} + ERROR + end + + # GitLab allows items to be marked as "alpha", which leverages GraphQL deprecations. + # TODO remove + deprecation_args = kwargs.extract!(:alpha, :deprecated) + + self.deprecation = Deprecation.parse(**deprecation_args) + return unless deprecation + + unless deprecation.valid? + raise ArgumentError, "Bad deprecation. #{deprecation.errors.full_messages.to_sentence}" + end + + kwargs[:deprecation_reason] = deprecation.deprecation_reason + kwargs[:description] = deprecation.edit_description(kwargs[:description]) + end + end + end +end diff --git a/lib/gitlab/graphql/deprecations/deprecation.rb b/lib/gitlab/graphql/deprecations/deprecation.rb new file mode 100644 index 00000000000..7f4cea7c635 --- /dev/null +++ b/lib/gitlab/graphql/deprecations/deprecation.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Deprecations + class Deprecation + REASON_RENAMED = :renamed + REASON_ALPHA = :alpha # TODO remove support in this class + + REASONS = { + REASON_RENAMED => 'This was renamed.', + REASON_ALPHA => 'This feature is in Alpha. It can be changed or removed at any time.' + }.freeze + + include ActiveModel::Validations + + validates :milestone, presence: true, format: { with: /\A\d+\.\d+\z/, message: 'must be milestone-ish' } + validates :reason, presence: true + validates :reason, + format: { with: /.*[^.]\z/, message: 'must not end with a period' }, + if: :reason_is_string? + validate :milestone_is_string + validate :reason_known_or_string + + def self.parse(alpha: nil, deprecated: nil) + options = alpha || deprecated + return unless options + + if alpha + raise ArgumentError, '`alpha` and `deprecated` arguments cannot be passed at the same time' \ + if deprecated + + options[:reason] = :alpha + end + + new(**options) + end + + def initialize(reason: nil, milestone: nil, replacement: nil) + @reason = reason.presence + @milestone = milestone.presence + @replacement = replacement.presence + end + + def ==(other) + return false unless other.is_a?(self.class) + + [reason_text, milestone, replacement] == [:reason_text, :milestone, :replacement].map do |attr| + other.send(attr) # rubocop: disable GitlabSecurity/PublicSend + end + end + alias_method :eql, :== + + def markdown(context: :inline) + parts = [ + "#{changed_in_milestone(format: :markdown)}.", + reason_text, + replacement_markdown.then { |r| "Use: #{r}." if r } + ].compact + + case context + when :block + ['WARNING:', *parts].join("\n") + when :inline + parts.join(' ') + end + end + + def replacement_markdown + return unless replacement.present? + return "`#{replacement}`" unless replacement.include?('.') # only fully qualified references can be linked + + "[`#{replacement}`](##{replacement.downcase.tr('.', '')})" + end + + def edit_description(original_description) + @original_description = original_description + return unless original_description + + original_description + description_suffix + end + + def original_description + return unless @original_description + return @original_description if @original_description.ends_with?('.') + + "#{@original_description}." + end + + def deprecation_reason + [ + reason_text, + replacement && "Please use `#{replacement}`.", + "#{changed_in_milestone}." + ].compact.join(' ') + end + + def alpha? + reason == REASON_ALPHA + end + + private + + attr_reader :reason, :milestone, :replacement + + def milestone_is_string + return if milestone.is_a?(String) + + errors.add(:milestone, 'must be a string') + end + + def reason_known_or_string + return if REASONS.key?(reason) + return if reason_is_string? + + errors.add(:reason, 'must be a known reason or a string') + end + + def reason_is_string? + reason.is_a?(String) + end + + def reason_text + @reason_text ||= REASONS[reason] || "#{reason.to_s.strip}." + end + + def description_suffix + " #{changed_in_milestone}: #{reason_text}" + end + + # Returns 'Deprecated in <milestone>' for proper deprecations. + # Retruns 'Introduced in <milestone>' for :alpha deprecations. + # Formatted to markdown or plain format. + def changed_in_milestone(format: :plain) + verb = if alpha? + 'Introduced' + else + 'Deprecated' + end + + case format + when :plain + "#{verb} in #{milestone}" + when :markdown + "**#{verb}** in #{milestone}" + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb index 43dddf4c4bc..f7bd1d43995 100644 --- a/lib/gitlab/graphql/markdown_field.rb +++ b/lib/gitlab/graphql/markdown_field.rb @@ -15,7 +15,7 @@ module Gitlab resolver_method = "#{name}_resolver".to_sym kwargs[:resolver_method] = resolver_method - kwargs[:description] ||= "The GitLab Flavored Markdown rendering of `#{method_name}`" + kwargs[:description] ||= "GitLab Flavored Markdown rendering of `#{method_name}`" # Adding complexity to rendered notes since that could cause queries. kwargs[:complexity] ||= 5 diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index cf06a2729d9..9cdc84ffaa3 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -91,7 +91,8 @@ module Gitlab end def print_field(field, indent: '') - if skips? && field.directives.any? { |d| d.name == 'client' } + if skips? && + (field.directives.any? { |d| d.name == 'client' || d.name == 'persist' } || field.name == '__persist') skipped = self.class.new(false) skipped.print_node(field) @@ -136,7 +137,7 @@ module Gitlab qs = [query] + all_imports(mode: mode).uniq.sort.map { |p| fragment(p).query } t = qs.join("\n\n").gsub(/\n\n+/, "\n\n") - return t unless /@client/.match?(t) + return t unless /(@client)|(persist)/.match?(t) doc = ::GraphQL.parse(t) printer = ClientFieldRedactor.new diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 3ef60be67a9..aec430f2686 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -59,8 +59,6 @@ module Gitlab end def dns_rebind_protection? - return false if Gitlab.http_proxy_env? - Gitlab::CurrentSettings.dns_rebinding_protection_enabled? end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 31952f75006..8fe5868ca57 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 35, + 'da_DK' => 34, 'de' => 16, 'en' => 100, 'eo' => 0, - 'es' => 34, + 'es' => 33, 'fil_PH' => 0, - 'fr' => 98, + 'fr' => 99, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 29, + 'ja' => 31, 'ko' => 20, - 'nb_NO' => 24, + 'nb_NO' => 23, 'nl_NL' => 0, 'pl_PL' => 3, 'pt_BR' => 57, - 'ro_RO' => 94, + 'ro_RO' => 91, 'ru' => 26, 'si_LK' => 11, 'tr_TR' => 10, - 'uk' => 54, + 'uk' => 55, 'zh_CN' => 98, 'zh_HK' => 1, - 'zh_TW' => 99 + 'zh_TW' => 98 }.freeze private_constant :TRANSLATION_LEVELS @@ -116,5 +116,43 @@ module Gitlab def with_default_locale(&block) with_locale(::I18n.default_locale, &block) end + + def setup(domain:, default_locale:) + setup_repositories(domain) + setup_default_locale(default_locale) + end + + private + + def setup_repositories(domain) + translation_repositories = [ + (po_repository(domain, 'jh/locale') if Gitlab.jh?), + po_repository(domain, 'locale') + ].compact + + FastGettext.add_text_domain( + domain, + type: :chain, + chain: translation_repositories, + ignore_fuzzy: true + ) + + FastGettext.default_text_domain = domain + end + + def po_repository(domain, path) + FastGettext::TranslationRepository.build( + domain, + path: Rails.root.join(path), + type: :po, + ignore_fuzzy: true + ) + end + + def setup_default_locale(locale) + FastGettext.default_locale = locale + FastGettext.default_available_locales = available_locales + ::I18n.available_locales = available_locales + end end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index f3cca97950d..6623d42f526 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -65,7 +65,7 @@ module Gitlab end def translations_have_multiple_lines? - translation_entries.any? { |translation| translation.is_a?(Array) } + translation_entries.any?(Array) end def msgid_contains_unescaped_chars? diff --git a/lib/gitlab/import_export/base/object_builder.rb b/lib/gitlab/import_export/base/object_builder.rb index 7dee0f783cc..0f24492ed3c 100644 --- a/lib/gitlab/import_export/base/object_builder.rb +++ b/lib/gitlab/import_export/base/object_builder.rb @@ -15,8 +15,8 @@ module Gitlab LRU_CACHE_SIZE = 1000 class ObjectBuilder - def self.build(*args) - new(*args).find + def self.build(...) + new(...).find end def initialize(klass, attributes) diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index d1fd45882d3..e3813070aa4 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -158,9 +158,9 @@ module Gitlab end def remove_encrypted_attributes! - return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + return unless relation_class.respond_to?(:attr_encrypted_attributes) && relation_class.attr_encrypted_attributes.any? - relation_class.encrypted_attributes.each_key do |key| + relation_class.attr_encrypted_attributes.each_key do |key| @relation_hash[key.to_s] = nil end end diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index 7f3254be3e8..e30414265be 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -98,6 +98,9 @@ methods: epics: - :state +# Add in this list the nested associations that are used to export the parent +# association, but are not present in the tree list. In other words, the associations +# that needs to be preloaded but do not need to be exported. preloads: export_reorders: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c2510bbe938..628c7c71539 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -176,3 +176,5 @@ module Gitlab end end end + +Gitlab::ImportExport::Importer.prepend_mod_with('Gitlab::ImportExport::Importer') diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index cf62f181366..389ab8b4c97 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -85,17 +85,30 @@ module Gitlab end def exportable_json_record(record, options, key) - associations = relations_schema[:include_if_exportable]&.dig(key) - return Raw.new(record.to_json(options)) unless associations && options[:include] + return Raw.new(record.to_json(options)) unless options[:include].any? + conditional_associations = relations_schema[:include_if_exportable]&.dig(key) + + filtered_options = + if conditional_associations.present? + filter_conditional_include(record, options, conditional_associations) + else + options + end + + Raw.new(authorized_record_json(record, filtered_options)) + end + + def filter_conditional_include(record, options, conditional_associations) filtered_options = options.deep_dup - associations.each do |association| + + conditional_associations.each do |association| filtered_options[:include].delete_if do |option| !exportable_json_association?(option, record, association.to_sym) end end - Raw.new(record.to_json(filtered_options)) + filtered_options end def exportable_json_association?(option, record, association) @@ -105,6 +118,34 @@ module Gitlab record.exportable_association?(association, current_user: current_user) end + def authorized_record_json(record, options) + include_keys = options[:include].flat_map(&:keys) + keys_to_authorize = record.try(:restricted_associations, include_keys) + return record.to_json(options) if keys_to_authorize.blank? + + record_hash = record.as_json(options).with_indifferent_access + filtered_record_hash(record, keys_to_authorize, record_hash).to_json(options) + end + + def filtered_record_hash(record, keys_to_authorize, record_hash) + keys_to_authorize.each do |key| + next unless record_hash[key].present? + + readable = record.try(:readable_records, key, current_user: current_user) + if record.has_many_association?(key) + readable_ids = readable.pluck(:id) + + record_hash[key].keep_if do |association_record| + readable_ids.include?(association_record[:id]) + end + else + record_hash[key] = nil unless readable.present? + end + end + + record_hash + end + def batch(relation, key) opts = { of: BATCH_SIZE } order_by = reorders(relation, key) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 99364996864..d97ffee8698 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -1005,6 +1005,7 @@ excluded_attributes: - :protected_branch_id create_access_levels: - :protected_tag_id + - :deploy_key_id deploy_access_levels: - :protected_environment_id boards: @@ -1104,6 +1105,9 @@ methods: issues: - :state +# Add in this list the nested associations that are used to export the parent +# association, but are not present in the tree list. In other words, the associations +# that needs to be preloaded but do not need to be exported. preloads: issues: project: :route @@ -1112,8 +1116,8 @@ preloads: # tags: # needed by tag_list project: # deprecated: needed by coverage_regex of Ci::Build merge_requests: - source_project: # needed by source_branch_sha and diff_head_sha - target_project: # needed by target_branch_sha + source_project: :route # needed by source_branch_sha and diff_head_sha + target_project: :route # needed by target_branch_sha assignees: # needed by assigne_id that is implemented by DeprecatedAssignee # Specify a custom export reordering for a given relationship diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 568315930d8..4134c428500 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -165,9 +165,21 @@ module Gitlab end def setup_protected_branch_access_level + return if root_group_owner? + return if @relation_hash['access_level'] == Gitlab::Access::NO_ACCESS + return if @relation_hash['access_level'] == Gitlab::Access::MAINTAINER + @relation_hash['access_level'] = Gitlab::Access::MAINTAINER end + def root_group_owner? + root_ancestor = @importable.root_ancestor + + return false unless root_ancestor.is_a?(::Group) + + root_ancestor.max_member_access_for_user(@user) == Gitlab::Access::OWNER + end + def compute_relative_position return unless max_relative_position diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 52234b50a1f..880b112d815 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -38,6 +38,10 @@ module Gitlab key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/o end + def self.generate_key + "#{KEY_PREFIX}:#{SecureRandom.uuid}" + end + attr_reader :key, :finished, :worker_label attr_accessor :jobs_remaining diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 8332e4f6d56..bdfbe2041cd 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -103,7 +103,7 @@ module Gitlab opts = standardize_opts(opts) Oj.load(string, opts) - rescue Oj::ParseError, EncodingError, Encoding::UndefinedConversionError => ex + rescue Oj::ParseError, EncodingError, Encoding::UndefinedConversionError, JSON::GeneratorError => ex raise parser_error, ex end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index f7cd28df5c9..bad2e265f73 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -3,6 +3,9 @@ require 'yaml' require 'json' require 'pathname' +require 'active_support' +require "active_support/core_ext/module/delegation" +require_relative 'encrypted_configuration' unless defined?(Gitlab::EncryptedConfiguration) require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) # This service is run independently of the main Rails process, @@ -42,6 +45,20 @@ module Gitlab } }.freeze + # Default path strings (this is a data duplication + # with Settings which is not pulled in - see the service + # comment at the top of this file) + DEFAULT_PATHS = { + shared_path: 'shared', + encrypted_settings_path: 'encrypted_settings', + incoming_email: { + encrypted_secret_filename: 'incoming_email.yaml.enc' + }, + service_desk_email: { + encrypted_secret_filename: 'service_desk_email.yaml.enc' + } + }.freeze + class << self def enabled_configs @enabled_configs ||= configs.select { |_key, config| enabled?(config) } @@ -74,6 +91,12 @@ module Gitlab config[:log_path] = File.expand_path(config[:log_path], RAILS_ROOT_DIR) + # override password/user from any encrypted secrets + if secrets = decrypted_secrets(config_key) + config[:password] = secrets[:password] if secrets[:password] + config[:user] = secrets[:user] if secrets[:user] + end + config end @@ -108,6 +131,58 @@ module Gitlab def load_yaml @yaml ||= YAML.load_file(config_file)[rails_env].deep_symbolize_keys end + + def application_secrets_file + ENV['MAIL_ROOM_GITLAB_SECRETS_FILE'] || File.expand_path('../../config/secrets.yml', __dir__) + end + + def application_secrets + @application_secrets ||= {}.tap do |application_secrets| + # Uses Rails::Secret.parse + # from: https://github.com/rails/rails/blob/v6.1.6.1/railties/lib/rails/secrets.rb#L24 + erb_processed_yaml = ERB.new(File.read(application_secrets_file)).result + yaml_secrets = + YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(erb_processed_yaml) : YAML.safe_load(erb_processed_yaml) + application_secrets.merge!(yaml_secrets["shared"].deep_symbolize_keys) if yaml_secrets["shared"] + application_secrets.merge!(yaml_secrets[rails_env].deep_symbolize_keys) if yaml_secrets[rails_env] + end + end + + def default_encrypted_secret_filename(config_key) + DEFAULT_PATHS[config_key][:encrypted_secret_filename] + end + + def encrypted_secret_file(config_key) + config = merged_configs(config_key) + return config[:encrypted_secret_file] if config[:encrypted_secret_file] + + config_yaml = load_yaml + # Path handling for shared.path / encrypted_settings.path is a duplicate + # of the logic in config/initializers/1_settings.rb + shared_path = File.expand_path(config_yaml.dig(:shared, :path) || + DEFAULT_PATHS[:shared_path], RAILS_ROOT_DIR) + encrypted_settings_path = + File.expand_path(config_yaml.dig(:encrypted_settings, :path) || + File.join(shared_path, DEFAULT_PATHS[:encrypted_settings_path]), + RAILS_ROOT_DIR) + File.join(encrypted_settings_path, default_encrypted_secret_filename(config_key)) + end + + def encrypted_configuration_settings(config_key) + { + content_path: encrypted_secret_file(config_key), + base_key: application_secrets[:encrypted_settings_key_base], + previous_keys: application_secrets[:rotated_encrypted_settings_key_base] || [] + } + end + + def decrypted_secrets(config_key) + settings = encrypted_configuration_settings(config_key) + return if settings[:base_key].nil? + + encrypted = Gitlab::EncryptedConfiguration.new(**settings) + encrypted.active? ? encrypted.config : nil + end end end end diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb index 5effafc9f5b..db0fd24983b 100644 --- a/lib/gitlab/memory/reporter.rb +++ b/lib/gitlab/memory/reporter.rb @@ -69,14 +69,14 @@ module Gitlab report_file = file_name(report) tmp_file_path = File.join(tmp_dir, report_file) - write_heap_dump_file(report, tmp_file_path) + write_compressed_file(report, tmp_file_path) File.join(@reports_path, report_file).tap do |report_file_path| FileUtils.mv(tmp_file_path, report_file_path) end end - def write_heap_dump_file(report, path) + def write_compressed_file(report, path) io_r, io_w = IO.pipe err_r, err_w = IO.pipe pid = nil diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index c94dbed1d46..cc335c00e26 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -6,47 +6,6 @@ module Gitlab # into a handler when the Ruby process violates defined limits # for an extended period of time. class Watchdog - # This handler does nothing. It returns `false` to indicate to the - # caller that the situation has not been dealt with so it will - # receive calls repeatedly if fragmentation remains high. - # - # This is useful for "dress rehearsals" in production since it allows - # us to observe how frequently the handler is invoked before taking action. - class NullHandler - include Singleton - - def call - # NOP - false - end - end - - # This handler sends SIGTERM and considers the situation handled. - class TermProcessHandler - def initialize(pid = $$) - @pid = pid - end - - def call - Process.kill(:TERM, @pid) - true - end - end - - # This handler invokes Puma's graceful termination handler, which takes - # into account a configurable grace period during which a process may - # remain unresponsive to a SIGTERM. - class PumaHandler - def initialize(puma_options = ::Puma.cli_config.options) - @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) - end - - def call - @worker.term - true - end - end - def initialize @configuration = Configuration.new @alive = true @@ -73,6 +32,7 @@ module Gitlab def stop stop_working(reason: 'background task stopped') + handler.stop if handler.respond_to?(:stop) end private @@ -111,7 +71,7 @@ module Gitlab def handler # This allows us to keep the watchdog running but turn it into "friendly mode" where # all that happens is we collect logs and Prometheus events for fragmentation violations. - return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) + return Handlers::NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) configuration.handler end diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb index 5c459220be8..6ab199bf816 100644 --- a/lib/gitlab/memory/watchdog/configuration.rb +++ b/lib/gitlab/memory/watchdog/configuration.rb @@ -48,7 +48,7 @@ module Gitlab end def handler - @handler ||= NullHandler.instance + @handler ||= Handlers::NullHandler.instance end def event_reporter diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb index 04c04cbde02..4a6640ba901 100644 --- a/lib/gitlab/memory/watchdog/configurator.rb +++ b/lib/gitlab/memory/watchdog/configurator.rb @@ -12,12 +12,12 @@ module Gitlab DEFAULT_MAX_HEAP_FRAG = 0.5 DEFAULT_MAX_MEM_GROWTH = 3.0 # grace_time / sleep_interval = max_strikes allowed for Sidekiq process to violate defined limits. - DEFAULT_SIDEKIQ_GRACE_TIME_S = 300 + DEFAULT_SIDEKIQ_GRACE_TIME_S = 900 class << self def configure_for_puma ->(config) do - config.handler = Gitlab::Memory::Watchdog::PumaHandler.new + config.handler = Gitlab::Memory::Watchdog::Handlers::PumaHandler.new config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', DEFAULT_SLEEP_INTERVAL_S).to_i config.monitors(&configure_monitors_for_puma) end @@ -25,7 +25,13 @@ module Gitlab def configure_for_sidekiq ->(config) do - config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new + # Give Sidekiq up to 30 seconds to allow existing jobs to finish after exceeding the limit + shutdown_timeout_seconds = ENV.fetch('SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT', 30).to_i + + config.handler = Gitlab::Memory::Watchdog::Handlers::SidekiqHandler.new( + shutdown_timeout_seconds, + sidekiq_sleep_time + ) config.sleep_time_seconds = sidekiq_sleep_time config.monitors(&configure_monitors_for_sidekiq) config.event_reporter = SidekiqEventReporter.new diff --git a/lib/gitlab/memory/watchdog/handlers/null_handler.rb b/lib/gitlab/memory/watchdog/handlers/null_handler.rb new file mode 100644 index 00000000000..127001003ce --- /dev/null +++ b/lib/gitlab/memory/watchdog/handlers/null_handler.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Handlers + # This handler does nothing. It returns `false` to indicate to the + # caller that the situation has not been dealt with so it will + # receive calls repeatedly if fragmentation remains high. + # + # This is useful for "dress rehearsals" in production since it allows + # us to observe how frequently the handler is invoked before taking action. + class NullHandler + include Singleton + + def call + # NOP + false + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/handlers/puma_handler.rb b/lib/gitlab/memory/watchdog/handlers/puma_handler.rb new file mode 100644 index 00000000000..fffd91733c8 --- /dev/null +++ b/lib/gitlab/memory/watchdog/handlers/puma_handler.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Handlers + # This handler invokes Puma's graceful termination handler, which takes + # into account a configurable grace period during which a process may + # remain unresponsive to a SIGTERM. + class PumaHandler + def initialize(puma_options = ::Puma.cli_config.options) + @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) + end + + def call + @worker.term + true + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb b/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb new file mode 100644 index 00000000000..47ed608c576 --- /dev/null +++ b/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Handlers + class SidekiqHandler + def initialize(shutdown_timeout_seconds, sleep_time_seconds) + @shutdown_timeout_seconds = shutdown_timeout_seconds + @sleep_time_seconds = sleep_time_seconds + @alive = true + end + + def call + # Tell Sidekiq to stop fetching new jobs + # We first SIGNAL and then wait given time + send_signal(:TSTP, $$, 'stop fetching new jobs', @shutdown_timeout_seconds) + return true unless @alive + + # Tell sidekiq to restart itself + # Keep extra safe to wait `Sidekiq[:timeout] + 2` seconds before SIGKILL + send_signal(:TERM, $$, 'gracefully shut down', Sidekiq[:timeout] + 2) + return true unless @alive + + # Ideally we should never reach this condition + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't + # If process is group leader, kill the whole pgroup, so we can be sure no children are left behind + send_signal(:KILL, Process.getpgrp == $$ ? 0 : $$, 'hard shut down') + + true + end + + def stop + @alive = false + end + + private + + def send_signal(signal, pid, explanation, wait_time = nil) + Sidekiq.logger.warn( + pid: pid, + worker_id: ::Prometheus::PidProvider.worker_id, + memwd_handler_class: self.class.to_s, + memwd_signal: signal, + memwd_explanation: explanation, + memwd_wait_time: wait_time, + message: "Sending signal and waiting" + ) + + ProcessManagement.signal(pid, signal) + + return unless wait_time + + deadline = Gitlab::Metrics::System.monotonic_time + wait_time + + # Sleep until timeout reached + sleep(@sleep_time_seconds) while @alive && Gitlab::Metrics::System.monotonic_time < deadline + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index d4f779ad79d..bdd28744137 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -100,8 +100,8 @@ module Gitlab end # Builds a metrics dashboard url based on the passed in arguments - def build_dashboard_url(*args) - Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) + def build_dashboard_url(...) + Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(...) end private diff --git a/lib/gitlab/metrics/environment.rb b/lib/gitlab/metrics/environment.rb new file mode 100644 index 00000000000..43f2fa3922f --- /dev/null +++ b/lib/gitlab/metrics/environment.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Environment + class << self + def web? + service?('web') + end + + def api? + service?('api') + end + + def git? + service?('git') + end + + def service?(name) + env_var = ENV.fetch('GITLAB_METRICS_INITIALIZE', '') + return true unless env_var.present? + + env_var == name + end + end + end + end +end diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index fc2e805047a..c361d755a12 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -58,8 +58,16 @@ module Gitlab end def endpoint_ids - ['SearchController#show', 'GET /api/:version/search', 'GET /api/:version/projects/:id/(-/)search', - 'GET /api/:version/groups/:id/(-/)search'] + api_endpoints = ['GET /api/:version/search', 'GET /api/:version/projects/:id/(-/)search', + 'GET /api/:version/groups/:id/(-/)search'] + web_endpoints = ['SearchController#show'] + + endpoints = [] + + endpoints += api_endpoints if Gitlab::Metrics::Environment.api? + endpoints += web_endpoints if Gitlab::Metrics::Environment.web? + + endpoints end def possible_labels diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb index 9fd4eec479e..6a5ef5614a3 100644 --- a/lib/gitlab/metrics/rails_slis.rb +++ b/lib/gitlab/metrics/rails_slis.rb @@ -5,8 +5,10 @@ module Gitlab module RailsSlis class << self def initialize_request_slis! - Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, possible_request_labels) - initialize_rails_request_error_rate + request_labels = possible_request_labels + + Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, request_labels) + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:rails_request, request_labels) Gitlab::Metrics::Sli::Apdex.initialize_sli(:graphql_query, possible_graphql_query_labels) end @@ -25,6 +27,8 @@ module Gitlab private def possible_graphql_query_labels + return [] unless Gitlab::Metrics::Environment.api? + ::Gitlab::Graphql::KnownOperations.default.operations.map do |op| { endpoint_id: op.to_caller_id, @@ -39,7 +43,27 @@ module Gitlab possible_controller_labels + possible_api_labels end + def possible_controller_labels + all_controller_labels.select do |labelset| + if known_git_endpoints.include?(labelset[:endpoint_id]) + Gitlab::Metrics::Environment.git? + else + Gitlab::Metrics::Environment.web? + end + end + end + def possible_api_labels + all_api_labels.select do |labelset| + if known_git_endpoints.include?(labelset[:endpoint_id]) + Gitlab::Metrics::Environment.git? + else + Gitlab::Metrics::Environment.api? + end + end + end + + def all_api_labels Gitlab::RequestEndpoints.all_api_endpoints.map do |route| endpoint_id = API::Base.endpoint_id_for_route(route) route_class = route.app.options[:for] @@ -54,7 +78,7 @@ module Gitlab end end - def possible_controller_labels + def all_controller_labels Gitlab::RequestEndpoints.all_controller_actions.map do |controller, action| { endpoint_id: controller.endpoint_id_for_action(action), @@ -64,10 +88,27 @@ module Gitlab end end - def initialize_rails_request_error_rate - return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development) - - Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:rails_request, possible_request_labels) + def known_git_endpoints + # This is a list of endpoints that endpoints that HAProxy redirects + # to the git fleet for GitLab.com. It is taken from + # https://thanos-query.ops.gitlab.net/graph?g0.expr=sum%20by%20(endpoint_id)(sli_aggregations%3Agitlab_sli_rails_request_total_rate6h%7Btype%3D%22git%22%2C%20env%3D%22gprd%22%7D%20%3E%200)&g0.tab=1&g0.stacked=0&g0.range_input=1h&g0.max_source_resolution=0s&g0.deduplicate=1&g0.partial_response=0&g0.store_matches=%5B%5D + [ + "GET /api/:version/internal/authorized_keys", + "GET /api/:version/internal/discover", + "POST /api/:version/internal/allowed", + "POST /api/:version/internal/lfs_authenticate", + "POST /api/:version/internal/two_factor_recovery_codes", + "ProjectsController#show", + "Repositories::GitHttpController#git_receive_pack", + "Repositories::GitHttpController#git_upload_pack", + "Repositories::GitHttpController#info_refs", + "Repositories::LfsApiController#batch", + "Repositories::LfsLocksApiController#index", + "Repositories::LfsLocksApiController#verify", + "Repositories::LfsStorageController#download", + "Repositories::LfsStorageController#upload_authorize", + "Repositories::LfsStorageController#upload_finalize" + ] end end end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index cfdac5264e0..f635deabf76 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -27,6 +27,8 @@ module Gitlab 'not_owned', 'source_code_management', FEATURE_CATEGORY_DEFAULT].freeze + REQUEST_URGENCY_KEY = 'gitlab.request_urgency' + def initialize(app) @app = app end @@ -125,8 +127,6 @@ module Gitlab end def record_error(urgency, status) - return unless Feature.enabled?(:gitlab_metrics_error_rate_sli, type: :development) - Gitlab::Metrics::RailsSlis.request_error_rate.increment( labels: labels_from_context.merge(request_urgency: urgency.name), error: ::Gitlab::Metrics.server_error?(status) @@ -142,7 +142,9 @@ module Gitlab def urgency_for_env(env) endpoint_urgency = - if env['api.endpoint'].present? + if env[REQUEST_URGENCY_KEY].present? + env[REQUEST_URGENCY_KEY] + elsif env['api.endpoint'].present? env['api.endpoint'].options[:for].try(:urgency_for_app, env['api.endpoint']) elsif env['action_controller.instance'].present? && env['action_controller.instance'].respond_to?(:urgency) env['action_controller.instance'].urgency diff --git a/lib/gitlab/metrics/subscribers/ldap.rb b/lib/gitlab/metrics/subscribers/ldap.rb index 9cac5f41090..3dae2d1fd88 100644 --- a/lib/gitlab/metrics/subscribers/ldap.rb +++ b/lib/gitlab/metrics/subscribers/ldap.rb @@ -73,8 +73,8 @@ module Gitlab def add_to_request_store(event) return unless Gitlab::SafeRequestStore.active? - Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 - Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + event.duration.to_f + Gitlab::SafeRequestStore[COUNTER] = self.class.count + 1 + Gitlab::SafeRequestStore[DURATION] = self.class.duration + convert_to_seconds(event.duration) end # Converts the observed events into Prometheus metrics @@ -85,18 +85,23 @@ module Gitlab # and so we only want the first part, which is the # true name of the event labels = { name: event.name.split(".").first } + duration = convert_to_seconds(event.duration) current_transaction.increment(:gitlab_net_ldap_total, 1, labels) do docstring 'Net::LDAP calls' label_keys labels.keys end - current_transaction.observe(:gitlab_net_ldap_duration_seconds, event.duration, labels) do + current_transaction.observe(:gitlab_net_ldap_duration_seconds, duration, labels) do docstring 'Net::LDAP time' buckets [0.001, 0.01, 0.1, 1.0, 2.0, 5.0] label_keys labels.keys end end + + def convert_to_seconds(duration_f) + (BigDecimal(duration_f.to_s) / BigDecimal("1000.0")).to_f + end end end end diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb index 8cb2729ff61..10b841f777e 100644 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -11,12 +11,12 @@ module Gitlab # Using delegate hides the stacktrace for some errors, so we choose to be explicit. # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62047#note_579031091 - def add_primary_menu_item(**args) - @menu_builder.add_primary_menu_item(**args) + def add_primary_menu_item(...) + @menu_builder.add_primary_menu_item(...) end - def add_secondary_menu_item(**args) - @menu_builder.add_secondary_menu_item(**args) + def add_secondary_menu_item(...) + @menu_builder.add_secondary_menu_item(...) end def add_shortcut(**args) diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb index a92860f7eb8..0e47672bb3c 100644 --- a/lib/gitlab/octokit/middleware.rb +++ b/lib/gitlab/octokit/middleware.rb @@ -11,7 +11,8 @@ module Gitlab Gitlab::UrlBlocker.validate!(env[:url], schemes: %w[http https], allow_localhost: allow_local_requests?, - allow_local_network: allow_local_requests? + allow_local_network: allow_local_requests?, + dns_rebind_protection: dns_rebind_protection? ) @app.call(env) @@ -22,6 +23,10 @@ module Gitlab def allow_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end + + def dns_rebind_protection? + Gitlab::CurrentSettings.dns_rebinding_protection_enabled? + end end end end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index b78cd2a6b95..a03533dcd9a 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -23,8 +23,6 @@ module Gitlab case provider_name when 'cas3' { on_single_sign_out: cas3_signout_handler } - when 'authentiq' - { remote_sign_out_handler: authentiq_signout_handler } when 'shibboleth' { fail_with_empty_uid: true } when 'google_oauth2' @@ -53,24 +51,12 @@ module Gitlab true end end - - def authentiq_signout_handler - lambda do |request| - authentiq_session = request.params['sid'] - if Gitlab::Auth::OAuth::Session.valid?(:authentiq, authentiq_session) - Gitlab::Auth::OAuth::Session.destroy(:authentiq, authentiq_session) - true - else - false - end - end - end end private - def add_provider_to_devise(*args) - @devise_config.omniauth(*args) + def add_provider_to_devise(...) + @devise_config.omniauth(...) end def arguments_for(provider) diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 38618a0ac06..8e02486d8c6 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -67,7 +67,7 @@ module Gitlab attr_reader :old_key, :new_key def otp_secret_settings - @otp_secret_settings ||= User.encrypted_attributes[:otp_secret] + @otp_secret_settings ||= User.attr_encrypted_attributes[:otp_secret] end def reencrypt(user, old_key, new_key) diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb index a24d958b7e5..81da34f1219 100644 --- a/lib/gitlab/pages/cache_control.rb +++ b/lib/gitlab/pages/cache_control.rb @@ -47,8 +47,15 @@ module Gitlab # cached settings hash to build the payload cache key to be invalidated. def clear_cache keys = cached_settings_hashes - .map { |hash| payload_cache_key_for(hash) } - .push(settings_cache_key) + .map { |hash| payload_cache_key_for(hash) } + .push(settings_cache_key) + + ::Gitlab::AppLogger.info( + message: 'clear pages cache', + pages_keys: keys, + pages_type: @type, + pages_id: @id + ) Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do Rails.cache.delete_multi(keys) diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index b974c0b2c7f..7619fc317a1 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -8,6 +8,7 @@ # 2. Allow `prepended do; end` work like `included do; end` # If we don't need anything above, we don't need this patch nor the concern! +require_dependency 'gitlab/environment' # rubocop:disable Gitlab/ModuleWithInstanceVariables module Gitlab module Patch @@ -51,7 +52,7 @@ module Gitlab end # Hack to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/23932 - extend class_methods_module if ENV['STATIC_VERIFICATION'] + extend class_methods_module if Gitlab::Environment.static_verification? end def prepended(base = nil, &block) diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 8a1dcc083e8..28d195238ea 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -46,8 +46,8 @@ module Gitlab @options = parse_options(options) end - def get(*args) - options.dig(*args) + def get(...) + options.dig(...) end # Allow #to_json serialization @@ -83,7 +83,7 @@ module Gitlab end def option_multi_value?(namespace, key) - MULTI_VALUE_OPTIONS.any? { |arr| arr == [namespace, key] } + MULTI_VALUE_OPTIONS.any?([namespace, key]) end def parse_option(option) diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index d9135d1bacb..118c8aad460 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -149,7 +149,13 @@ module Gitlab end def valid_type?(context) - types.blank? || types.any? { |type| context.quick_action_target.is_a?(type) } + types.blank? || types.any? do |type| + if context.quick_action_target.is_a?(WorkItem) + context.quick_action_target.supported_quick_action_commands.include?(name.to_sym) + else + context.quick_action_target.is_a?(type) + end + end end end end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index a12457d89c9..96e3112f32f 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -248,7 +248,7 @@ module Gitlab if severity if quick_action_target.persisted? - ::Issues::UpdateService.new(project: quick_action_target.project, current_user: current_user, params: { severity: severity }).execute(quick_action_target) + ::Issues::UpdateService.new(container: quick_action_target.project, current_user: current_user, params: { severity: severity }).execute(quick_action_target) else quick_action_target.build_issuable_severity(severity: severity) end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index f782f2802b6..ae8bc102f57 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -320,7 +320,7 @@ module Gitlab private def zoom_link_service - ::Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) + ::Issues::ZoomLinkService.new(container: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) end def zoom_link_params diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index ed4f6015603..4d15022cca5 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -9,9 +9,11 @@ module Gitlab # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. ALL_CLASSES = [ Gitlab::Redis::Cache, + Gitlab::Redis::DbLoadBalancing, Gitlab::Redis::Queues, Gitlab::Redis::RateLimiting, Gitlab::Redis::RepositoryCache, + Gitlab::Redis::ClusterRateLimiting, Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, Gitlab::Redis::TraceChunks diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 043f14630d5..647573e59fe 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -2,6 +2,16 @@ module Gitlab module Redis + # Match signature in + # https://github.com/rails/rails/blob/v6.1.7.2/activesupport/lib/active_support/cache/redis_cache_store.rb#L59 + ERROR_HANDLER = ->(method:, returning:, exception:) do + Gitlab::ErrorTracking.log_exception( + exception, + method: method, + returning: returning.inspect + ) + end + class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' @@ -12,9 +22,14 @@ module Gitlab redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: CACHE_NAMESPACE, - expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i # Cache should not grow forever + expires_in: default_ttl_seconds, + error_handler: ::Gitlab::Redis::ERROR_HANDLER } end + + def self.default_ttl_seconds + ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + end end end end diff --git a/lib/gitlab/redis/cluster_rate_limiting.rb b/lib/gitlab/redis/cluster_rate_limiting.rb new file mode 100644 index 00000000000..e9d1e4f0c3f --- /dev/null +++ b/lib/gitlab/redis/cluster_rate_limiting.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class ClusterRateLimiting < ::Gitlab::Redis::Wrapper + def self.config_fallback + Cache + end + end + end +end diff --git a/lib/gitlab/redis/db_load_balancing.rb b/lib/gitlab/redis/db_load_balancing.rb new file mode 100644 index 00000000000..01276445611 --- /dev/null +++ b/lib/gitlab/redis/db_load_balancing.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class DbLoadBalancing < ::Gitlab::Redis::Wrapper + class << self + # The data we store on DbLoadBalancing used to be stored on SharedState. + def config_fallback + SharedState + end + + private + + def redis + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, store_name) + end + end + end + end +end diff --git a/lib/gitlab/redis/duplicate_jobs.rb b/lib/gitlab/redis/duplicate_jobs.rb deleted file mode 100644 index c76d298da18..00000000000 --- a/lib/gitlab/redis/duplicate_jobs.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - # Pseudo-store to transition `Gitlab::SidekiqMiddleware::DuplicateJobs` from - # using `Sidekiq.redis` to using the `SharedState` redis store. - class DuplicateJobs < ::Gitlab::Redis::Wrapper - class << self - def store_name - 'SharedState' - end - - private - - def redis - primary_store = ::Redis.new(Gitlab::Redis::SharedState.params) - - # `Sidekiq.redis` is a namespaced redis connection. This means keys are actually being stored under - # "resque:gitlab:resque:gitlab:duplicate:". For backwards compatibility, we make the secondary store - # namespaced in the same way, but omit it from the primary so keys have proper format there. - # rubocop:disable Cop/RedisQueueUsage - secondary_store = ::Redis::Namespace.new( - Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE, redis: ::Redis.new(Gitlab::Redis::Queues.params) - ) - # rubocop:enable Cop/RedisQueueUsage - - MultiStore.new(primary_store, secondary_store, name.demodulize) - end - end - end - end -end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index aa8f390ac10..a102267d52b 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -46,13 +46,6 @@ module Gitlab # # Ref: https://www.rubydoc.info/github/redis/redis-rb/Redis/Commands # - ENUMERATOR_CACHE_HIT_VALIDATOR = { - scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? } - }.freeze - READ_CACHE_HIT_VALIDATOR = { exists: ->(val) { val != 0 }, exists?: ->(val) { val }, @@ -62,13 +55,17 @@ module Gitlab hgetall: ->(val) { val.is_a?(Hash) && !val.empty? }, hlen: ->(val) { val != 0 }, hmget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, + hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, mapped_hmget: ->(val) { val.is_a?(Hash) && !val.compact.empty? }, mget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, + scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, scard: ->(val) { val != 0 }, sismember: ->(val) { val }, smembers: ->(val) { val.is_a?(Array) && !val.empty? }, sscan: ->(val) { val != ['0', []] }, - ttl: ->(val) { val != 0 && val != -2 } + sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, + ttl: ->(val) { val != 0 && val != -2 }, # ttl returns -2 if the key does not exist. See https://redis.io/commands/ttl/ + zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? } }.freeze WRITE_COMMANDS = %i[ @@ -134,20 +131,6 @@ module Gitlab end end - ENUMERATOR_CACHE_HIT_VALIDATOR.each_key do |name| - define_method(name) do |*args, **kwargs, &block| - enumerator = if use_primary_and_secondary_stores? - read_command(name, *args, **kwargs) - else - default_store.send(name, *args, **kwargs) - end - - return enumerator if block.nil? - - enumerator.each(&block) - end - end - PIPELINED_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? @@ -186,11 +169,15 @@ module Gitlab end def use_primary_and_secondary_stores? - feature_enabled?("use_primary_and_secondary_stores_for") + feature_table_exists? && + Feature.enabled?("use_primary_and_secondary_stores_for_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage + !same_redis_store? end def use_primary_store_as_default? - feature_enabled?("use_primary_store_as_default_for") + feature_table_exists? && + Feature.enabled?("use_primary_store_as_default_for_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage + !same_redis_store? end def increment_pipelined_command_error_count(command_name) @@ -217,6 +204,14 @@ module Gitlab extra.merge(command_name: command_name, instance_name: instance_name)) end + def default_store + use_primary_store_as_default? ? primary_store : secondary_store + end + + def fallback_store + use_primary_store_as_default? ? secondary_store : primary_store + end + def ping(message = nil) if use_primary_and_secondary_stores? # Both stores have to response success for the ping to be considered success. @@ -231,23 +226,14 @@ module Gitlab private # @return [Boolean] - def feature_enabled?(prefix) - feature_table_exists? && - Feature.enabled?("#{prefix}_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage - !same_redis_store? - end - - # @return [Boolean] def feature_table_exists? + # Use table_exists? (which uses ActiveRecord's schema cache) instead of Feature.feature_flags_available? + # as the latter runs a ';' SQL query which causes a connection to be checked out. Feature::FlipperFeature.table_exists? rescue StandardError false end - def default_store - use_primary_store_as_default? ? primary_store : secondary_store - end - def log_method_missing(command_name, *_args) return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) @@ -275,26 +261,26 @@ module Gitlab def read_one_with_fallback(command_name, *args, **kwargs, &block) begin - value = send_command(primary_store, command_name, *args, **kwargs, &block) + value = send_command(default_store, command_name, *args, **kwargs, &block) rescue StandardError => e log_error(e, command_name, multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) end - return value if cache_hit?(command_name, value) + return value if block.nil? && cache_hit?(command_name, value) fallback_read(command_name, *args, **kwargs, &block) end def cache_hit?(command, value) - validator = READ_CACHE_HIT_VALIDATOR[command] || ENUMERATOR_CACHE_HIT_VALIDATOR[command] + validator = READ_CACHE_HIT_VALIDATOR[command] return false unless validator !value.nil? && validator.call(value) end def fallback_read(command_name, *args, **kwargs, &block) - value = send_command(secondary_store, command_name, *args, **kwargs, &block) + value = send_command(fallback_store, command_name, *args, **kwargs, &block) if value log_error(ReadFromPrimaryError.new, command_name) diff --git a/lib/gitlab/redis/rate_limiting.rb b/lib/gitlab/redis/rate_limiting.rb index 4ae1d55e4ce..12710bafbea 100644 --- a/lib/gitlab/redis/rate_limiting.rb +++ b/lib/gitlab/redis/rate_limiting.rb @@ -3,13 +3,28 @@ module Gitlab module Redis class RateLimiting < ::Gitlab::Redis::Wrapper - # The data we store on RateLimiting used to be stored on Cache. - def self.config_fallback - Cache - end + class << self + # The data we store on RateLimiting used to be stored on Cache. + def config_fallback + Cache + end + + def cache_store + @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new( + redis: pool, + namespace: Cache::CACHE_NAMESPACE, + error_handler: ::Gitlab::Redis::ERROR_HANDLER + ) + end + + private + + def redis + primary_store = ::Redis.new(::Gitlab::Redis::ClusterRateLimiting.params) + secondary_store = ::Redis.new(params) - def self.cache_store - @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new(redis: pool, namespace: Cache::CACHE_NAMESPACE) + MultiStore.new(primary_store, secondary_store, name.demodulize) + end end end end diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb index 8bfbfcfea60..6c7bc8c41d5 100644 --- a/lib/gitlab/redis/repository_cache.rb +++ b/lib/gitlab/redis/repository_cache.rb @@ -14,19 +14,10 @@ module Gitlab redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: Cache::CACHE_NAMESPACE, - # Cache should not grow forever - expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + expires_in: Cache.default_ttl_seconds, + error_handler: ::Gitlab::Redis::ERROR_HANDLER ) end - - private - - def redis - primary_store = ::Redis.new(params) - secondary_store = ::Redis.new(config_fallback.params) - - MultiStore.new(primary_store, secondary_store, store_name) - end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index e5e1e1d4165..c990655769c 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -59,16 +59,11 @@ module Gitlab config_file_path("redis.#{store_name.underscore}.yml"), # The current Redis instance may have been split off from another one - # (e.g. TraceChunks was split off from SharedState). There are - # installations out there where the lowest priority config source - # (resque.yml) contains bogus values. In those cases, config_file_name - # should resolve to the instance we originated from (the - # "config_fallback") rather than resque.yml. + # (e.g. TraceChunks was split off from SharedState). config_fallback&.config_file_name, # Global config sources: - ENV['GITLAB_REDIS_CONFIG_FILE'], - config_file_path('resque.yml') + ENV['GITLAB_REDIS_CONFIG_FILE'] ].compact.first end @@ -199,11 +194,17 @@ module Gitlab def fetch_config redis_yml = read_yaml(self.class.redis_yml_path).fetch(@rails_env, {}) instance_config_yml = read_yaml(self.class.config_file_name)[@rails_env] + resque_yml = read_yaml(self.class.config_file_path('resque.yml'))[@rails_env] [ redis_yml[self.class.store_name.underscore], + # There are installations out there where the lowest priority config source (resque.yml) contains bogus + # values. In those cases, the configuration should be read for the instance we originated from (the + # "config_fallback"), either from its specific config file or from redis.yml, before falling back to + # resque.yml. instance_config_yml, - self.class.config_fallback && redis_yml[self.class.config_fallback.store_name.underscore] + self.class.config_fallback && redis_yml[self.class.config_fallback.store_name.underscore], + resque_yml ].compact.first end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 828cf65fb82..e76056709e9 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -120,7 +120,8 @@ module Gitlab @debian_version_regex ||= %r{ \A(?: (?:([0-9]{1,9}):)? (?# epoch) - ([0-9][0-9a-z\.+~]*-?){1,15} (?# version-revision) + ([0-9][0-9a-z\.+~]*) (?# version) + (-[0-9a-z\.+~]+){0,14} (?# -revision) (?<!-) )\z}xi.freeze end @@ -139,6 +140,10 @@ module Gitlab @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}o.freeze end + def debian_direct_upload_filename_regex + @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb)\z}o.freeze + end + def helm_channel_regex @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z}.freeze end @@ -251,24 +256,48 @@ module Gitlab extend self extend Packages - def bulk_import_namespace_path_regex - # This regexp validates the string conforms to rules for a namespace path: - # i.e does not start with a non-alphanueric character except for periods or underscores, + def bulk_import_destination_namespace_path_regex + # This regexp validates the string conforms to rules for a destination_namespace path: + # i.e does not start with a non-alphanumeric character except for periods or underscores, + # contains only alphanumeric characters, forward slashes, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path' + # the regex also allows for an empty string ('') to be accepted as this is allowed in + # a bulk_import POST request + @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z)/i + end + + def bulk_import_source_full_path_regex + # This regexp validates the string conforms to rules for a source_full_path path: + # i.e does not start with a non-alphanumeric character except for periods or underscores, # contains only alphanumeric characters, forward slashes, periods, and underscores, # does not end with a period or forward slash, and has a relative path structure # with no http protocol chars or leading or trailing forward slashes # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' - @bulk_import_namespace_path_regex ||= %r/^([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+$/i + @bulk_import_source_full_path_regex ||= %r/\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z/i + end + + def bulk_import_destination_namespace_path_regex_message + "cannot start with a non-alphanumeric character except for periods or underscores, " \ + "can contain only alphanumeric characters, forward slashes, periods, and underscores, " \ + "cannot end with a period or forward slash, and has a relative path structure " \ + "with no http protocol chars or leading or trailing forward slashes" \ end def group_path_regex # This regexp validates the string conforms to rules for a group slug: - # i.e does not start with a non-alphanueric character except for periods or underscores, + # i.e does not start with a non-alphanumeric character except for periods or underscores, # contains only alphanumeric characters, periods, and underscores, - # does not end with a period or forward slash, and has a relative path structure - # with no http protocol chars or leading or trailing forward slashes - # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' - @group_path_regex ||= %r/^[.]?[^\W]([.]?[0-9a-z][-_]*)+$/i + # does not end with a period or forward slash, and has no leading or trailing forward slashes + # eg 'destination-path' or 'destination_pth' not 'example/com/destination/full/path' + @group_path_regex ||= %r/\A[.]?[^\W]([.]?[0-9a-z][-_]*)+\z/i + end + + def group_path_regex_message + "cannot start with a non-alphanumeric character except for periods or underscores, " \ + "can contain only alphanumeric characters, periods, and underscores, " \ + "cannot end with a period or forward slash, and has no leading or trailing forward slashes" \ end def project_name_regex @@ -406,30 +435,70 @@ module Gitlab }x.freeze end + MARKDOWN_CODE_BLOCK_REGEX = %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `>>>` blocks which are ignored by this filter + # ``` + + ^``` + .+? + \n```\ *$ + ) + }mx.freeze + + MARKDOWN_HTML_BLOCK_REGEX = %r{ + (?<html> + # HTML block: + # <tag> + # Anything, including `>>>` blocks which are ignored by this filter + # </tag> + + ^<[^>]+?>\ *\n + .+? + \n<\/[^>]+?>\ *$ + ) + }mx.freeze + + MARKDOWN_HTML_COMMENT_LINE_REGEX = %r{ + (?<html_comment_line> + # HTML comment line: + # <!-- some commented text --> + + ^<!--\ .*\ -->\ *$ + ) + }mx.freeze + + MARKDOWN_HTML_COMMENT_BLOCK_REGEX = %r{ + (?<html_comment_block> + # HTML comment block: + # <!-- some commented text + # additional text + # --> + + ^<!--.*\n + .+? + \n-->\ *$ + ) + }mx.freeze + def markdown_code_or_html_blocks @markdown_code_or_html_blocks ||= %r{ - (?<code> - # Code blocks: - # ``` - # Anything, including `>>>` blocks which are ignored by this filter - # ``` - - ^``` - .+? - \n```\ *$ - ) + #{MARKDOWN_CODE_BLOCK_REGEX} | - (?<html> - # HTML block: - # <tag> - # Anything, including `>>>` blocks which are ignored by this filter - # </tag> - - ^<[^>]+?>\ *\n - .+? - \n<\/[^>]+?>\ *$ - ) - }mx + #{MARKDOWN_HTML_BLOCK_REGEX} + }mx.freeze + end + + def markdown_code_or_html_comments + @markdown_code_or_html_comments ||= %r{ + #{MARKDOWN_CODE_BLOCK_REGEX} + | + #{MARKDOWN_HTML_COMMENT_LINE_REGEX} + | + #{MARKDOWN_HTML_COMMENT_BLOCK_REGEX} + }mx.freeze end # Based on Jira's project key format @@ -499,15 +568,6 @@ module Gitlab "Must start with a letter, and cannot end with '-' or '_'" end - def saved_reply_name_regex - @saved_reply_name_regex ||= /\A[a-z]([a-z0-9\-_]*[a-z0-9])?\z/.freeze - end - - def saved_reply_name_regex_message - "can contain only lowercase letters, digits, '_' and '-'. " \ - "Must start with a letter, and cannot end with '-' or '_'" - end - # One or more `part`s, separated by separator def sep_by_1(separator, part) %r(#{part} (#{separator} #{part})*)x diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index 8de2c2fe772..498eaf92381 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -50,12 +50,7 @@ module Gitlab end def self.store - if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || - Feature.enabled?(:use_primary_store_as_default_for_repository_cache) - Gitlab::Redis::RepositoryCache.cache_store - else - Rails.cache - end + Gitlab::Redis::RepositoryCache.cache_store end end end diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index ea90a341b1e..1f3c084e194 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -140,12 +140,7 @@ module Gitlab private def cache - if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || - Feature.enabled?(:use_primary_store_as_default_for_repository_cache) - Gitlab::Redis::RepositoryCache - else - Gitlab::Redis::Cache - end + Gitlab::Redis::RepositoryCache end def with(&blk) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index c67ca92af40..838f44c0f9e 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -68,12 +68,7 @@ module Gitlab private def cache - if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || - Feature.enabled?(:use_primary_store_as_default_for_repository_cache) - Gitlab::Redis::RepositoryCache - else - Gitlab::Redis::Cache - end + Gitlab::Redis::RepositoryCache end def with(&blk) diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index fd9fb8ab7e2..1684ecf6ff6 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -47,22 +47,6 @@ module Gitlab self._includers << klass end - def self.add_helpers(mod) - url_helpers.include mod - url_helpers.extend mod - - GitlabRoutingHelper.include mod - GitlabRoutingHelper.extend mod - - app_url_helpers = Gitlab::Application.routes.named_routes.url_helpers_module - app_url_helpers.include mod - app_url_helpers.extend mod - - _includers.each do |klass| - klass.include mod - end - end - # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index 60d3e360984..79d6cfc84a3 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include BlobActiveModel - attr_reader :project, :content_match, :blob_path, :highlight_line + attr_reader :project, :content_match, :blob_path, :highlight_line, :matched_lines_count PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze @@ -25,6 +25,7 @@ module Gitlab @binary_path = opts.fetch(:path, nil) @binary_basename = opts.fetch(:basename, nil) @ref = opts.fetch(:ref, nil) + @matched_lines_count = opts.fetch(:matched_lines_count, nil) @startline = opts.fetch(:startline, nil) @highlight_line = opts.fetch(:highlight_line, nil) @binary_data = opts.fetch(:data, nil) diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 4f7cd340461..b6e2209b475 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -251,16 +251,8 @@ module Gitlab !scheduled? && options[:if_deduplicated] == :reschedule_once end - def with_redis - if Feature.enabled?(:use_primary_and_secondary_stores_for_duplicate_jobs) || - Feature.enabled?(:use_primary_store_as_default_for_duplicate_jobs) - # TODO: Swap for Gitlab::Redis::SharedState after store transition - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 - Gitlab::Redis::DuplicateJobs.with { |redis| yield redis } - else - # Keep the old behavior intact if neither feature flag is turned on - Sidekiq.redis { |redis| yield redis } # rubocop:disable Cop/SidekiqRedisCall - end + def with_redis(&block) + Sidekiq.redis(&block) # rubocop:disable Cop/SidekiqRedisCall end end end diff --git a/lib/gitlab/slash_commands/application_help.rb b/lib/gitlab/slash_commands/application_help.rb index 94abc8b4508..8d747cab2a1 100644 --- a/lib/gitlab/slash_commands/application_help.rb +++ b/lib/gitlab/slash_commands/application_help.rb @@ -17,7 +17,7 @@ module Gitlab private def trigger - "#{params[:command]} [project name or alias]" + params[:command].to_s end def commands diff --git a/lib/gitlab/slash_commands/issue_close.rb b/lib/gitlab/slash_commands/issue_close.rb index 5d33f2fe62d..885c08ce9d5 100644 --- a/lib/gitlab/slash_commands/issue_close.rb +++ b/lib/gitlab/slash_commands/issue_close.rb @@ -29,7 +29,7 @@ module Gitlab private def close_issue(issue:) - ::Issues::CloseService.new(project: project, current_user: current_user).execute(issue) + ::Issues::CloseService.new(container: project, current_user: current_user).execute(issue) end def presenter(issue) diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb index 9f10da247d7..e42cdd0d433 100644 --- a/lib/gitlab/slash_commands/issue_move.rb +++ b/lib/gitlab/slash_commands/issue_move.rb @@ -29,7 +29,7 @@ module Gitlab return Gitlab::SlashCommands::Presenters::Access.new.not_found end - new_issue = ::Issues::MoveService.new(project: project, current_user: current_user) + new_issue = ::Issues::MoveService.new(container: project, current_user: current_user) .execute(old_issue, target_project) presenter(new_issue).present(old_issue) diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 1527fd263e0..508526ac500 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -37,7 +37,7 @@ module Gitlab private def create_issue(title:, description:) - ::Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute + ::Issues::CreateService.new(container: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute end def presenter(issue) diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 61b36308d20..395f7aab050 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -26,7 +26,7 @@ module Gitlab MESSAGE end - if text.start_with?('help') + if text && text.start_with?('help') <<~MESSAGE #{full_commands_message(trigger)} @@ -69,9 +69,9 @@ module Gitlab list = @commands .map do |command| if command < Gitlab::SlashCommands::IncidentManagement::IncidentCommand - "#{@params[:command]} #{command.help_message}" - else "#{trigger} #{command.help_message}" + else + "#{trigger} [project name or alias] #{command.help_message}" end end .join("\n") diff --git a/lib/gitlab/slug/path.rb b/lib/gitlab/slug/path.rb new file mode 100644 index 00000000000..434f36829a6 --- /dev/null +++ b/lib/gitlab/slug/path.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Slug + class Path + LEADING_DASHES = /\A-+/.freeze + # Eextract local email part if given an email. Will remove @ sign and everything following it. + EXTRACT_LOCAL_EMAIL_PART = /@.*\z/.freeze + FORBIDDEN_CHARACTERS = /[^a-zA-Z0-9_\-.]/.freeze + PATH_TRAILING_VIOLATIONS = %w[.git .atom .].freeze + DEFAULT_SLUG = 'blank' + + def initialize(input) + @input = input.dup.to_s + end + + def generate + slug = input.gsub(EXTRACT_LOCAL_EMAIL_PART, "") + slug = slug.gsub(FORBIDDEN_CHARACTERS, "") + + # Remove trailing violations ('.atom', '.git', or '.') + loop do + orig = slug + PATH_TRAILING_VIOLATIONS.each { |extension| slug = slug.chomp extension } + break if orig == slug + end + slug = slug.sub(LEADING_DASHES, "") + + # If all characters were of forbidden nature and filtered out we use this + # fallback to avoid empty paths + slug = DEFAULT_SLUG if slug.blank? + + slug + end + + alias_method :to_s, :generate + + private + + attr_reader :input + end + end +end diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb index 18275da3ef0..8b4c43786f7 100644 --- a/lib/gitlab/sql/set_operator.rb +++ b/lib/gitlab/sql/set_operator.rb @@ -35,9 +35,12 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ApplicationRecord.connection.unprepared_statement do - relations.map do |rel| - remove_order ? rel.reorder(nil).to_sql : rel.to_sql - end.reject(&:blank?) + relations.filter_map do |rel| + next if rel.is_a?(ActiveRecord::NullRelation) + + sql = remove_order ? rel.reorder(nil).to_sql : rel.to_sql + sql.presence + end end if fragments.any? diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index 87861b61119..7094db14c5d 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -7,7 +7,7 @@ module Gitlab # We may want to configure it through project settings in a future version. CUSTOM_DAY_AND_MONTH_LENGTH = { hours_per_day: 8, days_per_month: 20 }.freeze - def parse(string) + def parse(string, keep_zero: false) negative_time = string.start_with?('-') string = string.delete_prefix('-') @@ -15,7 +15,7 @@ module Gitlab begin ChronicDuration.parse( string, - CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours')) + CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours', keep_zero: keep_zero)) rescue StandardError nil end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 00e609511f2..b620e9b4560 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -121,8 +121,8 @@ module Gitlab end rescue SocketError # If the dns rebinding protection is not enabled or the domain - # is allowed we avoid the dns rebinding checks - return if domain_allowed?(uri) || !dns_rebind_protection + # is allowed, or HTTP_PROXY is set we avoid the dns rebinding checks + return if domain_allowed?(uri) || !dns_rebind_protection || Gitlab.http_proxy_env? # In the test suite we use a lot of mocked urls that are either invalid or # don't exist. In order to avoid modifying a ton of tests and factories diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric.rb new file mode 100644 index 00000000000..44a2d739a4a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_internal_pipelines_metric.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiInternalPipelinesMetric < DatabaseMetric + operation :count + + relation do + ::Ci::Pipeline.internal + end + + def value + return FALLBACK if Gitlab.com? + + super + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb new file mode 100644 index 00000000000..e6093691f48 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountIssuesCreatedManuallyFromAlertsMetric < DatabaseMetric + operation :count + + start { Issue.minimum(:id) } + finish { Issue.maximum(:id) } + + cache_start_and_finish_as :issue + + relation do + Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot) + end + + def value + return FALLBACK if Gitlab.com? + + super + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ml_candidates_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ml_candidates_metric.rb new file mode 100644 index 00000000000..e6e547c155c --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ml_candidates_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountMlCandidatesMetric < DatabaseMetric + operation :count + + relation { Ml::Candidate } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ml_experiments_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ml_experiments_metric.rb new file mode 100644 index 00000000000..3e589f28df1 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ml_experiments_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountMlExperimentsMetric < DatabaseMetric + operation :count + + relation { Ml::Experiment } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_candidates_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_candidates_metric.rb new file mode 100644 index 00000000000..bf1f5534a70 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_candidates_metric.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithMlCandidatesMetric < DatabaseMetric + operation :distinct_count, column: 'ml_experiments.project_id' + + relation do + Ml::Experiment.where('EXISTS (?)', + Ml::Candidate.where("\"ml_experiments\".\"id\" = \"ml_candidates\".\"experiment_id\"").select(1)) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_experiments_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_experiments_metric.rb new file mode 100644 index 00000000000..d575163b6b5 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_ml_experiments_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithMlExperimentsMetric < DatabaseMetric + operation :distinct_count, column: :project_id + + relation { Ml::Experiment } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_monitor_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_monitor_enabled_metric.rb new file mode 100644 index 00000000000..2a560233390 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_monitor_enabled_metric.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithMonitorEnabledMetric < DatabaseMetric + operation :count + + metric_options do + { + batch_size: 10_000 + } + end + + relation { ProjectFeature.where.not(monitor_access_level: ProjectFeature::DISABLED) } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_with_ml_candidates_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_users_with_ml_candidates_metric.rb new file mode 100644 index 00000000000..0ec7173e096 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_users_with_ml_candidates_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUsersWithMlCandidatesMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation { Ml::Candidate } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb new file mode 100644 index 00000000000..ab9c6f87023 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IncomingEmailEncryptedSecretsEnabledMetric < GenericMetric + value do + Gitlab::IncomingEmail.encrypted_secrets.active? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/jira_active_integrations_metric.rb b/lib/gitlab/usage/metrics/instrumentations/jira_active_integrations_metric.rb new file mode 100644 index 00000000000..13af3937f43 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/jira_active_integrations_metric.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class JiraActiveIntegrationsMetric < DatabaseMetric + operation :count + + def initialize(metric_definition) + super + + deployment_type = options[:deployment_type] + + return if deployment_type.in?(allowed_types) + + raise ArgumentError, "deployment_type '#{deployment_type}' must be one of: #{allowed_types.join(', ')}" + end + + relation do |options| + ::Integrations::Jira + .active + .joins(:jira_tracker_data) + .where(jira_tracker_data: { deployment_type: options[:deployment_type] }) + end + + private + + def allowed_types + Integrations::JiraTrackerData.deployment_types.keys + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb new file mode 100644 index 00000000000..4332043de8a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ServiceDeskEmailEncryptedSecretsEnabledMetric < GenericMetric + value do + Gitlab::ServiceDeskEmail.encrypted_secrets.active? + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index c105288fff0..53794854bd0 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -65,17 +65,10 @@ module Gitlab # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data - issues_created_manually_from_alerts = if Gitlab.com? - FALLBACK - else - count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) - end - { counts: { assignee_lists: count(List.assignee), ci_builds: count(::Ci::Build), - ci_internal_pipelines: Gitlab.com? ? FALLBACK : count(::Ci::Pipeline.internal), ci_external_pipelines: count(::Ci::Pipeline.external), ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source), ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source), @@ -116,8 +109,6 @@ module Gitlab issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data, issues_created_from_alerts: total_alert_issues, - issues_created_gitlab_alerts: issues_created_manually_from_alerts, - issues_created_manually_from_alerts: issues_created_manually_from_alerts, incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)), alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)), keys: count(Key), @@ -250,8 +241,8 @@ module Gitlab prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? }, prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? }, + web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { false }, signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, - web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } } @@ -598,8 +589,7 @@ module Gitlab { action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, - action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } + action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) } } end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index eb040e9e819..7f6d67e01c7 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -14,14 +14,11 @@ module Gitlab::UsageDataCounters Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: project.id) namespace = project.namespace - if Feature.enabled?(:route_hll_to_snowplow, namespace) - context = Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, - event: event_name).to_context - label = 'redis_hll_counters.ci_templates.ci_templates_total_unique_counts_monthly' - Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, - project: project, context: [context], user: user, - label: label) - end + context = Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: event_name).to_context + label = 'redis_hll_counters.ci_templates.ci_templates_total_unique_counts_monthly' + Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, + project: project, context: [context], user: user, label: label) end def ci_templates(relative_base = 'lib/gitlab/ci/templates') @@ -42,9 +39,9 @@ module Gitlab::UsageDataCounters expanded_template_name = expand_template_name(template_name) results = [expanded_template_name].tap do |result| template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml')) - data = YAML.safe_load(template.content, aliases: true) - [data['include']].compact.flatten.each do |ci_include| - if ci_include_template = ci_include['template'] + data = Gitlab::Ci::Config::Yaml.load!(template.content) + [data[:include]].compact.flatten.each do |ci_include| + if ci_include_template = ci_include[:template] result.concat(all_included_templates(ci_include_template)) end end diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 0b448f68153..2aebc1b8813 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -7,7 +7,6 @@ module Gitlab EDIT_BY_SFE = 'g_edit_by_sfe' EDIT_BY_WEB_IDE = 'g_edit_by_web_ide' EDIT_CATEGORY = 'ide_edit' - EDIT_BY_LIVE_PREVIEW = 'g_edit_by_live_preview' class << self def track_web_ide_edit_action(author:, time: Time.zone.now, project:) @@ -34,15 +33,6 @@ module Gitlab count_unique(EDIT_BY_SNIPPET_EDITOR, date_from, date_to) end - def count_edit_using_editor(date_from:, date_to:) - events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(EDIT_CATEGORY) - count_unique(events, date_from, date_to) - end - - def track_live_preview_edit_action(author:, time: Time.zone.now, project:) - track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time, project) - end - private def track_unique_action(event_name, author, time, project = nil) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 992cec2d174..b809e6c4e42 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -35,7 +35,6 @@ module Gitlab # - name: g_compliance_dashboard # Unique event name # redis_slot: compliance # Optional slot name, if not defined it will use name as a slot, used for totals # category: compliance # Group events in categories - # expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly # aggregation: daily # Aggregation level, keys are stored daily or weekly # feature_flag: # The event feature flag # @@ -203,8 +202,6 @@ module Gitlab end def expiry(event) - return event[:expiry].days if event[:expiry].present? - event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH end diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 477fa288874..a59ea36961d 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -34,6 +34,7 @@ module Gitlab ISSUE_COMMENT_ADDED = 'g_project_management_issue_comment_added' ISSUE_COMMENT_EDITED = 'g_project_management_issue_comment_edited' ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' + ISSUE_DESIGN_COMMENT_REMOVED = 'g_project_management_issue_design_comments_removed' class << self def track_issue_created_action(author:, project:) @@ -171,6 +172,11 @@ module Gitlab track_unique_action(ISSUE_CLONED, author) end + def track_issue_design_comment_removed_action(author:, project:) + track_snowplow_action(ISSUE_DESIGN_COMMENT_REMOVED, author, project) + track_unique_action(ISSUE_DESIGN_COMMENT_REMOVED, author) + end + private def track_snowplow_action(event_name, author, project) diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index b9f143a3a56..b13e3d631c7 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -595,3 +595,11 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_terraform_module_base + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_terraform_module + category: ci_templates + redis_slot: ci_templates + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index a64b7c4032b..ae15530f0d0 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -3,22 +3,18 @@ - name: g_edit_by_web_ide category: ide_edit redis_slot: edit - expiry: 29 aggregation: daily - name: g_edit_by_sfe category: ide_edit redis_slot: edit - expiry: 29 aggregation: daily - name: g_edit_by_snippet_ide category: ide_edit redis_slot: edit - expiry: 29 aggregation: daily - name: g_edit_by_live_preview category: ide_edit redis_slot: edit - expiry: 29 aggregation: daily - name: i_search_total category: search @@ -216,6 +212,10 @@ category: issues_edit redis_slot: project_management aggregation: daily +- name: g_project_management_issue_design_comments_removed + category: issues_edit + redis_slot: project_management + aggregation: daily - name: g_project_management_issue_time_estimate_changed category: issues_edit redis_slot: project_management @@ -240,6 +240,11 @@ category: issues_edit redis_slot: project_management aggregation: daily +# Runner group +- name: g_runner_fleet_read_jobs_statistics + category: runner + redis_slot: runner + aggregation: weekly # Secrets Management - name: i_snippets_show category: snippets @@ -250,7 +255,7 @@ category: terraform redis_slot: terraform aggregation: weekly -# Pipeline Authoring +# Pipeline Authoring group - name: o_pipeline_authoring_unique_users_committing_ciconfigfile category: pipeline_authoring redis_slot: pipeline_authoring @@ -259,6 +264,10 @@ category: pipeline_authoring redis_slot: pipeline_authoring aggregation: weekly +- name: i_ci_secrets_management_id_tokens_build_created + category: ci_secrets_management + redis_slot: ci_secrets_management + aggregation: weekly # Merge request widgets - name: users_expanding_secure_security_report redis_slot: secure @@ -297,7 +306,6 @@ - name: unique_active_user category: manage aggregation: weekly - expiry: 42 # Environments page - name: users_visiting_environments_pages category: environments diff --git a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml new file mode 100644 index 00000000000..e8b14de1769 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml @@ -0,0 +1,22 @@ +--- +- name: i_container_registry_push_tag_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +- name: i_container_registry_delete_tag_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +- name: i_container_registry_push_repository_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +- name: i_container_registry_delete_repository_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +- name: i_container_registry_create_repository_user + category: user_container_registry + aggregation: weekly + redis_slot: container_registry +
\ No newline at end of file diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 10dae35d0bf..c8768164710 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -76,10 +76,10 @@ module Gitlab project: project, namespace: project.namespace, user: user, - property: MR_CREATE_ACTION, - label: 'redis_hll_counters.code_review.i_code_review_create_mr_monthly', + property: MR_USER_CREATE_ACTION, + label: 'redis_hll_counters.code_review.i_code_review_user_create_mr_monthly', context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, - event: MR_CREATE_ACTION).to_context] + event: MR_USER_CREATE_ACTION).to_context] ) end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb index f2753c8f215..904729f114f 100644 --- a/lib/gitlab/usage_data_counters/web_ide_counter.rb +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -27,18 +27,6 @@ module Gitlab count('pipelines') end - def increment_previews_count - return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? - - count('previews') - end - - def increment_previews_success_count - return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? - - count('previews_success') - end - private def redis_key(event) diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 0d15475eebb..3a163e5dde9 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -91,6 +91,14 @@ module Gitlab count(Users::InProductMarketingEmail.where(track: track, series: series).where.not(cta_clicked_at: nil)) end # rubocop: enable CodeReuse/ActiveRecord + + def stage_manage_events(time_period) + # rubocop: disable CodeReuse/ActiveRecord + # rubocop: disable UsageData/LargeTable + estimate_batch_distinct_count(::Event.where(time_period), :author_id) + # rubocop: enable UsageData/LargeTable + # rubocop: enable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 8bd4cd2401d..b92e7dbb725 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -83,7 +83,11 @@ module Gitlab # Append path to host, making sure there's one single / in between def append_path(host, path) - "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" + "#{host.to_s.sub(%r{\/+$}, '')}/#{remove_leading_slashes(path)}" + end + + def remove_leading_slashes(str) + str.to_s.sub(%r{^/+}, '') end # A slugified version of the string, suitable for inclusion in URLs and diff --git a/lib/gitlab/utils/delegator_override.rb b/lib/gitlab/utils/delegator_override.rb index 15ba29d3916..446419378f8 100644 --- a/lib/gitlab/utils/delegator_override.rb +++ b/lib/gitlab/utils/delegator_override.rb @@ -6,7 +6,7 @@ module Gitlab # accidentally override important logic on the fabricated object. module DelegatorOverride def delegator_target(target_class) - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? unless self < ::SimpleDelegator raise ArgumentError, "'#{self}' is not a subclass of 'SimpleDelegator' class." @@ -16,14 +16,14 @@ module Gitlab end def delegator_override(*names) - return unless ENV['STATIC_VERIFICATION'] - raise TypeError unless names.all? { |n| n.is_a?(Symbol) } + return unless Gitlab::Environment.static_verification? + raise TypeError unless names.all?(Symbol) DelegatorOverride.validator(self).add_allowlist(names) end def delegator_override_with(mod) - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? raise TypeError unless mod.is_a?(Module) DelegatorOverride.validator(self).add_allowlist(mod.instance_methods) diff --git a/lib/gitlab/utils/email.rb b/lib/gitlab/utils/email.rb new file mode 100644 index 00000000000..c65d7165263 --- /dev/null +++ b/lib/gitlab/utils/email.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module Email + extend self + + # Replaces most visible characters with * to obfuscate an email address + # deform adds a fix number of * to ensure the address cannot be guessed. Also obfuscates TLD with ** + def obfuscated_email(email, deform: false) + regex = ::Gitlab::UntrustedRegexp.new('^(..?)(.*)(@.?)(.*)(\..+)$') + match = regex.match(email) + return email unless match + + if deform + # Ensure we can show two characters for the username, even if the username has + # only one character. Boring solution is to just duplicate the character. + email_start = match[1] + email_start += email_start if email_start.length == 1 + + email_start + '*' * 5 + match[3] + '*' * 5 + "#{match[5][0..1]}**" + else + match[1] + '*' * (match[2] || '').length + match[3] + '*' * (match[4] || '').length + match[5] + end + end + end + end +end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index f83ebba7c3f..7f43e25e50d 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_dependency 'gitlab/utils' +require_dependency 'gitlab/environment' module Gitlab module Utils @@ -117,7 +118,7 @@ module Gitlab # This would make sure we're overriding something. See: # https://gitlab.com/gitlab-org/gitlab/issues/1819 def override(method_name) - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? Override.extensions[self] ||= Extension.new(self) Override.extensions[self].add_method_name(method_name) @@ -126,7 +127,7 @@ module Gitlab def method_added(method_name) super - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? return unless Override.extensions[self]&.verify_override?(method_name) method_arity = instance_method(method_name).arity @@ -163,7 +164,7 @@ module Gitlab end def queue_verification(base, verify: false) - return unless ENV['STATIC_VERIFICATION'] + return unless Gitlab::Environment.static_verification? # We could check for Class in `override` # This could be `nil` if `override` was never called. diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 2a57ca9ae02..cb7f0a18a88 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -59,9 +59,7 @@ module Gitlab end def presents(*target_classes, as: nil) - if target_classes.any? { |k| k.is_a?(Symbol) } - raise ArgumentError, "Unsupported target class type: #{target_classes}." - end + raise ArgumentError, "Unsupported target class type: #{target_classes}." if target_classes.any?(Symbol) if self < ::Gitlab::View::Presenter::Delegated target_classes.each { |k| delegator_target(k) } diff --git a/lib/initializer_connections.rb b/lib/initializer_connections.rb index c8a6bb6c511..ae2809b7604 100644 --- a/lib/initializer_connections.rb +++ b/lib/initializer_connections.rb @@ -1,29 +1,32 @@ # frozen_string_literal: true module InitializerConnections - # Prevents any database connections within the block - # by using an empty connection handler - # rubocop:disable Database/MultipleDatabases - def self.with_disabled_database_connections + # Raises if new database connections established within the block + # + # NOTE: this does not prevent existing connections that is already checked out + # from being used. You will need other means to prevent that such as by + # clearing all connections as implemented in the + # `:clear_active_connections_again` initializer for routes + # + def self.raise_if_new_database_connection return yield if Gitlab::Utils.to_boolean(ENV['SKIP_RAISE_ON_INITIALIZE_CONNECTIONS']) - original_handler = ActiveRecord::Base.connection_handler - - dummy_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new - ActiveRecord::Base.connection_handler = dummy_handler + previous_connection_counts = ActiveRecord::Base.connection_handler.connection_pool_list.to_h do |pool| + [pool.db_config.name, pool.connections.size] + end yield - if dummy_handler&.connection_pool_names&.present? - raise "Unxpected connection_pools (#{dummy_handler.connection_pool_names}) ! Call `connects_to` before this block" + new_connection_counts = ActiveRecord::Base.connection_handler.connection_pool_list.to_h do |pool| + [pool.db_config.name, pool.connections.size] end - rescue ActiveRecord::ConnectionNotEstablished + + raise_database_connection_made_error unless previous_connection_counts == new_connection_counts + end + + def self.raise_database_connection_made_error message = "Database connection should not be called during initializers. Read more at https://docs.gitlab.com/ee/development/rails_initializers.html#database-connections-in-initializers" raise message - ensure - ActiveRecord::Base.connection_handler = original_handler - dummy_handler&.clear_all_connections! end - # rubocop:enable Database/MultipleDatabases end diff --git a/lib/kramdown/converter/commonmark.rb b/lib/kramdown/converter/commonmark.rb index a903d541d81..0055558989d 100644 --- a/lib/kramdown/converter/commonmark.rb +++ b/lib/kramdown/converter/commonmark.rb @@ -34,7 +34,7 @@ module Kramdown def convert_codeblock(el, _opts) # Although tildes are supported in CommonMark, backticks are more common "```#{el.options[:lang]}\n" + - el.value.split(/\n/).map { |l| l.empty? ? "" : "#{l}" }.join("\n") + + el.value.split("\n").map { |l| l.empty? ? "" : "#{l}" }.join("\n") + "\n```\n\n" end diff --git a/lib/kramdown/parser/atlassian_document_format.rb b/lib/kramdown/parser/atlassian_document_format.rb index 4ceb879a04c..d27697a59a6 100644 --- a/lib/kramdown/parser/atlassian_document_format.rb +++ b/lib/kramdown/parser/atlassian_document_format.rb @@ -219,7 +219,7 @@ module Kramdown # opportunity to replace it later. Mention name can have # spaces, so double quote it mention_text = ast_node.dig('attrs', 'text')&.gsub('@', '') - mention_text = %Q("#{mention_text}") if mention_text.match?(/ /) + mention_text = %Q("#{mention_text}") if mention_text&.include?(' ') mention_text = %Q(@adf-mention:#{mention_text}) add_text(mention_text, element, :text) diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb index 1301c6aa6fd..c37c6bb8561 100644 --- a/lib/peek/views/detailed_view.rb +++ b/lib/peek/views/detailed_view.rb @@ -33,7 +33,7 @@ module Peek def details call_details .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) + .map { |call| format_call_details(call) } end def warnings diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb index 05a2d3bd0c9..4d38e9513fb 100644 --- a/lib/prometheus/pid_provider.rb +++ b/lib/prometheus/pid_provider.rb @@ -27,7 +27,7 @@ module Prometheus def puma_worker_id if matches = process_name.match(/puma.*cluster worker ([0-9]+):/) "puma_#{matches[1]}" - elsif process_name =~ /puma/ + elsif process_name.include?('puma') "puma_master" else unknown_process_id diff --git a/lib/quality/seeders/issues.rb b/lib/quality/seeders/issues.rb index bf42913e043..cac034767f6 100644 --- a/lib/quality/seeders/issues.rb +++ b/lib/quality/seeders/issues.rb @@ -31,7 +31,7 @@ module Quality } params[:closed_at] = params[:created_at] + rand(35).days if params[:state] == 'closed' - create_result = ::Issues::CreateService.new(project: project, current_user: team.sample, params: params, spam_params: nil).execute_without_rate_limiting + create_result = ::Issues::CreateService.new(container: project, current_user: team.sample, params: params, spam_params: nil).execute_without_rate_limiting if create_result.success? created_issues_count += 1 diff --git a/lib/release_highlights/validator/entry.rb b/lib/release_highlights/validator/entry.rb index 2b4889e0131..fc9ce3a3fd0 100644 --- a/lib/release_highlights/validator/entry.rb +++ b/lib/release_highlights/validator/entry.rb @@ -6,12 +6,16 @@ module ReleaseHighlights include ActiveModel::Validations::Callbacks AVAILABLE_IN = %w(Free Premium Ultimate).freeze + HYPHENATED_ATTRIBUTES = [:self_managed, :gitlab_com].freeze attr_reader :entry + attr_accessor :available_in, :description, :gitlab_com, :image_url, :name, :published_at, :release, :self_managed, + :stage validates :name, :description, :stage, presence: true - validates :'self-managed', :'gitlab-com', inclusion: { in: [true, false], message: "must be a boolean" } - validates :documentation_link, :image_url, public_url: { dns_rebind_protection: true } + validates :self_managed, :gitlab_com, inclusion: { in: [true, false], message: "must be a boolean" } + validates :documentation_link, public_url: { dns_rebind_protection: true } + validates :image_url, public_url: { dns_rebind_protection: true }, allow_nil: true validates :release, numericality: true validate :validate_published_at validate :validate_available_in @@ -67,11 +71,13 @@ module ReleaseHighlights index = entry.children.find_index(node) next_node = entry.children[index + 1] + next_node&.to_ruby end def find_node(key) - entry.children.find { |node| node.try(:value) == key.to_s } + formatted_key = key.in?(HYPHENATED_ATTRIBUTES) ? key.to_s.dasherize : key.to_s + entry.children.find { |node| node.try(:value) == formatted_key } end end end diff --git a/lib/sidebars/menu_item.rb b/lib/sidebars/menu_item.rb index 7ed3e47ae34..efdedf6c3bd 100644 --- a/lib/sidebars/menu_item.rb +++ b/lib/sidebars/menu_item.rb @@ -4,9 +4,11 @@ module Sidebars class MenuItem include ::Sidebars::Concerns::LinkWithHtmlOptions - attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :hint_html_options + attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :hint_html_options, :has_pill, :pill_count + alias_method :has_pill?, :has_pill - def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, hint_html_options: {}) + # rubocop: disable Metrics/ParameterLists + def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, hint_html_options: {}, has_pill: false, pill_count: nil) @title = title @link = link @active_routes = active_routes @@ -15,7 +17,10 @@ module Sidebars @sprite_icon = sprite_icon @sprite_icon_html_options = sprite_icon_html_options @hint_html_options = hint_html_options + @has_pill = has_pill + @pill_count = pill_count end + # rubocop: enable Metrics/ParameterLists def show_hint? hint_html_options.present? diff --git a/lib/sidebars/projects/menus/learn_gitlab_menu.rb b/lib/sidebars/projects/menus/learn_gitlab_menu.rb deleted file mode 100644 index b6fae2af93d..00000000000 --- a/lib/sidebars/projects/menus/learn_gitlab_menu.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Sidebars - module Projects - module Menus - class LearnGitlabMenu < ::Sidebars::Menu - include Gitlab::Utils::StrongMemoize - - override :link - def link - project_learn_gitlab_path(context.project) - end - - override :active_routes - def active_routes - { controller: :learn_gitlab } - end - - override :title - def title - _('Learn GitLab') - end - - override :has_pill? - def has_pill? - context.learn_gitlab_enabled - end - - override :pill_count - def pill_count - strong_memoize(:pill_count) do - percentage = Onboarding::Completion.new( - context.project.namespace, - context.current_user - ).percentage - - "#{percentage}%" - end - end - - override :extra_nav_link_html_options - def extra_nav_link_html_options - { - class: 'home', - data: { - track_label: 'learn_gitlab' - } - } - end - - override :sprite_icon - def sprite_icon - 'bulb' - end - - override :render? - def render? - context.learn_gitlab_enabled - end - end - end - end -end diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index fea71e4aefd..7d1fa8b8fa7 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -12,6 +12,7 @@ module Sidebars add_item(error_tracking_menu_item) add_item(alert_management_menu_item) add_item(incidents_menu_item) + add_item(airflow_dashboard_menu_item) true end @@ -96,6 +97,20 @@ module Sidebars item_id: :incidents ) end + + def airflow_dashboard_menu_item + unless can?(context.current_user, :read_airflow_dags, context.project) && + Feature.enabled?(:airflow_dags, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :airflow) + end + + ::Sidebars::MenuItem.new( + title: _('Airflow'), + link: project_airflow_dags_path(context.project), + active_routes: { path: 'airflow/dags#show' }, + item_id: :airflow_dags + ) + end end end end diff --git a/lib/sidebars/projects/menus/repository_menu.rb b/lib/sidebars/projects/menus/repository_menu.rb index 735be5a5133..ec91ae741fe 100644 --- a/lib/sidebars/projects/menus/repository_menu.rb +++ b/lib/sidebars/projects/menus/repository_menu.rb @@ -56,11 +56,7 @@ module Sidebars end def commits_menu_item - link = if Feature.enabled?(:use_ref_type_parameter, context.project) - project_commits_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) - else - project_commits_path(context.project, context.current_ref) - end + link = project_commits_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) ::Sidebars::MenuItem.new( title: _('Commits'), @@ -93,11 +89,7 @@ module Sidebars def contributors_menu_item return false unless context.project.analytics_enabled? - link = if Feature.enabled?(:use_ref_type_parameter, context.project) - project_graph_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) - else - project_graph_path(context.project, context.current_ref) - end + link = project_graph_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) ::Sidebars::MenuItem.new( title: _('Contributors'), @@ -108,11 +100,7 @@ module Sidebars end def graphs_menu_item - link = if Feature.enabled?(:use_ref_type_parameter, context.project) - project_network_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) - else - project_network_path(context.project, context.current_ref) - end + link = project_network_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) ::Sidebars::MenuItem.new( title: _('Graph'), diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb index 8ae8f931aab..9d0f5eb87bd 100644 --- a/lib/sidebars/projects/panel.rb +++ b/lib/sidebars/projects/panel.rb @@ -19,7 +19,6 @@ module Sidebars def add_menus add_menu(Sidebars::Projects::Menus::ProjectInformationMenu.new(context)) - add_menu(Sidebars::Projects::Menus::LearnGitlabMenu.new(context)) add_menu(Sidebars::Projects::Menus::RepositoryMenu.new(context)) add_menu(Sidebars::Projects::Menus::IssuesMenu.new(context)) add_menu(Sidebars::Projects::Menus::ExternalIssueTrackerMenu.new(context)) diff --git a/lib/sidebars/your_work/menus/merge_requests_menu.rb b/lib/sidebars/your_work/menus/merge_requests_menu.rb index 695c2ffdf46..09c84666258 100644 --- a/lib/sidebars/your_work/menus/merge_requests_menu.rb +++ b/lib/sidebars/your_work/menus/merge_requests_menu.rb @@ -4,7 +4,8 @@ module Sidebars module YourWork module Menus class MergeRequestsMenu < ::Sidebars::Menu - include Gitlab::Utils::StrongMemoize + include IssuablesHelper + include MergeRequestsHelper override :link def link @@ -21,6 +22,14 @@ module Sidebars 'merge-request' end + override :configure_menu_items + def configure_menu_items + add_item(assigned_mrs_menu_item) + add_item(reviewer_mrs_menu_item) + + true + end + override :render? def render? !!context.current_user @@ -38,9 +47,36 @@ module Sidebars override :pill_count def pill_count - context.current_user.assigned_open_merge_requests_count + user_merge_requests_counts[:total] + end + + private + + def assigned_mrs_menu_item + link = merge_requests_dashboard_path(assignee_username: context.current_user.username) + + ::Sidebars::MenuItem.new( + title: _('Assigned'), + link: link, + active_routes: { page: link }, + has_pill: true, + pill_count: user_merge_requests_counts[:assigned], + item_id: :merge_requests_assigned + ) + end + + def reviewer_mrs_menu_item + link = merge_requests_dashboard_path(reviewer_username: context.current_user.username) + + ::Sidebars::MenuItem.new( + title: _('Review requests'), + link: link, + active_routes: { page: link }, + has_pill: true, + pill_count: user_merge_requests_counts[:review_requested], + item_id: :merge_requests_to_review + ) end - strong_memoize_attr :pill_count end end end diff --git a/lib/slack_markdown_sanitizer.rb b/lib/slack_markdown_sanitizer.rb new file mode 100644 index 00000000000..df3bec1a3c8 --- /dev/null +++ b/lib/slack_markdown_sanitizer.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module SlackMarkdownSanitizer + # Markup characters which are used for links in HTML, Markdown, + # and Slack "mrkdwn" syntax (`<http://example.com|Label>`). + UNSAFE_MARKUP_CHARACTERS = '<>[]|' + + def self.sanitize(string) + string&.delete(UNSAFE_MARKUP_CHARACTERS) + end +end diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 084e7c78906..47b356aa6e6 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -1,10 +1,10 @@ # frozen_string_literal: true -require 'logger' -require 'resolv-replace' - desc 'GitLab | Artifacts | Migrate files for artifacts to comply with new storage format' namespace :gitlab do + require 'logger' + require 'resolv-replace' + namespace :artifacts do task migrate: :environment do logger = Logger.new($stdout) diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 6647a10898f..06a032316c5 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'active_record/fixtures' - namespace :gitlab do + require 'active_record/fixtures' + namespace :backup do PID = Process.pid.freeze PID_FILE = "#{Rails.application.root}/tmp/backup_restore.pid" @@ -44,15 +44,13 @@ namespace :gitlab do namespace :db do task create: :gitlab_environment do lock do - Backup::Manager.new(progress).run_create_task('main_db') - Backup::Manager.new(progress).run_create_task('ci_db') + Backup::Manager.new(progress).run_create_task('db') end end task restore: :gitlab_environment do lock do - Backup::Manager.new(progress).run_restore_task('main_db') - Backup::Manager.new(progress).run_restore_task('ci_db') + Backup::Manager.new(progress).run_restore_task('db') end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 49d2d9fed03..1753483b091 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,7 +1,8 @@ # frozen_string_literal: true -require 'set' namespace :gitlab do + require 'set' + namespace :cleanup do desc "GitLab | Cleanup | Block users that have been removed in LDAP" task block_removed_ldap_users: :gitlab_environment do diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 9c92aa5eb28..c4dc7b938cc 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -283,6 +283,36 @@ namespace :gitlab do end end + namespace :execute_async_index_operations do + each_database(databases) do |database_name| + task database_name, [:pick] => :environment do |_, args| + args.with_defaults(pick: 2) + + if Feature.disabled?(:database_async_index_operations, type: :ops) + puts <<~NOTE.color(:yellow) + Note: database async index operations feature is currently disabled. + + Enable with: Feature.enable(:database_async_index_operations) + NOTE + exit + end + + Gitlab::Database::EachDatabase.each_database_connection(only: database_name) do + Gitlab::Database::AsyncIndexes.execute_pending_actions!(how_many: args[:pick].to_i) + end + end + end + + task :all, [:pick] => :environment do |_, args| + default_pick = Gitlab.dev_or_test_env? ? 1000 : 2 + args.with_defaults(pick: default_pick) + + each_database(databases) do |database_name| + Rake::Task["gitlab:db:execute_async_index_operations:#{database_name}"].invoke(args[:pick]) + end + end + end + desc 'Check if there have been user additions to the database' task active: :environment do if ActiveRecord::Base.connection.migration_context.needs_migration? @@ -306,7 +336,7 @@ namespace :gitlab do namespace :migration_testing do # Not possible to import Gitlab::Database::DATABASE_NAMES here # Specs verify that a task exists for each entry in that array. - all_databases = %i[main ci] + all_databases = %i[main ci main_clusterwide] task up: :environment do Gitlab::Database::Migrations::Runner.up(database: 'main', legacy_mode: true).run @@ -399,64 +429,73 @@ namespace :gitlab do namespace :dictionary do DB_DOCS_PATH = File.join(Rails.root, 'db', 'docs') + EE_DICTIONARY_PATH = File.join(Rails.root, 'ee', 'db', 'docs') desc 'Generate database docs yaml' task generate: :environment do - FileUtils.mkdir_p(DB_DOCS_PATH) unless Dir.exist?(DB_DOCS_PATH) + FileUtils.mkdir_p(DB_DOCS_PATH) + FileUtils.mkdir_p(EE_DICTIONARY_PATH) if Gitlab.ee? Rails.application.eager_load! - tables = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.tables } + version = Gem::Version.new(File.read('VERSION')) + milestone = version.release.segments[0..1].join('.') - views = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.views } + classes = {} - sources = tables + views + Gitlab::Database.database_base_models.each do |_, model_class| + tables = model_class.connection.tables - classes = sources.index_with { [] } + views = model_class.connection.views + + sources = tables + views + + model_classes = sources.index_with { [] } + + classes.merge!(model_classes) { |_, sources, new_sources| sources + new_sources } - Gitlab::Database.database_base_models.each do |_, model_class| model_class .descendants .reject(&:abstract_class) .reject { |c| c.name =~ /^(?:EE::)?Gitlab::(?:BackgroundMigration|DatabaseImporters)::/ } .reject { |c| c.name =~ /^HABTM_/ } .each { |c| classes[c.table_name] << c.name if classes.has_key?(c.table_name) } - end - version = Gem::Version.new(File.read('VERSION')) - milestone = version.release.segments[0..1].join('.') + sources.each do |source_name| + next if source_name.start_with?('_test_') # Ignore test tables - sources.each do |source_name| - file = dictionary_file_path(source_name, views) - key_name = "#{data_source_type(source_name, views)}_name" + database = model_class.connection_db_config.name + file = dictionary_file_path(source_name, views, database) + key_name = "#{data_source_type(source_name, views)}_name" - table_metadata = { - key_name => source_name, - 'classes' => classes[source_name]&.sort&.uniq, - 'feature_categories' => [], - 'description' => nil, - 'introduced_by_url' => nil, - 'milestone' => milestone - } + table_metadata = { + key_name => source_name, + 'classes' => classes[source_name]&.sort&.uniq, + 'feature_categories' => [], + 'description' => nil, + 'introduced_by_url' => nil, + 'milestone' => milestone + } - if File.exist?(file) - outdated = false + if File.exist?(file) + outdated = false - existing_metadata = YAML.safe_load(File.read(file)) + existing_metadata = YAML.safe_load(File.read(file)) - if existing_metadata[key_name] != table_metadata[key_name] - existing_metadata[key_name] = table_metadata[key_name] - outdated = true - end + if existing_metadata[key_name] != table_metadata[key_name] + existing_metadata[key_name] = table_metadata[key_name] + outdated = true + end - if existing_metadata['classes'].sort != table_metadata['classes'].sort - existing_metadata['classes'] = table_metadata['classes'] - outdated = true - end + if existing_metadata['classes'] && existing_metadata['classes'].sort != table_metadata['classes'].sort + existing_metadata['classes'] = table_metadata['classes'] + outdated = true + end - File.write(file, existing_metadata.to_yaml) if outdated - else - File.write(file, table_metadata.to_yaml) + File.write(file, existing_metadata.to_yaml) if outdated + else + File.write(file, table_metadata.to_yaml) + end end end end @@ -469,16 +508,17 @@ namespace :gitlab do 'table' end - def dictionary_file_path(source_name, views) + def dictionary_file_path(source_name, views, database) sub_directory = views.include?(source_name) ? 'views' : '' - File.join(DB_DOCS_PATH, sub_directory, "#{source_name}.yml") + path = database == 'geo' ? EE_DICTIONARY_PATH : DB_DOCS_PATH + + File.join(path, sub_directory, "#{source_name}.yml") end - # Temporary disable this, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85760#note_998452069 - # Rake::Task['db:migrate'].enhance do - # Rake::Task['gitlab:db:dictionary:generate'].invoke if Rails.env.development? - # end + Rake::Task['db:migrate'].enhance do + Rake::Task['gitlab:db:dictionary:generate'].invoke if Rails.env.development? + end end end end diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake index 212d60a7231..59478d47d12 100644 --- a/lib/tasks/gitlab/db/lock_writes.rake +++ b/lib/tasks/gitlab/db/lock_writes.rake @@ -4,49 +4,22 @@ namespace :gitlab do namespace :db do desc "GitLab | DB | Install prevent write triggers on all databases" task lock_writes: [:environment, 'gitlab:db:validate_config'] do - Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name| - schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) - - Gitlab::Database::LockWritesManager.tables_to_lock(connection) do |table_name, schema_name| - # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 - next if schema_name == :gitlab_geo - - lock_writes_manager = Gitlab::Database::LockWritesManager.new( - table_name: table_name, - connection: connection, - database_name: database_name, - with_retries: true, - logger: Logger.new($stdout), - dry_run: ENV['DRY_RUN'] == 'true' - ) - - if schemas_for_connection.include?(schema_name.to_sym) - lock_writes_manager.unlock_writes - else - lock_writes_manager.lock_writes - end - end - end + logger = Logger.new($stdout) + logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::INFO : Logger::WARN + Gitlab::Database::TablesLocker.new( + logger: logger, + dry_run: Gitlab::Utils.to_boolean(ENV['DRY_RUN'], default: false) + ).lock_writes end desc "GitLab | DB | Remove all triggers that prevents writes from all databases" task unlock_writes: :environment do - Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| - Gitlab::Database::LockWritesManager.tables_to_lock(connection) do |table_name, schema_name| - # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 - next if schema_name == :gitlab_geo - - lock_writes_manager = Gitlab::Database::LockWritesManager.new( - table_name: table_name, - connection: connection, - database_name: database_name, - with_retries: true, - logger: Logger.new($stdout) - ) - - lock_writes_manager.unlock_writes - end - end + logger = Logger.new($stdout) + logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::INFO : Logger::WARN + Gitlab::Database::TablesLocker.new( + logger: logger, + dry_run: Gitlab::Utils.to_boolean(ENV['DRY_RUN'], default: false) + ).unlock_writes end end end diff --git a/lib/tasks/gitlab/dependency_proxy/migrate.rake b/lib/tasks/gitlab/dependency_proxy/migrate.rake index fa35eacc59d..7a4b8800c34 100644 --- a/lib/tasks/gitlab/dependency_proxy/migrate.rake +++ b/lib/tasks/gitlab/dependency_proxy/migrate.rake @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'logger' - desc "GitLab | Dependency Proxy | Migrate dependency proxy files to remote storage" namespace :gitlab do namespace :dependency_proxy do diff --git a/lib/tasks/gitlab/incoming_email.rake b/lib/tasks/gitlab/incoming_email.rake new file mode 100644 index 00000000000..eaceeb91ad0 --- /dev/null +++ b/lib/tasks/gitlab/incoming_email.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :incoming_email do + namespace :secret do + desc 'GitLab | Incoming Email | Secret | Write Incoming Email secrets' + task write: [:environment] do + content = $stdin.tty? ? $stdin.gets : $stdin.read + Gitlab::EncryptedIncomingEmailCommand.write(content) + end + + desc 'GitLab | Incoming Email | Secret | Edit Incoming Email secrets' + task edit: [:environment] do + Gitlab::EncryptedIncomingEmailCommand.edit + end + + desc 'GitLab | Incoming Email | Secret | Show Incoming Email secrets' + task show: [:environment] do + Gitlab::EncryptedIncomingEmailCommand.show + end + end + end +end diff --git a/lib/tasks/gitlab/service_desk_email.rake b/lib/tasks/gitlab/service_desk_email.rake new file mode 100644 index 00000000000..39c9d40bf5c --- /dev/null +++ b/lib/tasks/gitlab/service_desk_email.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :service_desk_email do + namespace :secret do + desc 'GitLab | Service Desk Email | Secret | Write Service Desk Email secrets' + task write: [:environment] do + content = $stdin.tty? ? $stdin.gets : $stdin.read + Gitlab::EncryptedServiceDeskEmailCommand.write(content) + end + + desc 'GitLab | Service Desk Email | Secret | Edit Service Desk Email secrets' + task edit: [:environment] do + Gitlab::EncryptedServiceDeskEmailCommand.edit + end + + desc 'GitLab | Service Desk Email | Secret | Show Service Desk Email secrets' + task show: [:environment] do + Gitlab::EncryptedServiceDeskEmailCommand.show + end + end + end +end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index b3559bde988..77f5eb87725 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -14,7 +14,7 @@ namespace :tw do end def directory - @directory ||= File.dirname(path) + @directory ||= "#{File.dirname(path)}/" end end @@ -26,9 +26,10 @@ namespace :tw do CodeOwnerRule.new('Certify', '@msedlakjakubowski'), CodeOwnerRule.new('Code Review', '@aqualls'), CodeOwnerRule.new('Compliance', '@eread'), + CodeOwnerRule.new('Commerce Integrations', '@drcatherinepope'), CodeOwnerRule.new('Composition Analysis', '@rdickenson'), CodeOwnerRule.new('Configure', '@phillipwells'), - CodeOwnerRule.new('Container Registry', '@claytoncornell'), + CodeOwnerRule.new('Container Registry', '@dianalogan'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Conversion', '@kpaizee'), CodeOwnerRule.new('Database', '@aqualls'), @@ -50,15 +51,15 @@ namespace :tw do CodeOwnerRule.new('Knowledge', '@aqualls'), CodeOwnerRule.new('Application Performance', '@jglassman1'), CodeOwnerRule.new('Monitor', '@msedlakjakubowski'), - CodeOwnerRule.new('Observability', '@msedlakjakubowski'), + CodeOwnerRule.new('Observability', '@drcatherinepope'), CodeOwnerRule.new('Optimize', '@lciutacu'), - CodeOwnerRule.new('Package Registry', '@claytoncornell'), + CodeOwnerRule.new('Package Registry', '@dianalogan'), CodeOwnerRule.new('Pipeline Authoring', '@marcel.amirault'), - CodeOwnerRule.new('Pipeline Execution', '@marcel.amirault'), + CodeOwnerRule.new('Pipeline Execution', '@drcatherinepope'), CodeOwnerRule.new('Pipeline Insights', '@marcel.amirault'), CodeOwnerRule.new('Portfolio Management', '@msedlakjakubowski'), CodeOwnerRule.new('Product Analytics', '@lciutacu'), - CodeOwnerRule.new('Product Intelligence', '@claytoncornell'), + CodeOwnerRule.new('Product Intelligence', '@dianalogan'), CodeOwnerRule.new('Product Planning', '@msedlakjakubowski'), CodeOwnerRule.new('Project Management', '@msedlakjakubowski'), CodeOwnerRule.new('Provision', '@fneill'), @@ -69,15 +70,15 @@ namespace :tw do CodeOwnerRule.new('Runner', '@fneill'), CodeOwnerRule.new('Runner SaaS', '@fneill'), CodeOwnerRule.new('Pods', '@jglassman1'), - CodeOwnerRule.new('Security Policies', '@claytoncornell'), + CodeOwnerRule.new('Security Policies', '@dianalogan'), CodeOwnerRule.new('Source Code', '@aqualls'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Testing', '@eread'), - CodeOwnerRule.new('Threat Insights', '@claytoncornell'), + CodeOwnerRule.new('Threat Insights', '@dianalogan'), CodeOwnerRule.new('Tutorials', '@kpaizee'), CodeOwnerRule.new('Utilization', '@fneill'), - CodeOwnerRule.new('Vulnerability Research', '@claytoncornell'), + CodeOwnerRule.new('Vulnerability Research', '@dianalogan'), CodeOwnerRule.new('Organization', '@lciutacu') ].freeze @@ -122,16 +123,20 @@ namespace :tw do mappings << DocumentOwnerMapping.new(relative_file, writer) if document.has_a_valid_group? end - deduplicated_mappings = Set.new - - mappings.each do |mapping| + transformed_mappings = mappings.map do |mapping| if mapping.writer_owns_directory?(mappings) - deduplicated_mappings.add("#{mapping.directory}/ #{mapping.writer}") + DocumentOwnerMapping.new(mapping.directory, mapping.writer) else - deduplicated_mappings.add("#{mapping.path} #{mapping.writer}") + DocumentOwnerMapping.new(mapping.path, mapping.writer) end end + deduplicated_mappings = Set.new + + transformed_mappings + .reject { |mapping| transformed_mappings.any? { |m| m.path == mapping.directory && m.writer == mapping.writer } } + .each { |mapping| deduplicated_mappings.add("#{mapping.path} #{mapping.writer}") } + new_docs_owners = deduplicated_mappings.sort.join("\n") codeowners_path = Rails.root.join('.gitlab/CODEOWNERS') diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index c93be95e2e0..d2056338350 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -4,8 +4,8 @@ require 'benchmark' require 'rainbow/ext/string' class GithubImport - def self.run!(*args) - new(*args).run! + def self.run!(...) + new(...).run! end def initialize(token, gitlab_username, project_path, extras) @@ -35,8 +35,8 @@ class GithubImport private def show_warning! - puts "This will import GitHub #{@repo.full_name.bright} into GitLab #{@project_path.bright} as #{@current_user.name}" - puts "Permission checks are ignored. Press any key to continue.".color(:red) + puts "This will import GitHub #{@repo[:full_name].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" + puts 'Permission checks are ignored. Press any key to continue.'.color(:red) $stdin.getch @@ -57,43 +57,34 @@ class GithubImport end def new_project - Project.transaction do - namespace_path, _sep, name = @project_path.rpartition('/') - namespace = find_or_create_namespace(namespace_path) + namespace_path, _sep, project_name = @project_path.rpartition('/') + target_namespace = Namespace.find_by_full_path(namespace_path) + + raise 'Namespace or group to import repository into does not exist.' unless target_namespace + Project.transaction do project = Projects::CreateService.new( @current_user, - name: name, - path: name, - description: @repo.description, - namespace_id: namespace.id, + name: project_name, + path: project_name, + description: @repo[:description], + namespace_id: target_namespace.id, visibility_level: visibility_level, - skip_wiki: @repo.has_wiki + skip_wiki: @repo[:has_wiki] ).execute project.update!( import_type: 'github', - import_source: @repo.full_name, - import_url: @repo.clone_url.sub('://', "://#{@options[:token]}@") + import_source: @repo[:full_name], + import_url: @repo[:clone_url].sub('://', "://#{@options[:token]}@") ) project end end - def find_or_create_namespace(names) - return @current_user.namespace if names == @current_user.namespace_path - return @current_user.namespace unless @current_user.can_create_group? - - Groups::NestedCreateService.new(@current_user, group_path: names).execute - end - - def full_path_namespace(names) - @full_path_namespace ||= Namespace.find_by_full_path(names) - end - def visibility_level - @repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility + @repo[:private] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility end end @@ -110,17 +101,17 @@ class GithubRepos return found_github_repo if @github_repo repos.each do |repo| - print "ID: #{repo.id.to_s.bright}".color(:green) - print "\tName: #{repo.full_name}\n".color(:green) + print "ID: #{repo[:id].to_s.bright}".color(:green) + print "\tName: #{repo[:full_name]}\n".color(:green) end print 'ID? '.bright - repos.find { |repo| repo.id == repo_id } + repos.find { |repo| repo[:id] == repo_id } end def found_github_repo - repos.find { |repo| repo.full_name == @github_repo } + repos.find { |repo| repo[:full_name] == @github_repo } end def repo_id @@ -128,7 +119,7 @@ class GithubRepos end def repos - @client.octokit.list_repositories + @repos ||= @client.repos end end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 62d31803f6e..6e50e417776 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -34,6 +34,11 @@ unless Rails.env.production? exit(1) end + desc "GitLab | Lint | Lint docs Markdown files" + task :markdown do + sh "./scripts/lint-doc.sh" + end + desc "GitLab | Lint | Run several lint checks" task :all do status = 0 diff --git a/lib/tasks/migrate/schema_check.rake b/lib/tasks/migrate/schema_check.rake index 76f1f23c7bd..e15304afcff 100644 --- a/lib/tasks/migrate/schema_check.rake +++ b/lib/tasks/migrate/schema_check.rake @@ -6,10 +6,14 @@ task schema_version_check: :environment do schema_version = ActiveRecord::Migrator.current_version + minimum_migration_version = Gitlab::Database.read_minimum_migration_version + + raise 'Unable to find any migration files in db/migrate.' if minimum_migration_version.nil? + # Ensure migrations are being run from a supported schema version # A schema verison of 0 is a fresh db, and should be safe to run migrations # But a database with existing migrations less than our min version is not - if schema_version > 0 && schema_version < Gitlab::Database::MIN_SCHEMA_VERSION + if schema_version > 0 && schema_version < minimum_migration_version raise "Your current database version is too old to be migrated. " \ "You should upgrade to GitLab #{Gitlab::Database::MIN_SCHEMA_GITLAB_VERSION} before moving to this version. " \ "Please see https://docs.gitlab.com/ee/policy/maintenance.html#upgrade-recommendations" diff --git a/lib/unnested_in_filters/rewriter.rb b/lib/unnested_in_filters/rewriter.rb index ed1e4ce2d9f..9eb1c0b8273 100644 --- a/lib/unnested_in_filters/rewriter.rb +++ b/lib/unnested_in_filters/rewriter.rb @@ -45,7 +45,7 @@ module UnnestedInFilters def serialized_array_values values.map(&:value) - .then { array_type.serialize(_1) } + .then { |value| array_type.serialize(value) } .then { |array| quote(array) } end @@ -58,7 +58,7 @@ module UnnestedInFilters end def column - columns.find { _1.name == attribute } + columns.find { |column| column.name == attribute } end def unprepared_statement_collector @@ -180,12 +180,12 @@ module UnnestedInFilters end def filter_query - model.from(from).then { add_relation_defaults(_1) } + model.from(from).then { |relation| add_relation_defaults(relation) } end def index_only_filter_query model.where(model.primary_key => filter_query.select(model.primary_key)) - .then { add_relation_defaults(_1) } + .then { |relation| add_relation_defaults(relation) } end def add_relation_defaults(new_relation) @@ -229,7 +229,9 @@ module UnnestedInFilters # Actively filter any nodes that don't belong to the primary queried table to prevent sql type resolution issues # Context: https://gitlab.com/gitlab-org/gitlab/-/issues/370271#note_1151019824 def arel_in_nodes - where_clause_arel_nodes.select(&method(:in_predicate?)).select { model_column_names.include?(_1.left.name) } + where_clause_arel_nodes + .select { |arel_node| in_predicate?(arel_node) } + .select { |arel_node| model_column_names.include?(arel_node.left.name) } end # `ActiveRecord::WhereClause#ast` is returning a single node when there is only one @@ -270,7 +272,7 @@ module UnnestedInFilters end def order_attributes - @order_attributes ||= order_values.flat_map(&method(:extract_column_name)) + @order_attributes ||= order_values.flat_map { |order_value| extract_column_name(order_value) } end def extract_column_name(order_value) |