diff options
Diffstat (limited to 'lib')
311 files changed, 5486 insertions, 1405 deletions
diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb index d91d4a0d4d5..05eb7f8222b 100644 --- a/lib/api/admin/sidekiq.rb +++ b/lib/api/admin/sidekiq.rb @@ -12,11 +12,11 @@ module API namespace 'queues' do desc 'Drop jobs matching the given metadata from the Sidekiq queue' params do - Gitlab::ApplicationContext::KNOWN_KEYS.each do |key| + Gitlab::SidekiqQueue::ALLOWED_KEYS.each do |key| optional key, type: String, allow_blank: false end - at_least_one_of(*Gitlab::ApplicationContext::KNOWN_KEYS) + at_least_one_of(*Gitlab::SidekiqQueue::ALLOWED_KEYS) end delete ':queue_name' do result = diff --git a/lib/api/api.rb b/lib/api/api.rb index 40f1b2fa9d3..d0d96858f61 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -171,6 +171,7 @@ module API mount ::API::Deployments mount ::API::Environments mount ::API::ErrorTracking + mount ::API::ErrorTrackingClientKeys mount ::API::ErrorTrackingCollector mount ::API::Events mount ::API::FeatureFlags diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 4d6d38f2dce..03b59e7e6ad 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -7,8 +7,6 @@ module API before { authenticate_non_get! } - feature_category :continuous_integration - params do requires :id, type: String, desc: 'The project ID' end @@ -44,7 +42,6 @@ module API optional :ref, type: String, desc: 'The ref of pipelines' optional :sha, type: String, desc: 'The sha of pipelines' optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' - optional :name, type: String, desc: '(deprecated) The name of the user who triggered pipelines' optional :username, type: String, desc: 'The username of the user who triggered pipelines' optional :updated_before, type: DateTime, desc: 'Return pipelines updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :updated_after, type: DateTime, desc: 'Return pipelines updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' @@ -54,7 +51,7 @@ module API desc: 'Sort pipelines' optional :source, type: String, values: ::Ci::Pipeline.sources.keys end - get ':id/pipelines' do + get ':id/pipelines', feature_category: :continuous_integration do authorize! :read_pipeline, user_project authorize! :read_build, user_project @@ -70,7 +67,7 @@ module API requires :ref, type: String, desc: 'Reference' optional :variables, Array, desc: 'Array of variables available in the pipeline' end - post ':id/pipeline' do + post ':id/pipeline', feature_category: :continuous_integration do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20711') authorize! :create_pipeline, user_project @@ -97,7 +94,7 @@ module API params do optional :ref, type: String, desc: 'branch ref of pipeline' end - get ':id/pipelines/latest' do + get ':id/pipelines/latest', feature_category: :continuous_integration do authorize! :read_pipeline, latest_pipeline present latest_pipeline, with: Entities::Ci::Pipeline @@ -110,7 +107,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - get ':id/pipelines/:pipeline_id' do + get ':id/pipelines/:pipeline_id', feature_category: :continuous_integration do authorize! :read_pipeline, pipeline present pipeline, with: Entities::Ci::Pipeline @@ -126,7 +123,7 @@ module API use :pagination end - get ':id/pipelines/:pipeline_id/jobs' do + get ':id/pipelines/:pipeline_id/jobs', feature_category: :continuous_integration do authorize!(:read_pipeline, user_project) pipeline = user_project.all_pipelines.find(params[:pipeline_id]) @@ -149,7 +146,7 @@ module API use :pagination end - get ':id/pipelines/:pipeline_id/bridges' do + get ':id/pipelines/:pipeline_id/bridges', feature_category: :pipeline_authoring do authorize!(:read_build, user_project) pipeline = user_project.all_pipelines.find(params[:pipeline_id]) @@ -169,7 +166,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - get ':id/pipelines/:pipeline_id/variables' do + get ':id/pipelines/:pipeline_id/variables', feature_category: :pipeline_authoring do authorize! :read_pipeline_variable, pipeline present pipeline.variables, with: Entities::Ci::Variable @@ -182,7 +179,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - get ':id/pipelines/:pipeline_id/test_report' do + get ':id/pipelines/:pipeline_id/test_report', feature_category: :code_testing do authorize! :read_build, pipeline present pipeline.test_reports, with: TestReportEntity, details: true @@ -195,7 +192,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - get ':id/pipelines/:pipeline_id/test_report_summary' do + get ':id/pipelines/:pipeline_id/test_report_summary', feature_category: :code_testing do authorize! :read_build, pipeline present pipeline.test_report_summary, with: TestReportSummaryEntity @@ -208,7 +205,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - delete ':id/pipelines/:pipeline_id' do + delete ':id/pipelines/:pipeline_id', feature_category: :continuous_integration do authorize! :destroy_pipeline, pipeline destroy_conditionally!(pipeline) do @@ -223,7 +220,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - post ':id/pipelines/:pipeline_id/retry' do + post ':id/pipelines/:pipeline_id/retry', feature_category: :continuous_integration do authorize! :update_pipeline, pipeline pipeline.retry_failed(current_user) @@ -238,7 +235,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - post ':id/pipelines/:pipeline_id/cancel' do + post ':id/pipelines/:pipeline_id/cancel', feature_category: :continuous_integration do authorize! :update_pipeline, pipeline pipeline.cancel_running diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 7f755b1a4d4..93a40925c21 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -222,6 +222,56 @@ module API end end + resource :runners do + before { authenticate_non_get! } + + desc 'Resets runner registration token' do + success Entities::Ci::ResetRegistrationTokenResult + end + post 'reset_registration_token' do + authorize! :update_runners_registration_token + + ApplicationSetting.current.reset_runners_registration_token! + present ApplicationSetting.current_without_cache.runners_registration_token, with: Entities::Ci::ResetRegistrationTokenResult + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before { authenticate_non_get! } + + desc 'Resets runner registration token' do + success Entities::Ci::ResetRegistrationTokenResult + end + post ':id/runners/reset_registration_token' do + project = find_project! user_project.id + authorize! :update_runners_registration_token, project + + project.reset_runners_token! + present project.runners_token, with: Entities::Ci::ResetRegistrationTokenResult + end + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before { authenticate_non_get! } + + desc 'Resets runner registration token' do + success Entities::Ci::ResetRegistrationTokenResult + end + post ':id/runners/reset_registration_token' do + group = find_group! user_group.id + authorize! :update_runners_registration_token, group + + group.reset_runners_token! + present group.runners_token, with: Entities::Ci::ResetRegistrationTokenResult + end + end + helpers do def filter_runners(runners, scope, allowed_scopes: ::Ci::Runner::AVAILABLE_SCOPES) return runners unless scope.present? diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 5d8985455ad..10dc51556b9 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -336,7 +336,7 @@ module API lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| - next unless line.new_pos == params[:line] && line.type == params[:line_type] + next unless line.line == params[:line] && line.type == params[:line_type] break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/dependency_proxy.rb b/lib/api/dependency_proxy.rb index 3379bb2f029..185b8d5a15d 100644 --- a/lib/api/dependency_proxy.rb +++ b/lib/api/dependency_proxy.rb @@ -15,7 +15,7 @@ module API end end - before do + after_validation do authorize! :admin_group, user_group end @@ -35,6 +35,8 @@ module API # rubocop:disable CodeReuse/Worker PurgeDependencyProxyCacheWorker.perform_async(current_user.id, user_group.id) # rubocop:enable CodeReuse/Worker + + status :accepted end end end diff --git a/lib/api/entities/application_setting.rb b/lib/api/entities/application_setting.rb index f23fce40468..465c5f4112b 100644 --- a/lib/api/entities/application_setting.rb +++ b/lib/api/entities/application_setting.rb @@ -27,6 +27,14 @@ module API expose(*::ApplicationSettingsHelper.external_authorization_service_attributes) + # Also expose these columns under their new attribute names. + # + # TODO: Once we rename the columns, we have to swap this around and keep supporting the old names until v5. + # https://gitlab.com/gitlab-org/gitlab/-/issues/340031 + expose :throttle_unauthenticated_enabled, as: :throttle_unauthenticated_web_enabled + expose :throttle_unauthenticated_period_in_seconds, as: :throttle_unauthenticated_web_period_in_seconds + expose :throttle_unauthenticated_requests_per_period, as: :throttle_unauthenticated_web_requests_per_period + # support legacy names, can be removed in v5 expose :password_authentication_enabled_for_web, as: :password_authentication_enabled expose :password_authentication_enabled_for_web, as: :signin_enabled diff --git a/lib/api/entities/basic_project_details.rb b/lib/api/entities/basic_project_details.rb index 0b231906ccd..5c33af86b84 100644 --- a/lib/api/entities/basic_project_details.rb +++ b/lib/api/entities/basic_project_details.rb @@ -43,12 +43,20 @@ module API # N+1 is solved then by using `subject.topics.map(&:name)` # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20555 projects_relation.preload(:project_feature, :route) - .preload(:import_state, :topics) + .preload(:import_state, :topics, :topics_acts_as_taggable) .preload(:auto_devops) .preload(namespace: [:route, :owner]) end # rubocop: enable CodeReuse/ActiveRecord + def self.execute_batch_counting(projects_relation) + # Call the count methods on every project, so the BatchLoader would load them all at + # once when the entities are rendered + projects_relation.each(&:forks_count) + + super + end + private alias_method :project, :object diff --git a/lib/api/entities/blob.rb b/lib/api/entities/blob.rb index b14ef127b68..12700d99865 100644 --- a/lib/api/entities/blob.rb +++ b/lib/api/entities/blob.rb @@ -10,7 +10,7 @@ module API # in the future we can only return the filename here without the leading # directory path. # https://gitlab.com/gitlab-org/gitlab/issues/34521 - expose :filename, &:path + expose :path, as: :filename expose :id expose :ref expose :startline diff --git a/lib/api/entities/ci/pipeline_basic.rb b/lib/api/entities/ci/pipeline_basic.rb index 8086062dc9b..4d56176bdb3 100644 --- a/lib/api/entities/ci/pipeline_basic.rb +++ b/lib/api/entities/ci/pipeline_basic.rb @@ -4,11 +4,9 @@ module API module Entities module Ci class PipelineBasic < Grape::Entity - expose :id, :project_id, :sha, :ref, :status + expose :id, :project_id, :sha, :ref, :status, :source expose :created_at, :updated_at - expose :source, if: ->(pipeline, options) { ::Feature.enabled?(:pipeline_source_filter, options[:project], default_enabled: :yaml) } - expose :web_url do |pipeline, _options| Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline) end diff --git a/lib/api/entities/ci/reset_registration_token_result.rb b/lib/api/entities/ci/reset_registration_token_result.rb new file mode 100644 index 00000000000..23426432f68 --- /dev/null +++ b/lib/api/entities/ci/reset_registration_token_result.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + class ResetRegistrationTokenResult < Grape::Entity + expose(:token) {|object| object} + end + end + end +end diff --git a/lib/api/entities/clusters/agent_authorization.rb b/lib/api/entities/clusters/agent_authorization.rb new file mode 100644 index 00000000000..6c533fff105 --- /dev/null +++ b/lib/api/entities/clusters/agent_authorization.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Clusters + class AgentAuthorization < Grape::Entity + expose :agent_id, as: :id + expose :project, with: Entities::ProjectIdentity, as: :config_project + expose :config, as: :configuration + end + end + end +end diff --git a/lib/api/entities/commit_note.rb b/lib/api/entities/commit_note.rb index d08b6fc8225..fe91712b48d 100644 --- a/lib/api/entities/commit_note.rb +++ b/lib/api/entities/commit_note.rb @@ -5,7 +5,7 @@ module API class CommitNote < Grape::Entity expose :note expose(:path) { |note| note.diff_file.try(:file_path) if note.diff_note? } - expose(:line) { |note| note.diff_line.try(:new_line) if note.diff_note? } + expose(:line) { |note| note.diff_line.try(:line) if note.diff_note? } expose(:line_type) { |note| note.diff_line.try(:type) if note.diff_note? } expose :author, using: Entities::UserBasic expose :created_at diff --git a/lib/api/entities/compare.rb b/lib/api/entities/compare.rb index fe2f03db2af..75a36d9bb01 100644 --- a/lib/api/entities/compare.rb +++ b/lib/api/entities/compare.rb @@ -20,6 +20,10 @@ module API end expose :same, as: :compare_same_ref + + expose :web_url do |compare, _| + Gitlab::UrlBuilder.build(compare) + end end end end diff --git a/lib/api/entities/error_tracking.rb b/lib/api/entities/error_tracking.rb index a38e00ca295..b55cba05ea0 100644 --- a/lib/api/entities/error_tracking.rb +++ b/lib/api/entities/error_tracking.rb @@ -10,6 +10,13 @@ module API expose :api_url expose :integrated end + + class ClientKey < Grape::Entity + expose :id + expose :active + expose :public_key + expose :sentry_dsn + end end end end diff --git a/lib/api/entities/global_notification_setting.rb b/lib/api/entities/global_notification_setting.rb index f3ca64347f0..f35efad5d01 100644 --- a/lib/api/entities/global_notification_setting.rb +++ b/lib/api/entities/global_notification_setting.rb @@ -4,7 +4,7 @@ module API module Entities class GlobalNotificationSetting < Entities::NotificationSetting expose :notification_email do |notification_setting, options| - notification_setting.user.notification_email + notification_setting.user.notification_email_or_default end end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 890b42ed8c8..b0e53ac3794 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -126,6 +126,10 @@ module API expose :keep_latest_artifacts_available?, as: :keep_latest_artifact # rubocop: disable CodeReuse/ActiveRecord + def self.preload_resource(project) + ActiveRecord::Associations::Preloader.new.preload(project, project_group_links: { group: :route }) + end + def self.preload_relation(projects_relation, options = {}) # Preloading topics, should be done with using only `:topics`, # as `:topics` are defined as: `has_many :topics, through: :taggings` @@ -140,12 +144,21 @@ module API .preload(project_group_links: { group: :route }, fork_network: :root_project, fork_network_member: :forked_from_project, - forked_from_project: [:route, :topics, :group, :project_feature, namespace: [:route, :owner]]) + forked_from_project: [:route, :topics, :topics_acts_as_taggable, :group, :project_feature, namespace: [:route, :owner]]) end # rubocop: enable CodeReuse/ActiveRecord - def self.forks_counting_projects(projects_relation) - projects_relation + projects_relation.map(&:forked_from_project).compact + def self.execute_batch_counting(projects_relation) + # Call the count methods on every project, so the BatchLoader would load them all at + # once when the entities are rendered + projects_relation.each(&:open_issues_count) + projects_relation.map(&:forked_from_project).compact.each(&:forks_count) + + super + end + + def self.repositories_for_preload(projects_relation) + super + projects_relation.map(&:forked_from_project).compact.map(&:repository) end end end diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index 973e80dd5ef..5c46233a639 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -4,8 +4,10 @@ module API module Entities class User < UserBasic include UsersHelper + include ActionView::Helpers::SanitizeHelper + expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } - expose :bio, :bio_html, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization, :job_title, :pronouns + expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization, :job_title, :pronouns expose :bot?, as: :bot expose :work_information do |user| work_information(user) @@ -16,6 +18,12 @@ module API expose :following, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user| user.followees.size end + + # This is only for multi version compatibility reasons, as we removed user.bio_html + # to be removed in 14.4 + expose :bio_html do |user| + strip_tags(user.bio) + end end end end diff --git a/lib/api/entities/user_public.rb b/lib/api/entities/user_public.rb index 78f088d3c1a..5d0e464abe1 100644 --- a/lib/api/entities/user_public.rb +++ b/lib/api/entities/user_public.rb @@ -14,7 +14,7 @@ module API expose :two_factor_enabled?, as: :two_factor_enabled expose :external expose :private_profile - expose :commit_email + expose :commit_email_or_default, as: :commit_email end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index e50da4264b5..c032b80e39b 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -58,7 +58,8 @@ module API end params do requires :environment_id, type: Integer, desc: 'The environment ID' - optional :name, type: String, desc: 'The new environment name' + # TODO: disallow renaming via the API https://gitlab.com/gitlab-org/gitlab/-/issues/338897 + optional :name, type: String, desc: 'DEPRECATED: Renaming environment can lead to errors, this will be removed in 15.0' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' optional :slug, absence: { message: "is automatically generated and cannot be changed" } end diff --git a/lib/api/error_tracking.rb b/lib/api/error_tracking.rb index 3abf2831bd3..369efe3bf8c 100644 --- a/lib/api/error_tracking.rb +++ b/lib/api/error_tracking.rb @@ -6,24 +6,30 @@ module API feature_category :error_tracking + helpers do + def project_setting + @project_setting ||= user_project.error_tracking_setting + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before do + authorize! :admin_operations, user_project + + not_found!('Error Tracking Setting') unless project_setting + end + desc 'Get error tracking settings for the project' do detail 'This feature was introduced in GitLab 12.7.' success Entities::ErrorTracking::ProjectSetting end get ':id/error_tracking/settings' do - authorize! :admin_operations, user_project - - setting = user_project.error_tracking_setting - - not_found!('Error Tracking Setting') unless setting - - present setting, with: Entities::ErrorTracking::ProjectSetting + present project_setting, with: Entities::ErrorTracking::ProjectSetting end desc 'Enable or disable error tracking settings for the project' do @@ -36,12 +42,6 @@ module API end patch ':id/error_tracking/settings/' do - authorize! :admin_operations, user_project - - setting = user_project.error_tracking_setting - - not_found!('Error Tracking Setting') unless setting - update_params = { error_tracking_setting_attributes: { enabled: params[:active] } } @@ -53,7 +53,7 @@ module API result = ::Projects::Operations::UpdateService.new(user_project, current_user, update_params).execute if result[:status] == :success - present setting, with: Entities::ErrorTracking::ProjectSetting + present project_setting, with: Entities::ErrorTracking::ProjectSetting else result end diff --git a/lib/api/error_tracking_client_keys.rb b/lib/api/error_tracking_client_keys.rb new file mode 100644 index 00000000000..eaa84b7186c --- /dev/null +++ b/lib/api/error_tracking_client_keys.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module API + class ErrorTrackingClientKeys < ::API::Base + before { authenticate! } + + feature_category :error_tracking + + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + segment ':id/error_tracking' do + before do + authorize! :admin_operations, user_project + end + + desc 'List all client keys' do + detail 'This feature was introduced in GitLab 14.3.' + success Entities::ErrorTracking::ClientKey + end + get '/client_keys' do + collection = user_project.error_tracking_client_keys + + present paginate(collection), with: Entities::ErrorTracking::ClientKey + end + + desc 'Create a client key' do + detail 'This feature was introduced in GitLab 14.3.' + success Entities::ErrorTracking::ClientKey + end + post '/client_keys' do + key = user_project.error_tracking_client_keys.create! + + present key, with: Entities::ErrorTracking::ClientKey + end + + desc 'Delete a client key' do + detail 'This feature was introduced in GitLab 14.3.' + success Entities::ErrorTracking::ClientKey + end + delete '/client_keys/:key_id' do + key = user_project.error_tracking_client_keys.find(params[:key_id]) + key.destroy! + end + end + end + end +end diff --git a/lib/api/error_tracking_collector.rb b/lib/api/error_tracking_collector.rb index 13e8e476808..b1e0f6a858a 100644 --- a/lib/api/error_tracking_collector.rb +++ b/lib/api/error_tracking_collector.rb @@ -8,6 +8,8 @@ module API feature_category :error_tracking content_type :envelope, 'application/x-sentry-envelope' + content_type :json, 'application/json' + content_type :txt, 'text/plain' default_format :envelope before do @@ -33,17 +35,24 @@ module API end def active_client_key? + public_key = extract_public_key + + find_client_key(public_key) + end + + def extract_public_key + # Some SDK send public_key as a param. In this case we don't need to parse headers. + return params[:sentry_key] if params[:sentry_key].present? + begin - public_key = ::ErrorTracking::Collector::SentryAuthParser.parse(request)[:public_key] + ::ErrorTracking::Collector::SentryAuthParser.parse(request)[:public_key] rescue StandardError bad_request!('Failed to parse sentry request') end - - find_client_key(public_key) end end - desc 'Submit error tracking event to the project' do + desc 'Submit error tracking event to the project as envelope' do detail 'This feature was introduced in GitLab 14.1.' end params do @@ -89,5 +98,38 @@ module API # it is safe only for submission of new events. no_content! end + + desc 'Submit error tracking event to the project' do + detail 'This feature was introduced in GitLab 14.1.' + end + params do + requires :id, type: String, desc: 'The ID of a project' + end + post 'error_tracking/collector/api/:id/store' do + # There is a reason why we have such uncommon path. + # We depend on a client side error tracking software which + # modifies URL for its own reasons. + # + # When we give user a URL like this + # HOST/api/v4/error_tracking/collector/123 + # + # Then error tracking software will convert it like this: + # HOST/api/v4/error_tracking/collector/api/123/store/ + + begin + parsed_body = Gitlab::Json.parse(request.body.read) + rescue StandardError + bad_request!('Failed to parse sentry request') + end + + ::ErrorTracking::CollectErrorService + .new(project, nil, event: parsed_body) + .execute + + # Collector should never return any information back. + # Because DSN and public key are designed for public use, + # it is safe only for submission of new events. + no_content! + end end end diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb index fb5858bc10b..c1f958ac007 100644 --- a/lib/api/feature_flags.rb +++ b/lib/api/feature_flags.rb @@ -118,7 +118,7 @@ module API put do authorize_update_feature_flag! exclude_legacy_flags_check! - render_api_error!('PUT operations are not supported for legacy feature flags', :unprocessable_entity) if feature_flag.legacy_flag? + render_api_error!('PUT operations are not supported for legacy feature flags', :unprocessable_entity) unless feature_flag.new_version_flag? attrs = declared_params(include_missing: false) @@ -207,7 +207,7 @@ module API end def exclude_legacy_flags_check! - if feature_flag.legacy_flag? + unless feature_flag.new_version_flag? not_found! end end diff --git a/lib/api/files.rb b/lib/api/files.rb index f3de7fbe96b..9d2b7cce837 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -35,10 +35,9 @@ module API not_found!('Commit') unless @commit @repo = user_project.repository - @blob = @repo.blob_at(@commit.sha, params[:file_path]) + @blob = @repo.blob_at(@commit.sha, params[:file_path], limit: Gitlab::Git::Blob::LFS_POINTER_MAX_SIZE) not_found!('File') unless @blob - @blob.load_all_data! end def commit_response(attrs) @@ -48,13 +47,21 @@ module API } end + def content_sha + Rails.cache.fetch("blob_content_sha256:#{user_project.full_path}:#{@blob.id}") do + @blob.load_all_data! + + Digest::SHA256.hexdigest(@blob.data) + end + end + def blob_data { file_name: @blob.name, file_path: @blob.path, size: @blob.size, encoding: "base64", - content_sha256: Digest::SHA256.hexdigest(@blob.data), + content_sha256: content_sha, ref: params[:ref], blob_id: @blob.id, commit_id: @commit.id, @@ -154,6 +161,8 @@ module API get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! + @blob.load_all_data! + data = blob_data set_http_headers(data) diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index a57d6bbcd2a..5e184d35255 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -62,7 +62,7 @@ module API authorize_upload!(project) bad_request!('File is too large') if max_file_size_exceeded? - ::Gitlab::Tracking.event(self.options[:for].name, 'push_package', user: current_user, project: project, namespace: project.namespace) + track_package_event('push_package', :generic, project: project, user: current_user, namespace: project.namespace) create_package_file_params = declared_params.merge(build: current_authenticated_job) ::Packages::Generic::CreatePackageFileService @@ -96,7 +96,7 @@ module API package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version]) package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute! - ::Gitlab::Tracking.event(self.options[:for].name, 'pull_package', user: current_user, project: project, namespace: project.namespace) + track_package_event('pull_package', :generic, project: project, user: current_user, namespace: project.namespace) present_carrierwave_file!(package_file.file) end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 13daf05fc78..e726f9b61cc 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -6,7 +6,7 @@ module API before { authenticate! } before { authorize! :admin_group, user_group } - feature_category :continuous_integration + feature_category :pipeline_authoring helpers ::API::Helpers::VariablesHelpers diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 0896357cc73..a1123b6291b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -108,6 +108,20 @@ module API present paginate(groups), options end + def present_groups_with_pagination_strategies(params, groups) + return present_groups(params, groups) if current_user.present? || Feature.disabled?(:keyset_pagination_for_groups_api) + + options = { + with: Entities::Group, + current_user: nil, + statistics: false + } + + groups, options = with_custom_attributes(groups, options) + + present paginate_with_strategies(groups), options + end + def delete_group(group) destroy_conditionally!(group) do |group| ::Groups::DestroyService.new(group, current_user).async_execute @@ -168,7 +182,7 @@ module API end get do groups = find_groups(declared_params(include_missing: false), params[:id]) - present_groups params, groups + present_groups_with_pagination_strategies params, groups end desc 'Create a group. Available only for users who can create groups.' do diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index 4280744d8b4..8a7e84c9f87 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -44,15 +44,10 @@ module API get ":channel/index.yaml" do authorize_read_package!(authorized_user_project) - package_files = Packages::Helm::PackageFilesFinder.new( - authorized_user_project, - params[:channel], - order_by: 'created_at', - sort: 'desc' - ).execute + packages = Packages::Helm::PackagesFinder.new(authorized_user_project, params[:channel]).execute env['api.format'] = :yaml - present ::Packages::Helm::IndexPresenter.new(authorized_user_project, params[:id], package_files), + present ::Packages::Helm::IndexPresenter.new(params[:id], params[:channel], packages), with: ::API::Entities::Helm::Index end @@ -66,7 +61,7 @@ module API get ":channel/charts/:file_name.tgz", requirements: FILE_NAME_REQUIREMENTS do authorize_read_package!(authorized_user_project) - package_file = Packages::Helm::PackageFilesFinder.new(authorized_user_project, params[:channel], file_name: "#{params[:file_name]}.tgz").execute.last! + package_file = Packages::Helm::PackageFilesFinder.new(authorized_user_project, params[:channel], file_name: "#{params[:file_name]}.tgz").most_recent! track_package_event('pull_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace) diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index b1954f8ece9..185a10a250c 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -34,7 +34,17 @@ module API end def self.sort_options - %w[created_at updated_at priority due_date relative_position label_priority milestone_due popularity] + %w[ + created_at + due_date + label_priority + milestone_due + popularity + priority + relative_position + title + updated_at + ] end def issue_finder(args = {}) @@ -43,9 +53,11 @@ module API args.delete(:id) args[:not] ||= {} args[:milestone_title] ||= args.delete(:milestone) - args[:not][:milestone_title] ||= args[:not]&.delete(:milestone) + args[:milestone_wildcard_id] ||= args.delete(:milestone_id) + args[:not][:milestone_title] ||= args[:not].delete(:milestone) + args[:not][:milestone_wildcard_id] ||= args[:not].delete(:milestone_id) args[:label_name] ||= args.delete(:labels) - args[:not][:label_name] ||= args[:not]&.delete(:labels) + args[:not][:label_name] ||= args[:not].delete(:labels) args[:scope] = args[:scope].underscore if args[:scope] args[:sort] = "#{args[:order_by]}_#{args[:sort]}" args[:issue_types] ||= args.delete(:issue_type) diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index e72bbb931f0..1e89f9f97a2 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -50,24 +50,48 @@ module API GroupMembersFinder.new(group).execute end - def create_member(current_user, user, source, params) - source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + def present_members(members) + present members, with: Entities::Member, current_user: current_user, show_seat_info: params[:show_seat_info] end - def track_areas_of_focus(member, areas_of_focus) - return unless areas_of_focus + def present_member_invitations(invitations) + present invitations, with: Entities::Invitation, current_user: current_user + end + + def add_single_member_by_user_id(create_service_params) + source = create_service_params[:source] + user_id = create_service_params[:user_ids] + user = User.find_by(id: user_id) # rubocop: disable CodeReuse/ActiveRecord + + if user + conflict!('Member already exists') if member_already_exists?(source, user_id) + + instance = ::Members::CreateService.new(current_user, create_service_params) + instance.execute + + not_allowed! if instance.membership_locked # This currently can only be reached in EE if group membership is locked - areas_of_focus.each do |area_of_focus| - Gitlab::Tracking.event(::Members::CreateService.name, 'area_of_focus', label: area_of_focus, property: member.id.to_s) + member = instance.single_member + render_validation_error!(member) if member.invalid? + + present_members(member) + else + not_found!('User') end end - def present_members(members) - present members, with: Entities::Member, current_user: current_user, show_seat_info: params[:show_seat_info] + def add_multiple_members?(user_id) + user_id.include?(',') end - def present_member_invitations(invitations) - present invitations, with: Entities::Invitation, current_user: current_user + def add_single_member?(user_id) + user_id.present? + end + + private + + def member_already_exists?(source, user_id) + source.members.exists?(user_id: user_id) # rubocop: disable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index ce5db52fdbc..34e126c73fc 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -57,7 +57,11 @@ module API .by_path(namespace_path) next unless namespace - finder = ::Packages::Npm::PackageFinder.new(package_name, namespace: namespace) + finder = ::Packages::Npm::PackageFinder.new( + package_name, + namespace: namespace, + last_of_each_version: false + ) finder.last&.project_id end diff --git a/lib/api/helpers/pagination_strategies.rb b/lib/api/helpers/pagination_strategies.rb index 61cff37e4ab..8c2186768ea 100644 --- a/lib/api/helpers/pagination_strategies.rb +++ b/lib/api/helpers/pagination_strategies.rb @@ -3,10 +3,16 @@ module API module Helpers module PaginationStrategies - def paginate_with_strategies(relation, request_scope) + def paginate_with_strategies(relation, request_scope = nil) paginator = paginator(relation, request_scope) - yield(paginator.paginate(relation)).tap do |records, _| + result = if block_given? + yield(paginator.paginate(relation)) + else + paginator.paginate(relation) + end + + result.tap do |records, _| paginator.finalize(records) end end @@ -20,17 +26,31 @@ module API private def keyset_paginator(relation) - request_context = Gitlab::Pagination::Keyset::RequestContext.new(self) - unless Gitlab::Pagination::Keyset.available?(request_context, relation) + if cursor_based_keyset_pagination_supported?(relation) + request_context_class = Gitlab::Pagination::Keyset::CursorBasedRequestContext + paginator_class = Gitlab::Pagination::Keyset::CursorPager + availability_checker = Gitlab::Pagination::CursorBasedKeyset + else + request_context_class = Gitlab::Pagination::Keyset::RequestContext + paginator_class = Gitlab::Pagination::Keyset::Pager + availability_checker = Gitlab::Pagination::Keyset + end + + request_context = request_context_class.new(self) + + unless availability_checker.available?(request_context, relation) return error!('Keyset pagination is not yet available for this type of request', 405) end - Gitlab::Pagination::Keyset::Pager.new(request_context) + paginator_class.new(request_context) end def offset_paginator(relation, request_scope) offset_limit = limit_for_scope(request_scope) - if Gitlab::Pagination::Keyset.available_for_type?(relation) && offset_limit_exceeded?(offset_limit) + if (Gitlab::Pagination::Keyset.available_for_type?(relation) || + cursor_based_keyset_pagination_supported?(relation)) && + offset_limit_exceeded?(offset_limit) + return error!("Offset pagination has a maximum allowed offset of #{offset_limit} " \ "for requests that return objects of type #{relation.klass}. " \ "Remaining records can be retrieved using keyset pagination.", 405) @@ -39,6 +59,10 @@ module API Gitlab::Pagination::OffsetPagination.new(self) end + def cursor_based_keyset_pagination_supported?(relation) + Gitlab::Pagination::CursorBasedKeyset.available_for_type?(relation) + end + def keyset_pagination_enabled? params[:pagination] == 'keyset' end diff --git a/lib/api/helpers/settings_helpers.rb b/lib/api/helpers/settings_helpers.rb index a3ea1057bc8..82de4917f0b 100644 --- a/lib/api/helpers/settings_helpers.rb +++ b/lib/api/helpers/settings_helpers.rb @@ -10,10 +10,18 @@ module API end def self.optional_attributes - [*::ApplicationSettingsHelper.visible_attributes, - *::ApplicationSettingsHelper.external_authorization_service_attributes, - *::ApplicationSettingsHelper.deprecated_attributes, - :performance_bar_allowed_group_id].freeze + [ + *::ApplicationSettingsHelper.visible_attributes, + *::ApplicationSettingsHelper.external_authorization_service_attributes, + *::ApplicationSettingsHelper.deprecated_attributes, + :performance_bar_allowed_group_id, + # TODO: Once we rename these columns, we can remove them here and add the old + # names to `ApplicationSettingsHelper.deprecated_attributes` instead. + # https://gitlab.com/gitlab-org/gitlab/-/issues/340031 + :throttle_unauthenticated_web_enabled, + :throttle_unauthenticated_web_period_in_seconds, + :throttle_unauthenticated_web_requests_per_period + ].freeze end end end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 7af5c2ad2ee..d1ad3c1feb1 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -100,6 +100,23 @@ module API end end + namespace 'kubernetes/agent_configuration' do + desc 'POST agent configuration' do + detail 'Store configuration for an agent' + end + params do + requires :agent_id, type: Integer, desc: 'ID of the configured Agent' + requires :agent_config, type: JSON, desc: 'Configuration for the Agent' + end + post '/' do + agent = Clusters::Agent.find(params[:agent_id]) + + Clusters::Agents::RefreshAuthorizationService.new(agent, config: params[:agent_config]).execute + + no_content! + end + end + namespace 'kubernetes/usage_metrics' do desc 'POST usage metrics' do detail 'Updates usage metrics for agent' diff --git a/lib/api/issues.rb b/lib/api/issues.rb index a6565f913e3..39ce6e0b062 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -14,6 +14,10 @@ module API params :negatable_issue_filter_params do optional :labels, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' optional :milestone, type: String, desc: 'Milestone title' + optional :milestone_id, types: String, values: %w[Any None Upcoming Started], + desc: 'Return issues assigned to milestones without the specified timebox value ("Any", "None", "Upcoming" or "Started")' + mutually_exclusive :milestone_id, :milestone + optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IID array of issues' optional :author_id, type: Integer, desc: 'Return issues which are not authored by the user with the given ID' @@ -32,9 +36,14 @@ module API params :issues_stats_params do optional :labels, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' optional :milestone, type: String, desc: 'Milestone title' + # 'milestone_id' only accepts wildcard values 'Any', 'None', 'Upcoming', 'Started' + # the param has '_id' in the name to keep consistency (ex. assignee_id accepts id and wildcard values). + optional :milestone_id, types: String, values: %w[Any None Upcoming Started], + desc: 'Return issues assigned to milestones with the specified timebox value ("Any", "None", "Upcoming" or "Started")' optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IID array of issues' optional :search, type: String, desc: 'Search issues for text present in the title, description, or any combination of these' optional :in, type: String, desc: '`title`, `description`, or a string joining them with comma' + mutually_exclusive :milestone_id, :milestone optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' optional :author_username, type: String, desc: 'Return issues which are authored by the user with the given username' @@ -69,7 +78,7 @@ module API optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' optional :order_by, type: String, values: Helpers::IssuesHelpers.sort_options, default: 'created_at', - desc: 'Return issues ordered by `created_at` or `updated_at` fields.' + desc: 'Return issues ordered by `created_at`, `due_date`, `label_priority`, `milestone_due`, `popularity`, `priority`, `relative_position`, `title`, or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return issues sorted in `asc` or `desc` order.' optional :due_date, type: String, values: %w[0 overdue week month next_month_and_previous_two_weeks] << '', diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 945cdf3edb2..fa871b4bc57 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -13,18 +13,13 @@ module API post '/lint' do unauthorized! if (Gitlab::CurrentSettings.signup_disabled? || Gitlab::CurrentSettings.signup_limited?) && current_user.nil? - result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute + result = Gitlab::Ci::Lint.new(project: nil, current_user: current_user) + .validate(params[:content], dry_run: false) status 200 - - response = if result.errors.empty? - { status: 'valid', errors: [], warnings: result.warnings } - else - { status: 'invalid', errors: result.errors, warnings: result.warnings } - end - - response.tap do |response| - response[:merged_yaml] = result.merged_yaml if params[:include_merged_yaml] + Entities::Ci::Lint::Result.represent(result, current_user: current_user).serializable_hash.tap do |presented_result| + presented_result[:status] = presented_result[:valid] ? 'valid' : 'invalid' + presented_result.delete(:merged_yaml) unless params[:include_merged_yaml] end end end diff --git a/lib/api/members.rb b/lib/api/members.rb index 7130635281a..332520ccd26 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -96,42 +96,22 @@ module API optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'members-api' optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon' end - # rubocop: disable CodeReuse/ActiveRecord + post ":id/members" do ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/333434') source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - if params[:user_id].to_s.include?(',') - create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id], source: source }) + user_id = params[:user_id].to_s + create_service_params = params.except(:user_id).merge({ user_ids: user_id, source: source }) + if add_multiple_members?(user_id) ::Members::CreateService.new(current_user, create_service_params).execute - elsif params[:user_id].present? - member = source.members.find_by(user_id: params[:user_id]) - conflict!('Member already exists') if member - - user = User.find_by_id(params[:user_id]) - not_found!('User') unless user - - member = create_member(current_user, user, source, params) - - if !member - not_allowed! # This currently can only be reached in EE - elsif member.valid? && member.persisted? - present_members(member) - Gitlab::Tracking.event(::Members::CreateService.name, - 'create_member', - label: params[:invite_source], - property: 'existing_user', - user: current_user) - track_areas_of_focus(member, params[:areas_of_focus]) - else - render_validation_error!(member) - end + elsif add_single_member?(user_id) + add_single_member_by_user_id(create_service_params) end end - # rubocop: enable CodeReuse/ActiveRecord desc 'Updates a member of a group or project.' do success Entities::Member diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 7ab57982907..34af9eab511 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -202,7 +202,7 @@ module API options[:project] = user_project if Feature.enabled?(:api_caching_merge_requests, user_project, type: :development, default_enabled: :yaml) - present_cached merge_requests, expires_in: 10.minutes, **options + present_cached merge_requests, expires_in: 2.days, **options else present merge_requests, options end diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 7ff4439ce04..dbfc0a61577 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -48,14 +48,13 @@ module API put ':package_name', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do authorize_create_package!(project) - track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, user: current_user, namespace: project.namespace) - created_package = ::Packages::Npm::CreatePackageService .new(project, current_user, params.merge(build: current_authenticated_job)).execute if created_package[:status] == :error render_api_error!(created_package[:message], created_package[:http_status]) else + track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, user: current_user, namespace: project.namespace) created_package end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 28bcb382ecf..a92d904be84 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -152,6 +152,12 @@ module API ProjectsFinder.new(current_user: current_user, params: project_params).execute end + def present_project(project, options = {}) + options[:with].preload_resource(project) if options[:with].respond_to?(:preload_resource) + + present project, options + end + def present_projects(projects, options = {}) verify_statistics_order_by_projects! @@ -264,9 +270,9 @@ module API project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? - present project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, project), - current_user: current_user + present_project project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project), + current_user: current_user else if project.errors[:limit_reached].present? error!(project.errors[:limit_reached], 403) @@ -301,9 +307,9 @@ module API project = ::Projects::CreateService.new(user, attrs).execute if project.saved? - present project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, project), - current_user: current_user + present_project project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project), + current_user: current_user else render_validation_error!(project) end @@ -336,7 +342,7 @@ module API project, options = with_custom_attributes(user_project, options) - present project, options + present_project project, options end desc 'Fork new project for the current user or provided namespace.' do @@ -376,9 +382,11 @@ module API if forked_project.errors.any? conflict!(forked_project.errors.messages) else - present forked_project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, forked_project), - current_user: current_user + present_project forked_project, { + with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, forked_project), + current_user: current_user + } end end @@ -427,9 +435,9 @@ module API result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success - present user_project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, user_project), - current_user: current_user + present_project user_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project), + current_user: current_user else render_validation_error!(user_project) end @@ -443,7 +451,7 @@ module API ::Projects::UpdateService.new(user_project, current_user, archived: true).execute - present user_project, with: Entities::Project, current_user: current_user + present_project user_project, with: Entities::Project, current_user: current_user end desc 'Unarchive a project' do @@ -454,7 +462,7 @@ module API ::Projects::UpdateService.new(user_project, current_user, archived: false).execute - present user_project, with: Entities::Project, current_user: current_user + present_project user_project, with: Entities::Project, current_user: current_user end desc 'Star a project' do @@ -467,7 +475,7 @@ module API current_user.toggle_star(user_project) user_project.reset - present user_project, with: Entities::Project, current_user: current_user + present_project user_project, with: Entities::Project, current_user: current_user end end @@ -479,7 +487,7 @@ module API current_user.toggle_star(user_project) user_project.reset - present user_project, with: Entities::Project, current_user: current_user + present_project user_project, with: Entities::Project, current_user: current_user else not_modified! end @@ -528,7 +536,7 @@ module API result = ::Projects::ForkService.new(fork_from_project, current_user).execute(user_project) if result - present user_project.reset, with: Entities::Project, current_user: current_user + present_project user_project.reset, with: Entities::Project, current_user: current_user else render_api_error!("Project already forked", 409) if user_project.forked? end @@ -698,7 +706,7 @@ module API result = ::Projects::TransferService.new(user_project, current_user).execute(namespace) if result - present user_project, with: Entities::Project, current_user: current_user + present_project user_project, with: Entities::Project, current_user: current_user else render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400) end diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index 6dfd82d109f..db46602cd90 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -7,28 +7,35 @@ module API class_methods do def prepare_relation(projects_relation, options = {}) projects_relation = preload_relation(projects_relation, options) + execute_batch_counting(projects_relation) - # Call the forks count method on every project, so the BatchLoader would load them all at - # once when the entities are rendered - projects_relation.each(&:forks_count) + + preload_repository_cache(projects_relation) projects_relation end + # This is overridden by the specific Entity class to + # preload assocations that it needs def preload_relation(projects_relation, options = {}) projects_relation end - def forks_counting_projects(projects_relation) - projects_relation + # This is overridden by the specific Entity class to + # batch load certain counts + def execute_batch_counting(projects_relation) end - def batch_open_issues_counting(projects_relation) - ::Projects::BatchOpenIssuesCountService.new(projects_relation).refresh_cache + def preload_repository_cache(projects_relation) + repositories = repositories_for_preload(projects_relation) + + Gitlab::RepositoryCache::Preloader.new(repositories).preload( # rubocop:disable CodeReuse/ActiveRecord + %i[exists? root_ref has_visible_content? avatar readme_path] + ) end - def execute_batch_counting(projects_relation) - batch_open_issues_counting(projects_relation) + def repositories_for_preload(projects_relation) + projects_relation.map(&:repository) end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 20320d1b7ae..3c9255e3117 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -51,18 +51,22 @@ module API optional :ref, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' optional :path, type: String, desc: 'The path of the tree' optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' + use :pagination + optional :pagination, type: String, values: %w(legacy keyset), default: 'legacy', desc: 'Specify the pagination method' + + given pagination: -> (value) { value == 'keyset' } do + optional :page_token, type: String, desc: 'Record from which to start the keyset pagination' + end end get ':id/repository/tree' do - ref = params[:ref] || user_project.default_branch - path = params[:path] || nil + tree_finder = ::Repositories::TreeFinder.new(user_project, declared_params(include_missing: false)) + + not_found!("Tree") unless tree_finder.commit_exists? - commit = user_project.commit(ref) - not_found!('Tree') unless commit + tree = Gitlab::Pagination::GitalyKeysetPager.new(self, user_project).paginate(tree_finder) - tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) - entries = ::Kaminari.paginate_array(tree.sorted_entries) - present paginate(entries), with: Entities::TreeObject + present tree, with: Entities::TreeObject end desc 'Get raw blob contents from the repository' diff --git a/lib/api/settings.rb b/lib/api/settings.rb index aac195f0668..36f816ae638 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -176,6 +176,7 @@ module API optional :require_admin_approval_after_user_signup, type: Boolean, desc: 'Require explicit admin approval for new signups' optional :whats_new_variant, type: String, values: ApplicationSetting.whats_new_variants.keys, desc: "What's new variant, possible values: `all_tiers`, `current_tier`, and `disabled`." optional :floc_enabled, type: Grape::API::Boolean, desc: 'Enable FloC (Federated Learning of Cohorts)' + optional :user_deactivation_emails_enabled, type: Boolean, desc: 'Send emails to users upon account deactivation' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", @@ -225,6 +226,16 @@ module API attrs[:asset_proxy_allowlist] = attrs.delete(:asset_proxy_whitelist) end + # Also accept these attributes under their new names. + # + # TODO: Once we rename the columns, we have to swap this around and keep supporting the old names until v5. + # https://gitlab.com/gitlab-org/gitlab/-/issues/340031 + %w[enabled period_in_seconds requests_per_period].each do |suffix| + old_name = :"throttle_unauthenticated_#{suffix}" + new_name = :"throttle_unauthenticated_web_#{suffix}" + attrs[old_name] = attrs.delete(new_name) if attrs.has_key?(new_name) + end + # since 13.0 it's not possible to disable hashed storage - support can be removed in 14.0 attrs.delete(:hashed_storage_enabled) if attrs.has_key?(:hashed_storage_enabled) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index a595129fd6a..85a299c5673 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -11,7 +11,7 @@ module API }, gitlab_ci_ymls: { gitlab_version: 8.9, - feature_category: :continuous_integration + feature_category: :pipeline_authoring }, dockerfiles: { gitlab_version: 8.15, diff --git a/lib/api/users.rb b/lib/api/users.rb index 2608fb87e22..e3271b8b9b2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -615,6 +615,22 @@ module API end end + desc 'Reject a pending user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + post ':id/reject', feature_category: :authentication_and_authorization do + user = find_user_by_id(params) + + result = ::Users::RejectService.new(current_user).execute(user) + + if result[:success] + present user + else + render_api_error!(result[:message], result[:http_status]) + end + end + # rubocop: enable CodeReuse/ActiveRecord desc 'Deactivate an active user. Available only for admins.' params do @@ -687,6 +703,38 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'Ban a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + post ':id/ban', feature_category: :authentication_and_authorization do + authenticated_as_admin! + user = find_user_by_id(params) + + result = ::Users::BanService.new(current_user).execute(user) + if result[:status] == :success + true + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Unban a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + post ':id/unban', feature_category: :authentication_and_authorization do + authenticated_as_admin! + user = find_user_by_id(params) + + result = ::Users::UnbanService.new(current_user).execute(user) + if result[:status] == :success + true + else + render_api_error!(result[:message], result[:http_status]) + end + end + desc 'Get memberships' do success Entities::Membership end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index 55fd68fd6e8..47b63990262 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -22,8 +22,8 @@ module Backup end args = [] - args += ['-parallel', @parallel.to_s] if type == :create && @parallel - args += ['-parallel-storage', @parallel_storage.to_s] if type == :create && @parallel_storage + args += ['-parallel', @parallel.to_s] if @parallel + args += ['-parallel-storage', @parallel_storage.to_s] if @parallel_storage @stdin, stdout, @thread = Open3.popen2(ENV, bin_path, command, '-path', backup_repos_path, *args) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 52810b0fb35..6c5350082e8 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -47,10 +47,12 @@ module Backup return end - directory = connect_to_remote_directory(Gitlab.config.backup.upload) + directory = connect_to_remote_directory + upload = directory.files.create(create_attributes) - if directory.files.create(create_attributes) + if upload progress.puts "done".color(:green) + upload else puts "uploading backup to #{remote_directory} failed".color(:red) raise Backup::Error, 'Backup failed' @@ -206,11 +208,16 @@ module Backup @backup_file_list.map {|item| item.gsub("#{FILE_NAME_SUFFIX}", "")} end - def connect_to_remote_directory(options) - config = ObjectStorage::Config.new(options) - config.load_provider + def object_storage_config + @object_storage_config ||= begin + config = ObjectStorage::Config.new(Gitlab.config.backup.upload) + config.load_provider + config + end + end - connection = ::Fog::Storage.new(config.credentials) + def connect_to_remote_directory + connection = ::Fog::Storage.new(object_storage_config.credentials) # We only attempt to create the directory for local backups. For AWS # and other cloud providers, we cannot guarantee the user will have @@ -280,10 +287,8 @@ module Backup key: remote_target, body: File.open(File.join(backup_path, tar_file)), multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, - encryption: Gitlab.config.backup.upload.encryption, - encryption_key: Gitlab.config.backup.upload.encryption_key, storage_class: Gitlab.config.backup.upload.storage_class - } + }.merge(encryption_attributes) # Google bucket-only policies prevent setting an ACL. In any case, by default, # all objects are set to the default ACL, which is project-private: @@ -293,6 +298,19 @@ module Backup attrs end + def encryption_attributes + return object_storage_config.fog_attributes if object_storage_config.aws_server_side_encryption_enabled? + + # Use customer-managed keys. Also, this preserves + # backward-compatibility for existing usages of `SSE-S3` that + # don't set `backup.upload.storage_options.server_side_encryption` + # to `'AES256'`. + { + encryption_key: Gitlab.config.backup.upload.encryption_key, + encryption: Gitlab.config.backup.upload.encryption + } + end + def google_provider? Gitlab.config.backup.upload.connection&.provider&.downcase == 'google' end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index ae293073ba2..393cf4108a1 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -2,12 +2,16 @@ module Backup class Pages < Backup::Files + # pages used to deploy tmp files to this path + # if some of these files are still there, we don't need them in the backup + LEGACY_PAGES_TMP_PATH = '@pages.tmp' + attr_reader :progress def initialize(progress) @progress = progress - super('pages', Gitlab.config.pages.path, excludes: [::Projects::UpdatePagesService::TMP_EXTRACT_PATH]) + super('pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH]) end end end diff --git a/lib/banzai/filter/playable_link_filter.rb b/lib/banzai/filter/playable_link_filter.rb index 0a043aa809c..5b290ff08f6 100644 --- a/lib/banzai/filter/playable_link_filter.rb +++ b/lib/banzai/filter/playable_link_filter.rb @@ -52,7 +52,7 @@ module Banzai doc.document.create_element(media_type, media_element_attrs) end - def download_paragraph(doc, element) + def download_link(doc, element) link_content = element['title'] || element['alt'] link_element_attrs = { @@ -67,19 +67,15 @@ module Banzai link_element_attrs['data-canonical-src'] = element['data-canonical-src'] end - link = doc.document.create_element('a', link_content, link_element_attrs) - - doc.document.create_element('p').tap do |paragraph| - paragraph.children = link - end + doc.document.create_element('a', link_content, link_element_attrs) end def media_node(doc, element) - container_element_attrs = { class: "#{media_type}-container" } + container_element_attrs = { class: "media-container #{media_type}-container" } - doc.document.create_element( "div", container_element_attrs).tap do |container| + doc.document.create_element('span', container_element_attrs).tap do |container| container.add_child(media_element(doc, element)) - container.add_child(download_paragraph(doc, element)) + container.add_child(download_link(doc, element)) end end end diff --git a/lib/banzai/filter/references/label_reference_filter.rb b/lib/banzai/filter/references/label_reference_filter.rb index 3ae9c5f8d90..a019ae0108e 100644 --- a/lib/banzai/filter/references/label_reference_filter.rb +++ b/lib/banzai/filter/references/label_reference_filter.rb @@ -23,7 +23,8 @@ module Banzai label_relation = labels.where(title: label_names) end - return Label.none if (relation = [id_relation, label_relation].compact).empty? + relation = [id_relation, label_relation].compact + return Label.none if relation.all?(Label.none) Label.from_union(relation) end diff --git a/lib/banzai/filter/references/milestone_reference_filter.rb b/lib/banzai/filter/references/milestone_reference_filter.rb index d992e667056..94f7106d31e 100644 --- a/lib/banzai/filter/references/milestone_reference_filter.rb +++ b/lib/banzai/filter/references/milestone_reference_filter.rb @@ -23,7 +23,8 @@ module Banzai milestone_relation = find_milestones(parent, false).where(name: milestone_names) end - return Milestone.none if (relation = [iid_relation, milestone_relation].compact).empty? + relation = [iid_relation, milestone_relation].compact + return Milestone.none if relation.all?(Milestone.none) Milestone.from_union(relation).includes(:project, :group) end @@ -116,11 +117,11 @@ module Banzai # We don't support IID lookups because IIDs can clash between # group/project milestones and group/subgroup milestones. - params[:group_ids] = self_and_ancestors_ids(parent) unless find_by_iid + params[:group_ids] = group_and_ancestors_ids(parent) unless find_by_iid end end - def self_and_ancestors_ids(parent) + def group_and_ancestors_ids(parent) if group_context?(parent) parent.self_and_ancestors.select(:id) elsif project_context?(parent) diff --git a/lib/banzai/filter/references/reference_cache.rb b/lib/banzai/filter/references/reference_cache.rb index 816ce973cad..b2d47aba2d6 100644 --- a/lib/banzai/filter/references/reference_cache.rb +++ b/lib/banzai/filter/references/reference_cache.rb @@ -28,11 +28,18 @@ module Banzai @references_per_parent[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - if Feature.enabled?(:milestone_reference_pattern, default_enabled: :yaml) - doc_search(refs) - else - node_search(nodes, refs) + prepare_doc_for_scan(filter.doc).to_enum(:scan, regex).each do + parent_path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + + ident = filter.identifier($~) + refs[parent_path] << ident if ident end + + refs end end @@ -163,39 +170,6 @@ module Banzai delegate :project, :group, :parent, :parent_type, to: :filter - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/336268 - def node_search(nodes, refs) - nodes.each do |node| - prepare_node_for_scan(node).scan(regex) do - parent_path = if parent_type == :project - full_project_path($~[:namespace], $~[:project]) - else - full_group_path($~[:group]) - end - - ident = filter.identifier($~) - refs[parent_path] << ident if ident - end - end - - refs - end - - def doc_search(refs) - prepare_doc_for_scan(filter.doc).to_enum(:scan, regex).each do - parent_path = if parent_type == :project - full_project_path($~[:namespace], $~[:project]) - else - full_group_path($~[:group]) - end - - ident = filter.identifier($~) - refs[parent_path] << ident if ident - end - - refs - end - def regex strong_memoize(:regex) do [ @@ -215,13 +189,6 @@ module Banzai filter.requires_unescaping? ? unescape_html_entities(html) : html end - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/336268 - def prepare_node_for_scan(node) - html = node.to_html - - filter.requires_unescaping? ? unescape_html_entities(html) : html - end - def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 0c015ba00c7..831baa9a778 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -212,9 +212,8 @@ module Banzai def gather_references(nodes, ids_only: false) nodes = nodes_user_can_reference(current_user, nodes) visible = nodes_visible_to_user(current_user, nodes) - not_visible = nodes - visible - { visible: referenced_by(visible, ids_only: ids_only), not_visible: not_visible } + { visible: referenced_by(visible, ids_only: ids_only), nodes: nodes, visible_nodes: visible } end # Returns a Hash containing the projects for a given list of HTML nodes. diff --git a/lib/bulk_imports/groups/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index 1a709179bf9..aa9221cceee 100644 --- a/lib/bulk_imports/groups/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module BulkImports - module Groups + module Common module Pipelines class EntityFinisher def self.ndjson_pipeline? diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb new file mode 100644 index 00000000000..4cec1ad1462 --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetProjectsQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!, $cursor: String, $per_page: Int) { + group(fullPath: $full_path) { + projects(includeSubgroups: false, first: $per_page, after: $cursor) { + page_info: pageInfo { + next_page: endCursor + has_next_page: hasNextPage + } + nodes { + name + full_path: fullPath + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE + } + end + + def base_path + %w[data group projects] + end + + def data_path + base_path << 'nodes' + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb new file mode 100644 index 00000000000..c318675e649 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class ProjectEntitiesPipeline + include Pipeline + + extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectsQuery + transformer Common::Transformers::ProhibitedAttributesTransformer + + def transform(context, data) + { + source_type: :project_entity, + source_full_path: data['full_path'], + destination_name: data['name'], + destination_namespace: context.entity.group.full_path, + parent_id: context.entity.id + } + end + + def load(context, data) + context.bulk_import.entities.create!(data) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb new file mode 100644 index 00000000000..8c3b6975b73 --- /dev/null +++ b/lib/bulk_imports/groups/stage.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + class Stage < ::BulkImports::Stage + private + + def config + @config ||= { + group: { + pipeline: BulkImports::Groups::Pipelines::GroupPipeline, + stage: 0 + }, + avatar: { + pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, + stage: 1 + }, + subgroups: { + pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, + stage: 1 + }, + members: { + pipeline: BulkImports::Groups::Pipelines::MembersPipeline, + stage: 1 + }, + labels: { + pipeline: BulkImports::Groups::Pipelines::LabelsPipeline, + stage: 1 + }, + milestones: { + pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, + stage: 1 + }, + badges: { + pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + stage: 1 + }, + boards: { + pipeline: BulkImports::Groups::Pipelines::BoardsPipeline, + stage: 2 + }, + finisher: { + pipeline: BulkImports::Common::Pipelines::EntityFinisher, + stage: 3 + } + }.merge(project_entities_pipeline) + end + + def project_entities_pipeline + if ::Feature.enabled?(:bulk_import_projects, default_enabled: :yaml) + { + project_entities: { + pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline, + stage: 1 + } + } + else + {} + end + end + end + end +end + +::BulkImports::Groups::Stage.prepend_mod_with('BulkImports::Groups::Stage') diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index f27818dae18..6798936576b 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -69,8 +69,8 @@ module BulkImports # Multiple transformers can be defined within a single # pipeline and run sequentially for each record in the # following order: - # - Transformers defined using `transformer` class method # - Instance method `transform` + # - Transformers defined using `transformer` class method # # Instance method `transform` is always the last to run. # diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb new file mode 100644 index 00000000000..2aec496880f --- /dev/null +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module GetProjectQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!) { + project(fullPath: $full_path) { + description + visibility + archived + created_at: createdAt + shared_runners_enabled: sharedRunnersEnabled + container_registry_enabled: containerRegistryEnabled + only_allow_merge_if_pipeline_succeeds: onlyAllowMergeIfPipelineSucceeds + only_allow_merge_if_all_discussions_are_resolved: onlyAllowMergeIfAllDiscussionsAreResolved + request_access_enabled: requestAccessEnabled + printing_merge_request_link_enabled: printingMergeRequestLinkEnabled + remove_source_branch_after_merge: removeSourceBranchAfterMerge + autoclose_referenced_issues: autocloseReferencedIssues + suggestion_commit_message: suggestionCommitMessage + wiki_enabled: wikiEnabled + } + } + GRAPHQL + end + + def variables(context) + { full_path: context.entity.source_full_path } + end + + def base_path + %w[data project] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_pipeline.rb new file mode 100644 index 00000000000..c9da33fe8e3 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_pipeline.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectPipeline + include Pipeline + + abort_on_failure! + + extractor ::BulkImports::Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectQuery + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + transformer ::BulkImports::Projects::Transformers::ProjectAttributesTransformer + + def load(context, data) + project = ::Projects::CreateService.new(context.current_user, data).execute + + if project.persisted? + context.entity.update!(project: project) + + project + else + raise(::BulkImports::Error, "Unable to import project #{project.full_path}. #{project.errors.full_messages}.") + end + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb new file mode 100644 index 00000000000..b606003091b --- /dev/null +++ b/lib/bulk_imports/projects/stage.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + class Stage < ::BulkImports::Stage + private + + def config + @config ||= { + group: { + pipeline: BulkImports::Projects::Pipelines::ProjectPipeline, + stage: 0 + }, + finisher: { + pipeline: BulkImports::Common::Pipelines::EntityFinisher, + stage: 1 + } + } + end + end + end +end + +::BulkImports::Projects::Stage.prepend_mod_with('BulkImports::Projects::Stage') diff --git a/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb new file mode 100644 index 00000000000..24c55d8dbb1 --- /dev/null +++ b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Transformers + class ProjectAttributesTransformer + PROJECT_IMPORT_TYPE = 'gitlab_project_migration' + + def transform(context, data) + entity = context.entity + visibility = data.delete('visibility') + + data['name'] = entity.destination_name + data['path'] = entity.destination_name.parameterize + data['import_type'] = PROJECT_IMPORT_TYPE + data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility] if visibility.present? + data['namespace_id'] = Namespace.find_by_full_path(entity.destination_namespace)&.id if entity.destination_namespace.present? + + data.transform_keys!(&:to_sym) + end + end + end + end +end diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index b1bceecbaea..103623cd030 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -2,55 +2,8 @@ module BulkImports class Stage - include Singleton - - CONFIG = { - group: { - pipeline: BulkImports::Groups::Pipelines::GroupPipeline, - stage: 0 - }, - avatar: { - pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, - stage: 1 - }, - subgroups: { - pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, - stage: 1 - }, - members: { - pipeline: BulkImports::Groups::Pipelines::MembersPipeline, - stage: 1 - }, - labels: { - pipeline: BulkImports::Groups::Pipelines::LabelsPipeline, - stage: 1 - }, - milestones: { - pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, - stage: 1 - }, - badges: { - pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, - stage: 1 - }, - boards: { - pipeline: BulkImports::Groups::Pipelines::BoardsPipeline, - stage: 2 - }, - finisher: { - pipeline: BulkImports::Groups::Pipelines::EntityFinisher, - stage: 3 - } - }.freeze - def self.pipelines - instance.pipelines - end - - def self.pipeline_exists?(name) - pipelines.any? do |(_, pipeline)| - pipeline.to_s == name.to_s - end + new.pipelines end def pipelines @@ -65,9 +18,8 @@ module BulkImports private def config - @config ||= CONFIG + # To be implemented in a sub-class + NotImplementedError end end end - -::BulkImports::Stage.prepend_mod_with('BulkImports::Stage') diff --git a/lib/error_tracking/collector/dsn.rb b/lib/error_tracking/collector/dsn.rb new file mode 100644 index 00000000000..665181328f3 --- /dev/null +++ b/lib/error_tracking/collector/dsn.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module ErrorTracking + module Collector + class Dsn + # Build a sentry compatible DSN URL for GitLab collector. + # + # The expected URL looks like that: + # https://PUBLIC_KEY@gitlab.example.com/api/v4/error_tracking/collector/PROJECT_ID + # + def self.build_url(public_key, project_id) + gitlab = Settings.gitlab + + custom_port = Settings.gitlab_on_standard_port? ? nil : ":#{gitlab.port}" + + base_url = [ + gitlab.protocol, + "://", + public_key, + '@', + gitlab.host, + custom_port, + gitlab.relative_url_root + ].join('') + + "#{base_url}/api/v4/error_tracking/collector/#{project_id}" + end + end + end +end diff --git a/lib/gem_extensions/active_record/association.rb b/lib/gem_extensions/active_record/association.rb index 91a9f45ce7e..c6634a0524a 100644 --- a/lib/gem_extensions/active_record/association.rb +++ b/lib/gem_extensions/active_record/association.rb @@ -15,7 +15,7 @@ module GemExtensions def scope if disable_joins - DisableJoins::Associations::AssociationScope.create.scope(self) + ::GemExtensions::ActiveRecord::DisableJoins::Associations::AssociationScope.create.scope(self) else super end @@ -25,7 +25,7 @@ module GemExtensions if klass @association_scope ||= begin # rubocop:disable Gitlab/ModuleWithInstanceVariables if disable_joins - DisableJoins::Associations::AssociationScope.scope(self) + ::GemExtensions::ActiveRecord::DisableJoins::Associations::AssociationScope.scope(self) else super end diff --git a/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb b/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb index 1e4476330a2..9ff80829bc3 100644 --- a/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb +++ b/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb @@ -64,7 +64,7 @@ module GemExtensions end if scope.order_values.empty? && ordered - split_scope = DisableJoins::Relation.create(scope.klass, key, join_ids) + split_scope = ::GemExtensions::ActiveRecord::DisableJoins::Relation.create(scope.klass, key, join_ids) split_scope.where_clause += scope.where_clause split_scope else diff --git a/lib/gitlab/action_cable/request_store_callbacks.rb b/lib/gitlab/action_cable/request_store_callbacks.rb new file mode 100644 index 00000000000..a9f30b0fc10 --- /dev/null +++ b/lib/gitlab/action_cable/request_store_callbacks.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module ActionCable + module RequestStoreCallbacks + def self.install + ::ActionCable::Server::Worker.set_callback :work, :around, &wrapper + ::ActionCable::Channel::Base.set_callback :subscribe, :around, &wrapper + ::ActionCable::Channel::Base.set_callback :unsubscribe, :around, &wrapper + end + + def self.wrapper + lambda do |_, inner| + ::Gitlab::WithRequestStore.with_request_store do + inner.call + end + end + end + end + end +end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index a7312ac759a..f6ee08defcf 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -338,6 +338,10 @@ module Gitlab Gitlab::PathRegex.repository_git_route_regex.match?(current_request.path) end + def git_lfs_request? + Gitlab::PathRegex.repository_git_lfs_route_regex.match?(current_request.path) + end + def archive_request? current_request.path.include?('/-/archive/') end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 1c5ded2e8ed..feb5fea4c85 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -54,7 +54,7 @@ module Gitlab Users::UpdateService.new(gl_user, user: gl_user).execute! - gl_user.block if block_after_save + gl_user.block_pending_approval if block_after_save log.info "(#{provider}) saving user #{auth_hash.email} from login with admin => #{gl_user.admin}, extern_uid => #{auth_hash.uid}" gl_user diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb index 6d1df95c66d..236c6b6eb9a 100644 --- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb +++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb @@ -73,7 +73,7 @@ module Gitlab # violation. We can safely roll-back the nested transaction and perform # a lookup instead to retrieve the record. def create_record - subject.transaction(requires_new: true) do + subject.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions InternalId.create!( **scope, usage: usage_value, diff --git a/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb b/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb new file mode 100644 index 00000000000..67f4690868e --- /dev/null +++ b/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillIterationCadenceIdForBoards + def perform(*args) + end + end + end +end + +Gitlab::BackgroundMigration::BackfillIterationCadenceIdForBoards.prepend_mod_with('Gitlab::BackgroundMigration::BackfillIterationCadenceIdForBoards') diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index f5c8796bd18..a9eaeb0562d 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -21,7 +21,7 @@ module Gitlab shard_id = shards.fetch(name, nil) return shard_id if shard_id.present? - Shard.transaction(requires_new: true) do + Shard.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions create!(name) end rescue ActiveRecord::RecordNotUnique diff --git a/lib/gitlab/background_migration/backfill_projects_with_coverage.rb b/lib/gitlab/background_migration/backfill_projects_with_coverage.rb new file mode 100644 index 00000000000..ca262c0bd59 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_projects_with_coverage.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill project_ci_feature_usages for a range of projects with coverage + class BackfillProjectsWithCoverage + class ProjectCiFeatureUsage < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'project_ci_feature_usages' + end + + COVERAGE_ENUM_VALUE = 1 + INSERT_DELAY_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + report_results = ActiveRecord::Base.connection.execute <<~SQL + SELECT DISTINCT project_id, default_branch + FROM ci_daily_build_group_report_results + WHERE id BETWEEN #{start_id} AND #{end_id} + SQL + + report_results.to_a.in_groups_of(sub_batch_size, false) do |batch| + ProjectCiFeatureUsage.insert_all(build_values(batch)) + + sleep INSERT_DELAY_SECONDS + end + end + + private + + def build_values(batch) + batch.map do |data| + { + project_id: data['project_id'], + feature: COVERAGE_ENUM_VALUE, + default_branch: data['default_branch'] + } + end + end + end + end +end diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb new file mode 100644 index 00000000000..31b5b5cdb73 --- /dev/null +++ b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to extract the project topics into a separate `topics` table + class ExtractProjectTopicsIntoSeparateTable + # Temporary AR table for tags + class Tag < ActiveRecord::Base + self.table_name = 'tags' + end + + # Temporary AR table for taggings + class Tagging < ActiveRecord::Base + self.table_name = 'taggings' + belongs_to :tag + end + + # Temporary AR table for topics + class Topic < ActiveRecord::Base + self.table_name = 'topics' + end + + # Temporary AR table for project topics + class ProjectTopic < ActiveRecord::Base + self.table_name = 'project_topics' + belongs_to :topic + end + + # Temporary AR table for projects + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + def perform(start_id, stop_id) + Tagging.includes(:tag).where(taggable_type: 'Project', id: start_id..stop_id).each do |tagging| + if Project.exists?(id: tagging.taggable_id) && tagging.tag + begin + topic = Topic.find_or_create_by(name: tagging.tag.name) + project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic) + + tagging.delete if project_topic.persisted? + rescue StandardError => e + Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id) + end + else + tagging.delete + end + end + + mark_job_as_succeeded(start_id, stop_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb index c096dae0631..3605b157f4f 100644 --- a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb +++ b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb @@ -14,7 +14,7 @@ module Gitlab mail( template_path: 'unconfirm_mailer', template_name: 'unconfirm_notification_email', - to: @user.notification_email, + to: @user.notification_email_or_default, subject: subject('GitLab email verification request') ) end diff --git a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb index e694e5359cd..7d150b9cd83 100644 --- a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb +++ b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb @@ -14,7 +14,7 @@ module Gitlab # The number of rows in merge_request_diff_commits to get in a single # query. - COMMIT_ROWS_PER_QUERY = 10_000 + COMMIT_ROWS_PER_QUERY = 1_000 # The number of rows in merge_request_diff_commits to update in a single # query. @@ -78,6 +78,8 @@ module Gitlab # rubocop: enable Style/Documentation def perform(start_id, stop_id) + return if already_processed?(start_id, stop_id) + # This Hash maps user names + emails to their corresponding rows in # merge_request_diff_commit_users. user_mapping = {} @@ -94,6 +96,13 @@ module Gitlab ) end + def already_processed?(start_id, stop_id) + Database::BackgroundMigrationJob + .for_migration_execution('MigrateMergeRequestDiffCommitUsers', [start_id, stop_id]) + .succeeded + .any? + end + # Returns the data we'll use to determine what merge_request_diff_commits # rows to update, and what data to use for populating their # commit_author_id and committer_id columns. diff --git a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb new file mode 100644 index 00000000000..43a7032e682 --- /dev/null +++ b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A background migration that finished any pending + # MigrateMergeRequestDiffCommitUsers jobs, and schedules new jobs itself. + # + # This migration exists so we can bypass rescheduling issues (e.g. jobs + # getting dropped after too many retries) that may occur when + # MigrateMergeRequestDiffCommitUsers jobs take longer than expected. + class StealMigrateMergeRequestDiffCommitUsers + def perform(start_id, stop_id) + MigrateMergeRequestDiffCommitUsers.new.perform(start_id, stop_id) + schedule_next_job + end + + def schedule_next_job + next_job = Database::BackgroundMigrationJob + .for_migration_class('MigrateMergeRequestDiffCommitUsers') + .pending + .first + + return unless next_job + + BackgroundMigrationWorker.perform_in( + 5.minutes, + 'StealMigrateMergeRequestDiffCommitUsers', + next_job.arguments + ) + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 2c60b2e36cb..e0eee64dc58 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -56,7 +56,7 @@ module Gitlab log_info(stage: "complete") - Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i) + Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) true end diff --git a/lib/gitlab/branch_push_merge_commit_analyzer.rb b/lib/gitlab/branch_push_merge_commit_analyzer.rb index a8f601f2451..ddf2086363c 100644 --- a/lib/gitlab/branch_push_merge_commit_analyzer.rb +++ b/lib/gitlab/branch_push_merge_commit_analyzer.rb @@ -114,7 +114,7 @@ module Gitlab # If child commit is a direct ancestor, its first parent is also a direct ancestor. # We assume direct ancestors matches the trail of the target branch over time, # This assumption is correct most of the time, especially for gitlab managed merges, - # but there are exception cases which can't be solved (https://stackoverflow.com/a/49754723/474597) + # but there are exception cases which can't be solved. def mark_all_direct_ancestors(commit) loop do commit = get_commit(commit.parent_ids.first) diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 89c85cb50be..947efee43a9 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -7,6 +7,10 @@ module Gitlab # The default timeout of the cache keys. TIMEOUT = 24.hours.to_i + LONGER_TIMEOUT = 72.hours.to_i + + SHORTER_TIMEOUT = 15.minutes.to_i + WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2] local existing = tonumber(redis.call("get", key)) diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb index 0538fe68474..fd5d701b858 100644 --- a/lib/gitlab/changelog/config.rb +++ b/lib/gitlab/changelog/config.rb @@ -34,17 +34,17 @@ module Gitlab '(?:-(?P<pre>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))' \ '?(?:\+(?P<meta>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$' - attr_accessor :date_format, :categories, :template, :tag_regex + attr_accessor :date_format, :categories, :template, :tag_regex, :always_credit_user_ids - def self.from_git(project) + def self.from_git(project, user = nil) if (yaml = project.repository.changelog_config) - from_hash(project, YAML.safe_load(yaml)) + from_hash(project, YAML.safe_load(yaml), user) else new(project) end end - def self.from_hash(project, hash) + def self.from_hash(project, hash, user = nil) config = new(project) if (date = hash['date_format']) @@ -72,6 +72,14 @@ module Gitlab config.tag_regex = regex end + config.always_credit_user_ids = Set.new + if (group_paths = Array(hash['include_groups'])) + group_paths.each do |group_path| + group = Group.find_by_full_path(group_path) + config.always_credit_user_ids.merge(group&.users_ids_of_direct_members&.compact) if user&.can?(:read_group, group) + end + end + config end @@ -89,7 +97,11 @@ module Gitlab end def contributor?(user) - @project.team.contributor?(user) + @project.team.contributor?(user&.id) + end + + def always_credit_author?(user) + always_credit_user_ids&.include?(user&.id) || false end def category(name) diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb index c0b6a5c5679..a0d598c7464 100644 --- a/lib/gitlab/changelog/release.rb +++ b/lib/gitlab/changelog/release.rb @@ -42,6 +42,7 @@ module Gitlab 'reference' => author.to_reference(full: true), 'contributor' => @config.contributor?(author) } + entry['author']['credit'] = entry['author']['contributor'] || @config.always_credit_author?(author) end if merge_request diff --git a/lib/gitlab/changelog/template.tpl b/lib/gitlab/changelog/template.tpl index 584939dff51..68c1c624394 100644 --- a/lib/gitlab/changelog/template.tpl +++ b/lib/gitlab/changelog/template.tpl @@ -4,7 +4,7 @@ {% each entries %} - [{{ title }}]({{ commit.reference }})\ -{% if author.contributor %} by {{ author.reference }}{% end %}\ +{% if author.credit %} by {{ author.reference }}{% end %}\ {% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %} {% end %} diff --git a/lib/gitlab/checks/base_single_checker.rb b/lib/gitlab/checks/base_single_checker.rb index f93902055c9..06519833d7c 100644 --- a/lib/gitlab/checks/base_single_checker.rb +++ b/lib/gitlab/checks/base_single_checker.rb @@ -30,5 +30,3 @@ module Gitlab end end end - -Gitlab::Checks::BaseSingleChecker.prepend_mod_with('Gitlab::Checks::BaseSingleChecker') diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 9ecc93f871b..3ce2e50c548 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -76,23 +76,33 @@ module Gitlab result end + def single_change_accesses + @single_changes_accesses ||= + changes.map do |change| + commits = + if change[:newrev].blank? || Gitlab::Git.blank_ref?(change[:newrev]) + [] + else + Gitlab::Lazy.new { commits_for(change[:newrev]) } + end + + Checks::SingleChangeAccess.new( + change, + user_access: user_access, + project: project, + protocol: protocol, + logger: logger, + commits: commits + ) + end + end + protected def single_access_checks! # Iterate over all changes to find if user allowed all of them to be applied - changes.each do |change| - commits = Gitlab::Lazy.new { commits_for(change[:newrev]) } if Feature.enabled?(:changes_batch_commits) - - # If user does not have access to make at least one change, cancel all - # push by allowing the exception to bubble up - Checks::SingleChangeAccess.new( - change, - user_access: user_access, - project: project, - protocol: protocol, - logger: logger, - commits: commits - ).validate! + single_change_accesses.each do |single_change_access| + single_change_access.validate! end end @@ -102,3 +112,5 @@ module Gitlab end end end + +Gitlab::Checks::ChangesAccess.prepend_mod_with('Gitlab::Checks::ChangesAccess') diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index d576953c1a0..3cfed8e5e2c 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -45,7 +45,7 @@ module Gitlab end def read_zip_file!(file_path) - if ::Feature.enabled?(:ci_new_artifact_file_reader, job.project, default_enabled: false) + if ::Feature.enabled?(:ci_new_artifact_file_reader, job.project, default_enabled: :yaml) read_with_new_artifact_file_reader(file_path) else read_with_legacy_artifact_file_reader(file_path) diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index eaaf9f69102..145772c7a92 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -53,7 +53,7 @@ module Gitlab description: 'Set retry default value.', inherit: false - entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings, + entry :tags, Entry::Tags, description: 'Set the default tags.', inherit: false diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index bd4d5f33689..f867189d521 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -85,7 +85,7 @@ module Gitlab description: 'Retry configuration for this job.', inherit: true - entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings, + entry :tags, Entry::Tags, description: 'Set the tags.', inherit: true diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 3543b5493bd..2549c35ebd6 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -16,6 +16,7 @@ module Gitlab PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit allow_failure when needs resource_group].freeze + MAX_NESTING_LEVEL = 10 included do validations do @@ -31,7 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true - validates :rules, nested_array_of_hashes: true + validates :rules, nested_array_of_hashes_or_arrays: { max_level: MAX_NESTING_LEVEL } validates :resource_group, type: String end end diff --git a/lib/gitlab/ci/config/entry/tags.rb b/lib/gitlab/ci/config/entry/tags.rb new file mode 100644 index 00000000000..ca3b48372e2 --- /dev/null +++ b/lib/gitlab/ci/config/entry/tags.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an array of tags. + # + class Tags < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + TAGS_LIMIT = 50 + + validations do + validates :config, array_of_strings: true + + validate do + next unless ::Feature.enabled?(:ci_build_tags_limit, default_enabled: :yaml) + + if config.is_a?(Array) && config.size >= TAGS_LIMIT + errors.add(:config, _("must be less than the limit of %{tag_limit} tags") % { tag_limit: TAGS_LIMIT }) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index bc03658aab8..7334a112ccf 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,8 +6,40 @@ module Gitlab VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC' VALID_SYNTAX_SAMPLE_CRON = '* * * * *' - def self.parse_natural(expression, cron_timezone = 'UTC') - new(Fugit::Nat.parse(expression)&.original, cron_timezone) + class << self + def parse_natural(expression, cron_timezone = 'UTC') + new(Fugit::Nat.parse(expression)&.original, cron_timezone) + end + + # This method generates compatible expressions that can be + # parsed by Fugit::Nat.parse to generate a cron line. + # It takes start date of the cron and cadence in the following format: + # cadence = { + # unit: 'day/week/month/year' + # duration: 1 + # } + def parse_natural_with_timestamp(starts_at, cadence) + case cadence[:unit] + when 'day' # Currently supports only 'every 1 day'. + "#{starts_at.min} #{starts_at.hour} * * *" + when 'week' # Currently supports only 'every 1 week'. + "#{starts_at.min} #{starts_at.hour} * * #{starts_at.wday}" + when 'month' + unless [1, 3, 6, 12].include?(cadence[:duration]) + raise NotImplementedError, "The cadence #{cadence} is not supported" + end + + "#{starts_at.min} #{starts_at.hour} #{starts_at.mday} #{fall_in_months(cadence[:duration], starts_at)} *" + when 'year' # Currently supports only 'every 1 year'. + "#{starts_at.min} #{starts_at.hour} #{starts_at.mday} #{starts_at.month} *" + else + raise NotImplementedError, "The cadence unit #{cadence[:unit]} is not implemented" + end + end + + def fall_in_months(offset, start_date) + (1..(12 / offset)).map { |i| start_date.next_month(offset * i).month }.join(',') + end end def initialize(cron, cron_timezone = 'UTC') diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index cd2c135dd7e..8c1067b9bc6 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -21,7 +21,7 @@ module Gitlab def initialize(project:, current_user:, sha: nil) @project = project @current_user = current_user - @sha = sha || project.repository.commit&.sha + @sha = sha || project&.repository&.commit&.sha end def validate(content, dry_run: false) diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 41acb4d5040..1cf4f252ab9 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -86,6 +86,7 @@ module Gitlab def create_finding(data, remediations = []) identifiers = create_identifiers(data['identifiers']) + flags = create_flags(data['flags']) links = create_links(data['links']) location = create_location(data['location'] || {}) signatures = create_signatures(tracking_data(data)) @@ -111,6 +112,7 @@ module Gitlab scanner: create_scanner(data['scanner']), scan: report&.scan, identifiers: identifiers, + flags: flags, links: links, remediations: remediations, raw_metadata: data.to_json, @@ -205,6 +207,18 @@ module Gitlab url: identifier['url'])) end + def create_flags(flags) + return [] unless flags.is_a?(Array) + + flags.map { |flag| create_flag(flag) }.compact + end + + def create_flag(flag) + return unless flag.is_a?(Hash) + + ::Gitlab::Ci::Reports::Security::Flag.new(type: flag['type'], origin: flag['origin'], description: flag['description']) + end + def create_links(links) return [] unless links.is_a?(Array) diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 3d92886cba8..143b930c669 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -12,7 +12,7 @@ module Gitlab end def initialize(report_type) - @report_type = report_type + @report_type = report_type.to_sym end delegate :validate, to: :schemer diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index d3bc3a38f1f..6feb693221b 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -5,9 +5,6 @@ module Gitlab module Pipeline module Chain class Build < Chain::Base - include Gitlab::Allowable - include Chain::Helpers - def perform! @pipeline.assign_attributes( source: @command.source, @@ -23,35 +20,12 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - locked: @command.project.default_pipeline_lock, - variables_attributes: variables_attributes - ) + locked: @command.project.default_pipeline_lock) end def break? @pipeline.errors.any? end - - private - - def variables_attributes - variables = Array(@command.variables_attributes) - - # We allow parent pipelines to pass variables to child pipelines since - # these variables are coming from internal configurations. We will check - # permissions to :set_pipeline_variables when those are injected upstream, - # to the parent pipeline. - # In other scenarios (e.g. multi-project pipelines or run pipeline via UI) - # the variables are provided from the outside and those should be guarded. - return variables if @command.creates_child_pipeline? - - if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project) - error("Insufficient permissions to set pipeline variables") - variables = [] - end - - variables - end end end end diff --git a/lib/gitlab/ci/pipeline/chain/build/associations.rb b/lib/gitlab/ci/pipeline/chain/build/associations.rb index eb49c56bcd7..b5d63691849 100644 --- a/lib/gitlab/ci/pipeline/chain/build/associations.rb +++ b/lib/gitlab/ci/pipeline/chain/build/associations.rb @@ -6,7 +6,25 @@ module Gitlab module Chain class Build class Associations < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + def perform! + assign_pipeline_variables + assign_source_pipeline + end + + def break? + @pipeline.errors.any? + end + + private + + def assign_pipeline_variables + @pipeline.variables_attributes = variables_attributes + end + + def assign_source_pipeline return unless @command.bridge @pipeline.build_source_pipeline( @@ -17,8 +35,45 @@ module Gitlab ) end - def break? - false + def variables_attributes + variables = Array(@command.variables_attributes) + variables = apply_permissions(variables) + validate_uniqueness(variables) + end + + def apply_permissions(variables) + # We allow parent pipelines to pass variables to child pipelines since + # these variables are coming from internal configurations. We will check + # permissions to :set_pipeline_variables when those are injected upstream, + # to the parent pipeline. + # In other scenarios (e.g. multi-project pipelines or run pipeline via UI) + # the variables are provided from the outside and those should be guarded. + return variables if @command.creates_child_pipeline? + + if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project) + error("Insufficient permissions to set pipeline variables") + variables = [] + end + + variables + end + + def validate_uniqueness(variables) + duplicated_keys = variables + .map { |var| var[:key] } + .tally + .filter_map { |key, count| key if count > 1 } + + if duplicated_keys.empty? + variables + else + error(duplicate_variables_message(duplicated_keys), config_error: true) + [] + end + end + + def duplicate_variables_message(keys) + "Duplicate variable #{'name'.pluralize(keys.size)}: #{keys.join(', ')}" end end end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 1c0dfbdbee3..f637001f9f8 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -7,15 +7,19 @@ module Gitlab class CancelPendingPipelines < Chain::Base include Chain::Helpers + BATCH_SIZE = 25 + + # rubocop: disable CodeReuse/ActiveRecord def perform! return unless project.auto_cancel_pending_pipelines? Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| - cancelables.find_each do |cancelable| - cancelable.auto_cancel_running(pipeline) + cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| + auto_cancel_interruptible_pipelines(cancelables_batch.ids) end end end + # rubocop: enable CodeReuse/ActiveRecord def break? false @@ -23,16 +27,21 @@ module Gitlab private - # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - project.all_pipelines.ci_and_parent_sources - .where(ref: pipeline.ref) - .where.not(id: pipeline.same_family_pipeline_ids) - .where.not(sha: project.commit(pipeline.ref).try(:id)) + project.all_pipelines.created_after(1.week.ago) + .ci_and_parent_sources + .for_ref(pipeline.ref) + .id_not_in(pipeline.same_family_pipeline_ids) + .where_not_sha(project.commit(pipeline.ref).try(:id)) .alive_or_scheduled + end + + def auto_cancel_interruptible_pipelines(pipeline_ids) + ::Ci::Pipeline + .id_in(pipeline_ids) .with_only_interruptible_builds + .each { |cancelable| cancelable.auto_cancel_running(pipeline) } end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 626eba97817..c9bc4ec411d 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -87,6 +87,13 @@ module Gitlab @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end + def observe_step_duration(step_class, duration) + if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml) + metrics.pipeline_creation_step_duration_histogram + .observe({ step: step_class.name }, duration.seconds) + end + end + def observe_creation_duration(duration) metrics.pipeline_creation_duration_histogram .observe({}, duration.seconds) diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb index 8a19e433483..092e7d43371 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb @@ -11,8 +11,12 @@ module Gitlab strong_memoize(:content) do next unless external_project_path? - path_file, path_project = ci_config_path.split('@', 2) - YAML.dump('include' => [{ 'project' => path_project, 'file' => path_file }]) + path_file, path_project, ref = extract_location_tokens + + config_location = { 'project' => path_project, 'file' => path_file } + config_location['ref'] = ref if ref.present? + + YAML.dump('include' => [config_location]) end end @@ -26,6 +30,18 @@ module Gitlab def external_project_path? ci_config_path =~ /\A.+(yml|yaml)@.+\z/ end + + # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname + def extract_location_tokens + path_file, path_project = ci_config_path.split('@', 2) + + if path_project.include? ":" + project, ref = path_project.split(':', 2) + [path_file, project, ref] + else + [path_file, path_project] + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index bbfc6759b35..845eb6c7a42 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -14,9 +14,16 @@ module Gitlab def build! @sequence.each do |step_class| + step_start = ::Gitlab::Metrics::System.monotonic_time step = step_class.new(@pipeline, @command) step.perform! + + @command.observe_step_duration( + step_class, + ::Gitlab::Metrics::System.monotonic_time - step_start + ) + break if step.break? end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 10de77afe74..28df9f5386c 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -4,6 +4,8 @@ module Gitlab module Ci module Pipeline class Metrics + extend Gitlab::Utils::StrongMemoize + def self.pipeline_creation_duration_histogram name = :gitlab_ci_pipeline_creation_duration_seconds comment = 'Pipeline creation duration' @@ -13,6 +15,17 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_creation_step_duration_histogram + strong_memoize(:pipeline_creation_step_histogram) do + name = :gitlab_ci_pipeline_creation_step_duration_seconds + comment = 'Duration of each pipeline creation step' + labels = { step: nil } + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 15.0, 20.0, 50.0, 240.0] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + def self.pipeline_security_orchestration_policy_processing_duration_histogram name = :gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds comment = 'Pipeline security orchestration policy processing duration' diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index c393fed26de..934bf22d8ad 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -15,12 +15,7 @@ module Gitlab @context = context @pipeline = context.pipeline @seed_attributes = attributes - @stages_for_needs_lookup = if Feature.enabled?(:ci_same_stage_job_needs, @pipeline.project, default_enabled: :yaml) - (previous_stages + [current_stage]).compact - else - previous_stages - end - + @stages_for_needs_lookup = (previous_stages + [current_stage]).compact @needs_attributes = dig(:needs_attributes) @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @@ -123,6 +118,8 @@ module Gitlab return { environment: nil } end + build.persisted_environment = environment + { deployment: Seed::Deployment.new(build, environment).to_resource, metadata_attributes: { diff --git a/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb index f8ea6d4184c..a29fef6eb34 100644 --- a/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb @@ -28,7 +28,16 @@ module Gitlab def expanded_resource_group_key strong_memoize(:expanded_resource_group_key) do - ExpandVariables.expand(resource_group_key, -> { processable.simple_variables }) + ExpandVariables.expand(resource_group_key, -> { variables }) + end + end + + def variables + processable.simple_variables.tap do |variables| + # Adding persisted environment variables + if processable.persisted_environment.present? + variables.concat(processable.persisted_environment.predefined_variables) + end end end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 859aeb35f26..7f45d626922 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -97,7 +97,9 @@ module Gitlab def observe_queue_size(size_proc, runner_type) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) - self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f) + size = size_proc.call.to_f + self.class.queue_size_total.observe({ runner_type: runner_type }, size) + self.class.current_queue_size.set({ runner_type: runner_type }, size) end def observe_queue_time(metric, runner_type) @@ -199,6 +201,15 @@ module Gitlab end end + def self.current_queue_size + strong_memoize(:current_queue_size) do + name = :gitlab_ci_current_queue_size + comment = 'Current size of initialized CI/CD builds queue' + + Gitlab::Metrics.gauge(name, comment) + end + end + def self.queue_iteration_duration_seconds strong_memoize(:queue_iteration_duration_seconds) do name = :gitlab_ci_queue_iteration_duration_seconds diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index dc1c51b3ed0..39531e12f69 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -10,6 +10,7 @@ module Gitlab attr_reader :compare_key attr_reader :confidence attr_reader :identifiers + attr_reader :flags attr_reader :links attr_reader :location attr_reader :metadata_version @@ -30,10 +31,11 @@ module Gitlab delegate :file_path, :start_line, :end_line, to: :location - def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers + @flags = flags @links = links @location = location @metadata_version = metadata_version @@ -58,6 +60,7 @@ module Gitlab compare_key confidence identifiers + flags links location metadata_version diff --git a/lib/gitlab/ci/reports/security/flag.rb b/lib/gitlab/ci/reports/security/flag.rb new file mode 100644 index 00000000000..7e6cc758864 --- /dev/null +++ b/lib/gitlab/ci/reports/security/flag.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Flag + attr_reader :type, :origin, :description + + MAP = { 'flagged-as-likely-false-positive' => :false_positive }.freeze + DEFAULT_FLAG_TYPE = :false_positive + + def flag_type + MAP.fetch(type, DEFAULT_FLAG_TYPE) + end + + def initialize(type: nil, origin: nil, description: nil) + @type = type + @origin = origin + @description = description + end + + def to_hash + { + flag_type: flag_type, + origin: origin, + description: description + }.compact + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index dbbb9a01dab..ee210e51232 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -32,7 +32,8 @@ module Gitlab user_blocked: 'pipeline user was blocked', ci_quota_exceeded: 'no more CI minutes available', no_matching_runner: 'no matching runner available', - trace_size_exceeded: 'log size limit exceeded' + trace_size_exceeded: 'log size limit exceeded', + builds_disabled: 'project builds are disabled' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml new file mode 100644 index 00000000000..9f0e9bcc1f2 --- /dev/null +++ b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml @@ -0,0 +1,87 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android.gitlab-ci.yml + +# Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny +# If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template. + +image: openjdk:11-jdk + +variables: + + # ANDROID_COMPILE_SDK is the version of Android you're compiling with. + # It should match compileSdkVersion. + ANDROID_COMPILE_SDK: "30" + + # ANDROID_BUILD_TOOLS is the version of the Android build tools you are using. + # It should match buildToolsVersion. + ANDROID_BUILD_TOOLS: "30.0.3" + + # It's what version of the command line tools we're going to download from the official site. + # Official Site-> https://developer.android.com/studio/index.html + # There, look down below at the cli tools only, sdk tools package is of format: + # commandlinetools-os_type-ANDROID_SDK_TOOLS_latest.zip + # when the script was last modified for latest compileSdkVersion, it was which is written down below + ANDROID_SDK_TOOLS: "7583922" + +# Packages installation before running script +before_script: + - apt-get --quiet update --yes + - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + + # Setup path as ANDROID_SDK_ROOT for moving/exporting the downloaded sdk into it + - export ANDROID_SDK_ROOT="${PWD}/android-home" + # Create a new directory at specified location + - install -d $ANDROID_SDK_ROOT + # Here we are installing androidSDK tools from official source, + # (the key thing here is the url from where you are downloading these sdk tool for command line, so please do note this url pattern there and here as well) + # after that unzipping those tools and + # then running a series of SDK manager commands to install necessary android SDK packages that'll allow the app to build + - wget --output-document=$ANDROID_SDK_ROOT/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip + # move to the archive at ANDROID_SDK_ROOT + - pushd $ANDROID_SDK_ROOT + - unzip -d cmdline-tools cmdline-tools.zip + - pushd cmdline-tools + # since commandline tools version 7583922 the root folder is named "cmdline-tools" so we rename it if necessary + - mv cmdline-tools tools || true + - popd + - popd + - export PATH=$PATH:${ANDROID_SDK_ROOT}/cmdline-tools/tools/bin/ + + # Nothing fancy here, just checking sdkManager version + - sdkmanager --version + + # use yes to accept all licenses + - yes | sdkmanager --licenses || true + - sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" + - sdkmanager "platform-tools" + - sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}" + + # Not necessary, but just for surity + - chmod +x ./gradlew + +# Basic android and gradle stuff +# Check linting +lintDebug: + interruptible: true + stage: build + script: + - ./gradlew -Pci --console=plain :app:lintDebug -PbuildDir=lint + +# Make Project +assembleDebug: + interruptible: true + stage: build + script: + - ./gradlew assembleDebug + artifacts: + paths: + - app/build/outputs/ + +# Run all tests, if any fails, interrupt the pipeline(fail it) +debugTests: + interruptible: true + stage: test + script: + - ./gradlew -Pci --console=plain :app:testDebug diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index cf99d722e4d..5efa557d7eb 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,9 @@ +variables: + AUTO_BUILD_IMAGE_VERSION: 'v1.0.0' + build: stage: build - image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v1.0.0' + image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' variables: DOCKER_TLS_CERTDIR: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml new file mode 100644 index 00000000000..6a3b0cfa9e7 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -0,0 +1,41 @@ +# WARNING: This latest template is for internal FEATURE-FLAG TESTING ONLY. +# It is not meant to be used with `include:`. +# This template is scheduled for removal when testing is complete: https://gitlab.com/gitlab-org/gitlab/-/issues/337987 + +variables: + AUTO_BUILD_IMAGE_VERSION: 'v1.3.1' + +build: + stage: build + image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' + variables: + DOCKER_TLS_CERTDIR: '' + services: + - name: 'docker:20.10.6-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] + script: + - | + if [[ -z "$CI_COMMIT_TAG" ]]; then + export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} + else + export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} + fi + - /build/build.sh + rules: + - if: '$BUILD_DISABLED' + when: never + - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +build_artifact: + stage: build + script: + - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n" + - exit 1 + rules: + - if: '$BUILD_DISABLED' + when: never + - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 43ecc4b96d5..00b771f1e5c 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24-gitlab.1" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 208951fa1a1..e0627b85aba 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,8 @@ +variables: + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0' + .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 5c466f0984c..2df985cfbb5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,8 @@ +variables: + AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0' + .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" dependencies: [] review: @@ -96,6 +99,8 @@ canary: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' @@ -125,6 +130,8 @@ canary: production: <<: *production_template rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$STAGING_ENABLED' @@ -141,6 +148,8 @@ production_manual: <<: *production_template allow_failure: false rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$INCREMENTAL_ROLLOUT_ENABLED' @@ -177,6 +186,8 @@ production_manual: resource_group: production allow_failure: true rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' @@ -190,6 +201,8 @@ production_manual: .timed_rollout_template: &timed_rollout_template <<: *rollout_template rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' diff --git a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml index a130b09c51a..1ec1aa60d88 100644 --- a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml @@ -41,9 +41,9 @@ echo "Adopting Helm v2 manifests from $release" # some resource kinds must be listed explicitly https://github.com/kubernetes/kubernetes/issues/42885 for name in $(kubectl -n "$KUBE_NAMESPACE" get all,ingress,daemonset -o name -l chart="$chart"); do - kubectl annotate --overwrite "$name" meta.helm.sh/release-name="$release" - kubectl annotate --overwrite "$name" meta.helm.sh/release-namespace="$KUBE_NAMESPACE" - kubectl label --overwrite "$name" app.kubernetes.io/managed-by=Helm + kubectl annotate -n "$KUBE_NAMESPACE" --overwrite "$name" meta.helm.sh/release-name="$release" + kubectl annotate -n "$KUBE_NAMESPACE" --overwrite "$name" meta.helm.sh/release-namespace="$KUBE_NAMESPACE" + kubectl label -n "$KUBE_NAMESPACE" --overwrite "$name" app.kubernetes.io/managed-by=Helm done done # migrate each release diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index c458ab6a00a..081a3a6cc78 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -16,6 +16,9 @@ stages: init: extends: .terraform:init +fmt: + extends: .terraform:fmt + validate: extends: .terraform:validate diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 39c3374e534..e696c75253e 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -20,7 +20,6 @@ cache: key: "${TF_ROOT}" paths: - ${TF_ROOT}/.terraform/ - - ${TF_ROOT}/.terraform.lock.hcl .init: &init stage: init diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index c30860ad174..3a70e6bc4b8 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -20,7 +20,6 @@ cache: key: "${TF_ROOT}" paths: - ${TF_ROOT}/.terraform/ - - ${TF_ROOT}/.terraform.lock.hcl .terraform:init: &terraform_init stage: init @@ -28,6 +27,14 @@ cache: - cd ${TF_ROOT} - gitlab-terraform init +.terraform:fmt: &terraform_fmt + stage: validate + needs: [] + script: + - cd ${TF_ROOT} + - gitlab-terraform fmt -check -recursive + allow_failure: true + .terraform:validate: &terraform_validate stage: validate script: diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index dd88953b9a4..841f17767eb 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -21,7 +21,7 @@ # # The deploy stage copies the exe and msi from build stage to a network drive # You need to have the network drive mapped as Local System user for gitlab-runner service to see it -# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), +# The best way to persist the mapping is via a scheduled task # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes # place project specific paths in variables to make the rest of the script more generic diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index f9798023838..72a94dcd412 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -4,6 +4,7 @@ module Gitlab module Ci class Trace include ::Gitlab::ExclusiveLeaseHelpers + include ::Gitlab::Utils::StrongMemoize include Checksummable LOCK_TTL = 10.minutes @@ -23,6 +24,8 @@ module Gitlab attr_reader :job delegate :old_trace, to: :job + delegate :can_attempt_archival_now?, :increment_archival_attempts!, + :archival_attempts_message, to: :trace_metadata def initialize(job) @job = job @@ -188,11 +191,7 @@ module Gitlab def unsafe_archive! raise ArchiveError, 'Job is not finished yet' unless job.complete? - if trace_artifact - unsafe_trace_cleanup! - - raise AlreadyArchivedError, 'Could not archive again' - end + unsafe_trace_conditionally_cleanup_before_retry! if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| @@ -212,12 +211,19 @@ module Gitlab end end - def unsafe_trace_cleanup! + def already_archived? + # TODO check checksum to ensure archive completed successfully + # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619 + trace_artifact.archived_trace_exists? + end + + def unsafe_trace_conditionally_cleanup_before_retry! return unless trace_artifact - if trace_artifact.archived_trace_exists? + if already_archived? # An archive already exists, so make sure to remove the trace chunks erase_trace_chunks! + raise AlreadyArchivedError, 'Could not archive again' else # An archive already exists, but its associated file does not, so remove it trace_artifact.destroy! @@ -251,11 +257,19 @@ module Gitlab File.open(path) do |stream| # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307 - job.create_job_artifacts_trace!( + trace_artifact = job.create_job_artifacts_trace!( project: job.project, file_type: :trace, file: stream, file_sha256: self.class.hexdigest(path)) + + trace_metadata.track_archival!(trace_artifact.id) + end + end + + def trace_metadata + strong_memoize(:trace_metadata) do + job.ensure_trace_metadata! end end diff --git a/lib/gitlab/ci/trace/backoff.rb b/lib/gitlab/ci/trace/backoff.rb new file mode 100644 index 00000000000..c13d88cced1 --- /dev/null +++ b/lib/gitlab/ci/trace/backoff.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # Trace::Backoff class is responsible for calculating a backoff value + # for when to be able to retry archiving a build's trace + # + # Because we're updating `last_archival_attempt_at` timestamp with every + # failed archival attempt, we need to be sure that sum of the backoff values + # for 1..MAX_ATTEMPTS is under 7 days(CHUNK_REDIS_TTL). + # + class Backoff + include Gitlab::Utils::StrongMemoize + + MAX_JITTER_VALUE = 4 + + attr_reader :archival_attempts + + def initialize(archival_attempts) + @archival_attempts = archival_attempts + end + + def value + (((chunks_ttl / (3.5 * max_attempts)) * archival_attempts) / 1.hour).hours + end + + # This formula generates an increasing delay between executions + # 9.6, 19.2, 28.8, 38.4, 48.0 + a random amount of time to + # change the order of execution for the jobs. + # With maximum value for each call to rand(4), this sums up to 6.8 days + # and with minimum values is 6 days. + # + def value_with_jitter + value + jitter + end + + private + + def jitter + rand(MAX_JITTER_VALUE).hours + end + + def chunks_ttl + ::Ci::BuildTraceChunks::RedisBase::CHUNK_REDIS_TTL + end + + def max_attempts + ::Ci::BuildTraceMetadata::MAX_ATTEMPTS + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index fdc598c025a..2d31049a0c9 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -3,7 +3,6 @@ module Gitlab module Ci class Trace - # This was inspired from: http://stackoverflow.com/a/10219411/1520132 class Stream BUFFER_SIZE = 4096 LIMIT_SIZE = 500.kilobytes diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index ef9ba1b73c7..09c75a2b3f1 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -10,7 +10,7 @@ module Gitlab def initialize(variables = [], errors = nil) @variables = [] - @variables_by_key = {} + @variables_by_key = Hash.new { |h, k| h[k] = [] } @errors = errors variables.each { |variable| self.append(variable) } @@ -19,7 +19,7 @@ module Gitlab def append(resource) item = Collection::Item.fabricate(resource) @variables.append(item) - @variables_by_key[item[:key]] = item + @variables_by_key[item[:key]] << item self end @@ -46,7 +46,12 @@ module Gitlab end def [](key) - @variables_by_key[key] + all(key)&.last + end + + def all(key) + vars = @variables_by_key[key] + vars unless vars.empty? end def size @@ -72,7 +77,7 @@ module Gitlab match = Regexp.last_match if match[:key] # we matched variable - if variable = @variables_by_key[match[:key]] + if variable = self[match[:key]] variable.value elsif keep_undefined match[0] @@ -85,7 +90,7 @@ module Gitlab end def sort_and_expand_all(project, keep_undefined: false) - return self if Feature.disabled?(:variable_inside_variable, project) + return self if Feature.disabled?(:variable_inside_variable, project, default_enabled: :yaml) sorted = Sort.new(self) return self.class.new(self, sorted.errors) unless sorted.valid? diff --git a/lib/gitlab/ci/variables/collection/sort.rb b/lib/gitlab/ci/variables/collection/sort.rb index 90a929b8a07..62637825c15 100644 --- a/lib/gitlab/ci/variables/collection/sort.rb +++ b/lib/gitlab/ci/variables/collection/sort.rb @@ -42,7 +42,7 @@ module Gitlab depends_on = var_item.depends_on return unless depends_on - depends_on.filter_map { |var_ref_name| @collection[var_ref_name] }.each(&block) + depends_on.filter_map { |var_ref_name| @collection.all(var_ref_name) }.flatten.each(&block) end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index c94fa84f608..1aa3dbc5e47 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -47,9 +47,7 @@ module Gitlab validate_job!(name, job) end - if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml) - YamlProcessor::Dag.check_circular_dependencies!(@jobs) - end + YamlProcessor::Dag.check_circular_dependencies!(@jobs) end def validate_job!(name, job) @@ -103,16 +101,8 @@ module Gitlab job_stage_index = stage_index(name) dependency_stage_index = stage_index(dependency) - if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml) - unless dependency_stage_index.present? && dependency_stage_index <= job_stage_index - error!("#{name} job: #{dependency_type} #{dependency} is not defined in current or prior stages") - end - else - # A dependency might be defined later in the configuration - # with a stage that does not exist - unless dependency_stage_index.present? && dependency_stage_index < job_stage_index - error!("#{name} job: #{dependency_type} #{dependency} is not defined in prior stages") - end + unless dependency_stage_index.present? && dependency_stage_index <= job_stage_index + error!("#{name} job: #{dependency_type} #{dependency} is not defined in current or prior stages") end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 13c6eaf4993..5d2bf3cfebf 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -90,14 +90,22 @@ module Gitlab end end - class NestedArrayOfHashesValidator < ArrayOfHashesValidator + class NestedArrayOfHashesOrArraysValidator < ArrayOfHashesValidator include NestedArrayHelpers def validate_each(record, attribute, value) - unless validate_nested_array(value, 1, &method(:validate_array_of_hashes)) + max_level = options.fetch(:max_level, 1) + + unless validate_nested_array(value, max_level, &method(:validate_hash)) record.errors.add(attribute, 'should be an array containing hashes and arrays of hashes') end end + + private + + def validate_hash(value) + value.is_a?(Hash) + end end class ArrayOrStringValidator < ActiveModel::EachValidator diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index 80c9abecd8e..f3a3818f010 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -9,9 +9,6 @@ module Gitlab include Gitlab::Utils::StrongMemoize - MAX_YAML_SIZE = 1.megabyte - MAX_YAML_DEPTH = 100 - def initialize(config, additional_permitted_classes: []) @config = YAML.safe_load(config, permitted_classes: [Symbol, *additional_permitted_classes], @@ -52,8 +49,8 @@ module Gitlab def deep_size strong_memoize(:deep_size) do Gitlab::Utils::DeepSize.new(@config, - max_size: MAX_YAML_SIZE, - max_depth: MAX_YAML_DEPTH) + max_size: Gitlab::CurrentSettings.current_application_settings.max_yaml_size_bytes, + max_depth: Gitlab::CurrentSettings.current_application_settings.max_yaml_depth) end end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 4e430d8937d..7d7c604d86a 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -33,6 +33,7 @@ module Gitlab .having(action: :commented) events = Event + .select(:project_id, :target_type, :action, :date, :total_amount) .from_union([repo_events, issue_events, mr_events, note_events]) .map(&:attributes) diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index 50a8f189df0..e30e526f017 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -16,6 +16,10 @@ module Gitlab def value raise NotImplementedError, "Expected #{self.name} to implement value" end + + private + + attr_reader :project, :options end end end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index ea16226a865..403cec5ed19 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -24,3 +24,5 @@ module Gitlab end end end + +Gitlab::CycleAnalytics::Summary::Deploy.prepend_mod_with('Gitlab::CycleAnalytics::Summary::Deploy') diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index acad19e096c..385ac40cf13 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,7 +2,11 @@ module Gitlab module Database + DATABASE_NAMES = %w[main ci].freeze + + MAIN_DATABASE_NAME = 'main' CI_DATABASE_NAME = 'ci' + DEFAULT_POOL_HEADROOM = 10 # This constant is used when renaming tables concurrently. # If you plan to rename a table using the `rename_table_safely` method, add your table here one milestone before the rename. @@ -59,6 +63,20 @@ module Gitlab DATABASES[PRIMARY_DATABASE_NAME] end + # We configure the database connection pool size automatically based on the + # configured concurrency. We also add some headroom, to make sure we don't + # run out of connections when more threads besides the 'user-facing' ones + # are running. + # + # Read more about this in + # doc/development/database/client_side_connection_pool.md + def self.default_pool_size + headroom = + (ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i + + Gitlab::Runtime.max_threads + headroom + end + def self.has_config?(database_name) Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s) end @@ -145,11 +163,19 @@ module Gitlab def self.allow_cross_joins_across_databases(url:) # this method is implemented in: # spec/support/database/prevent_cross_joins.rb + yield end + # This method will allow cross database modifications within the block + # Example: + # + # allow_cross_database_modification_within_transaction(url: 'url-to-an-issue') do + # create(:build) # inserts ci_build and project record in one transaction + # end def self.allow_cross_database_modification_within_transaction(url:) - # this method is implemented in: + # this method will be overridden in: # spec/support/database/cross_database_modification_check.rb + yield end def self.add_post_migrate_path_to_rails(force: false) @@ -172,14 +198,30 @@ module Gitlab ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) end - def self.db_config_name(ar_connection) - if ar_connection.respond_to?(:pool) && - ar_connection.pool.respond_to?(:db_config) && - ar_connection.pool.db_config.respond_to?(:name) - return ar_connection.pool.db_config.name - end + def self.db_config_for_connection(connection) + return unless connection - 'unknown' + # The LB connection proxy does not have a direct db_config + # that can be referenced + return if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) + + # During application init we might receive `NullPool` + return unless connection.respond_to?(:pool) && + connection.pool.respond_to?(:db_config) + + connection.pool.db_config + end + + # At the moment, the connection can only be retrieved by + # Gitlab::Database::LoadBalancer#read or #read_write or from the + # ActiveRecord directly. Therefore, if the load balancer doesn't + # recognize the connection, this method returns the primary role + # directly. In future, we may need to check for other sources. + # Expected returned names: + # main, main_replica, ci, ci_replica, unknown + def self.db_config_name(connection) + db_config = db_config_for_connection(connection) + db_config&.name || 'unknown' end def self.read_only? @@ -207,9 +249,13 @@ module Gitlab extend ActiveSupport::Concern class_methods do - # A monkeypatch over ActiveRecord::Base.transaction. - # It provides observability into transactional methods. + # A patch over ActiveRecord::Base.transaction that provides + # observability into transactional methods. def transaction(**options, &block) + if options[:requires_new] && connection.transaction_open? + ::Gitlab::Database::Metrics.subtransactions_increment(self.name) + end + ActiveSupport::Notifications.instrument('transaction.active_record', { connection: connection }) do super(**options, &block) end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index dff6376270a..2f990aba2fb 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -55,11 +55,14 @@ module Gitlab schema_creation = ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaCreation.new(ApplicationRecord.connection) definition = schema_creation.accept(create_index) - async_index = PostgresAsyncIndex.safe_find_or_create_by!(name: index_name) do |rec| + async_index = PostgresAsyncIndex.find_or_create_by!(name: index_name) do |rec| rec.table_name = table_name rec.definition = definition end + async_index.definition = definition + async_index.save! # No-op if definition is not changed + Gitlab::AppLogger.info( message: 'Prepared index for async creation', table_name: async_index.table_name, @@ -68,8 +71,6 @@ module Gitlab async_index end - private - def async_index_creation_available? ApplicationRecord.connection.table_exists?(:postgres_async_indexes) && Feature.enabled?(:database_async_index_creation, type: :ops) diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 03bd02d7554..32765cb6a56 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -4,6 +4,7 @@ module Gitlab module Database module BackgroundMigration class BatchedJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + include EachBatch include FromUnion self.table_name = :batched_background_migration_jobs diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 9d66824da51..d9fc2ea48f6 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -68,6 +68,17 @@ module Gitlab ) end + def retry_failed_jobs! + batched_jobs.failed.each_batch(of: 100) do |batch| + self.class.transaction do + batch.lock.each(&:split_and_retry!) + self.active! + end + end + + self.active! + end + def next_min_value last_job&.max_value&.next || min_value end diff --git a/lib/gitlab/database/connection.rb b/lib/gitlab/database/connection.rb index 21861e4fba8..cda6220ee6c 100644 --- a/lib/gitlab/database/connection.rb +++ b/lib/gitlab/database/connection.rb @@ -5,8 +5,6 @@ module Gitlab # Configuration settings and methods for interacting with a PostgreSQL # database, with support for multiple databases. class Connection - DEFAULT_POOL_HEADROOM = 10 - attr_reader :scope # Initializes a new `Database`. @@ -20,20 +18,6 @@ module Gitlab @open_transactions_baseline = 0 end - # We configure the database connection pool size automatically based on - # the configured concurrency. We also add some headroom, to make sure we - # don't run out of connections when more threads besides the 'user-facing' - # ones are running. - # - # Read more about this in - # doc/development/database/client_side_connection_pool.md - def default_pool_size - headroom = - (ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i - - Gitlab::Runtime.max_threads + headroom - end - def config # The result of this method must not be cached, as other methods may use # it after making configuration changes and expect those changes to be @@ -48,7 +32,7 @@ module Gitlab end def pool_size - config[:pool] || default_pool_size + config[:pool] || Database.default_pool_size end def username @@ -77,7 +61,9 @@ module Gitlab def db_config_with_default_pool_size db_config_object = scope.connection_db_config - config = db_config_object.configuration_hash.merge(pool: default_pool_size) + config = db_config_object + .configuration_hash + .merge(pool: Database.default_pool_size) ActiveRecord::DatabaseConfigurations::HashConfig.new( db_config_object.env_name, @@ -88,7 +74,16 @@ module Gitlab # Disables prepared statements for the current database connection. def disable_prepared_statements - scope.establish_connection(config.merge(prepared_statements: false)) + db_config_object = scope.connection_db_config + config = db_config_object.configuration_hash.merge(prepared_statements: false) + + hash_config = ActiveRecord::DatabaseConfigurations::HashConfig.new( + db_config_object.env_name, + db_config_object.name, + config + ) + + scope.establish_connection(hash_config) end # Check whether the underlying database is in read-only mode @@ -174,8 +169,11 @@ module Gitlab end def exists? - connection - + # We can't _just_ check if `connection` raises an error, as it will + # point to a `ConnectionProxy`, and obtaining those doesn't involve any + # database queries. So instead we obtain the database version, which is + # cached after the first call. + connection.schema_cache.database_version true rescue StandardError false @@ -189,6 +187,19 @@ module Gitlab row['system_identifier'] end + def pg_wal_lsn_diff(location1, location2) + lsn1 = connection.quote(location1) + lsn2 = connection.quote(location2) + + query = <<-SQL.squish + SELECT pg_wal_lsn_diff(#{lsn1}, #{lsn2}) + AS result + SQL + + row = connection.select_all(query).first + row['result'] if row + end + # @param [ActiveRecord::Connection] ar_connection # @return [String] def get_write_location(ar_connection) diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 08f108eb8e4..bbfbf83222f 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -36,89 +36,42 @@ module Gitlab # Returns a Hash containing the load balancing configuration. def self.configuration - Gitlab::Database.main.config[:load_balancing] || {} - end - - # Returns the maximum replica lag size in bytes. - def self.max_replication_difference - (configuration['max_replication_difference'] || 8.megabytes).to_i - end - - # Returns the maximum lag time for a replica. - def self.max_replication_lag_time - (configuration['max_replication_lag_time'] || 60.0).to_f - end - - # Returns the interval (in seconds) to use for checking the status of a - # replica. - def self.replica_check_interval - (configuration['replica_check_interval'] || 60).to_f - end - - # Returns the additional hosts to use for load balancing. - def self.hosts - configuration['hosts'] || [] - end - - def self.service_discovery_enabled? - configuration.dig('discover', 'record').present? - end - - def self.service_discovery_configuration - conf = configuration['discover'] || {} - - { - nameserver: conf['nameserver'] || 'localhost', - port: conf['port'] || 8600, - record: conf['record'], - record_type: conf['record_type'] || 'A', - interval: conf['interval'] || 60, - disconnect_timeout: conf['disconnect_timeout'] || 120, - use_tcp: conf['use_tcp'] || false - } - end - - def self.pool_size - Gitlab::Database.main.pool_size + @configuration ||= Configuration.for_model(ActiveRecord::Base) end # Returns true if load balancing is to be enabled. def self.enable? return false if Gitlab::Runtime.rake? - return false unless self.configured? - true + configured? end - # Returns true if load balancing has been configured. Since - # Sidekiq does not currently use load balancing, we - # may want Web application servers to detect replication lag by - # posting the write location of the database if load balancing is - # configured. def self.configured? - hosts.any? || service_discovery_enabled? + configuration.load_balancing_enabled? || + configuration.service_discovery_enabled? end def self.start_service_discovery - return unless service_discovery_enabled? + return unless configuration.service_discovery_enabled? - ServiceDiscovery.new(service_discovery_configuration).start + ServiceDiscovery + .new(proxy.load_balancer, **configuration.service_discovery) + .start end # Configures proxying of requests. - def self.configure_proxy(proxy = ConnectionProxy.new(hosts)) - ActiveRecord::Base.load_balancing_proxy = proxy + def self.configure_proxy + lb = LoadBalancer.new(configuration, primary_only: !enable?) + ActiveRecord::Base.load_balancing_proxy = ConnectionProxy.new(lb) # Populate service discovery immediately if it is configured - if service_discovery_enabled? - ServiceDiscovery.new(service_discovery_configuration).perform_service_discovery + if configuration.service_discovery_enabled? + ServiceDiscovery + .new(lb, **configuration.service_discovery) + .perform_service_discovery end end - def self.active_record_models - ActiveRecord::Base.descendants - end - DB_ROLES = [ ROLE_PRIMARY = :primary, ROLE_REPLICA = :replica, @@ -126,24 +79,12 @@ module Gitlab ].freeze # Returns the role (primary/replica) of the database the connection is - # connecting to. At the moment, the connection can only be retrieved by - # Gitlab::Database::LoadBalancer#read or #read_write or from the - # ActiveRecord directly. Therefore, if the load balancer doesn't - # recognize the connection, this method returns the primary role - # directly. In future, we may need to check for other sources. + # connecting to. def self.db_role_for_connection(connection) - return ROLE_UNKNOWN unless connection - - # The connection proxy does not have a role assigned - # as this is dependent on a execution context - return ROLE_UNKNOWN if connection.is_a?(ConnectionProxy) - - # During application init we might receive `NullPool` - return ROLE_UNKNOWN unless connection.respond_to?(:pool) && - connection.pool.respond_to?(:db_config) && - connection.pool.db_config.respond_to?(:name) + db_config = Database.db_config_for_connection(connection) + return ROLE_UNKNOWN unless db_config - if connection.pool.db_config.name.ends_with?(LoadBalancer::REPLICA_SUFFIX) + if db_config.name.ends_with?(LoadBalancer::REPLICA_SUFFIX) ROLE_REPLICA else ROLE_PRIMARY diff --git a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb new file mode 100644 index 00000000000..4feba989a0a --- /dev/null +++ b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + module ActionCableCallbacks + def self.install + ::ActionCable::Server::Worker.set_callback :work, :around, &wrapper + ::ActionCable::Channel::Base.set_callback :subscribe, :around, &wrapper + ::ActionCable::Channel::Base.set_callback :unsubscribe, :around, &wrapper + end + + def self.wrapper + lambda do |_, inner| + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! + + inner.call + ensure + ::Gitlab::Database::LoadBalancing.proxy.load_balancer.release_host + ::Gitlab::Database::LoadBalancing::Session.clear_session + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb new file mode 100644 index 00000000000..238f55fd98e --- /dev/null +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Configuration settings for a single LoadBalancer instance. + class Configuration + attr_accessor :hosts, :max_replication_difference, + :max_replication_lag_time, :replica_check_interval, + :service_discovery, :model + + # Creates a configuration object for the given ActiveRecord model. + def self.for_model(model) + cfg = model.connection_db_config.configuration_hash.deep_symbolize_keys + lb_cfg = cfg[:load_balancing] || {} + config = new(model) + + if (diff = lb_cfg[:max_replication_difference]) + config.max_replication_difference = diff + end + + if (lag = lb_cfg[:max_replication_lag_time]) + config.max_replication_lag_time = lag.to_f + end + + if (interval = lb_cfg[:replica_check_interval]) + config.replica_check_interval = interval.to_f + end + + if (hosts = lb_cfg[:hosts]) + config.hosts = hosts + end + + discover = lb_cfg[:discover] || {} + + # We iterate over the known/default keys so we don't end up with + # random keys in our configuration hash. + config.service_discovery.each do |key, _| + if (value = discover[key]) + config.service_discovery[key] = value + end + end + + config + end + + def initialize(model, hosts = []) + @max_replication_difference = 8.megabytes + @max_replication_lag_time = 60.0 + @replica_check_interval = 60.0 + @model = model + @hosts = hosts + @service_discovery = { + nameserver: 'localhost', + port: 8600, + record: nil, + record_type: 'A', + interval: 60, + disconnect_timeout: 120, + use_tcp: false + } + end + + def pool_size + # The pool size may change when booting up GitLab, as GitLab enforces + # a certain number of threads. If a Configuration is memoized, this + # can lead to incorrect pool sizes. + # + # To support this scenario, we always attempt to read the pool size + # from the model's configuration. + @model.connection_db_config.configuration_hash[:pool] || + Database.default_pool_size + end + + def load_balancing_enabled? + hosts.any? || service_discovery_enabled? + end + + def service_discovery_enabled? + service_discovery[:record].present? + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 938f4951532..1be63da8896 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -34,15 +34,15 @@ module Gitlab ).freeze # hosts - The hosts to use for load balancing. - def initialize(hosts = []) - @load_balancer = LoadBalancer.new(hosts) + def initialize(load_balancer) + @load_balancer = load_balancer end def select_all(arel, name = nil, binds = [], preparable: nil) if arel.respond_to?(:locked) && arel.locked # SELECT ... FOR UPDATE queries should be sent to the primary. - write_using_load_balancer(:select_all, arel, name, binds, - sticky: true) + current_session.write! + write_using_load_balancer(:select_all, arel, name, binds) else read_using_load_balancer(:select_all, arel, name, binds) end @@ -56,7 +56,8 @@ module Gitlab STICKY_WRITES.each do |name| define_method(name) do |*args, **kwargs, &block| - write_using_load_balancer(name, *args, sticky: true, **kwargs, &block) + current_session.write! + write_using_load_balancer(name, *args, **kwargs, &block) end end @@ -65,13 +66,20 @@ module Gitlab track_read_only_transaction! read_using_load_balancer(:transaction, *args, **kwargs, &block) else - write_using_load_balancer(:transaction, *args, sticky: true, **kwargs, &block) + current_session.write! + write_using_load_balancer(:transaction, *args, **kwargs, &block) end ensure untrack_read_only_transaction! end + def respond_to_missing?(name, include_private = false) + @load_balancer.read_write do |connection| + connection.respond_to?(name, include_private) + end + end + # Delegates all unknown messages to a read-write connection. def method_missing(...) if current_session.fallback_to_replicas_for_ambiguous_queries? @@ -102,18 +110,13 @@ module Gitlab # name - The name of the method to call on a connection object. # sticky - If set to true the session will stick to the master after # the write. - def write_using_load_balancer(name, *args, sticky: false, **kwargs, &block) + def write_using_load_balancer(...) if read_only_transaction? raise WriteInsideReadOnlyTransactionError, 'A write query is performed inside a read-only transaction' end @load_balancer.read_write do |connection| - # Sticking has to be enabled before calling the method. Not doing so - # could lead to methods called in a block still being performed on a - # secondary instead of on a primary (when necessary). - current_session.write! if sticky - - connection.send(name, *args, **kwargs, &block) + connection.send(...) end end diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb index 4c5357ae8e3..acd7df0a263 100644 --- a/lib/gitlab/database/load_balancing/host.rb +++ b/lib/gitlab/database/load_balancing/host.rb @@ -29,11 +29,15 @@ module Gitlab @host = host @port = port @load_balancer = load_balancer - @pool = load_balancer.create_replica_connection_pool(::Gitlab::Database::LoadBalancing.pool_size, host, port) + @pool = load_balancer.create_replica_connection_pool( + load_balancer.configuration.pool_size, + host, + port + ) @online = true @last_checked_at = Time.zone.now - interval = ::Gitlab::Database::LoadBalancing.replica_check_interval + interval = load_balancer.configuration.replica_check_interval @intervals = (interval..(interval * 2)).step(0.5).to_a end @@ -108,7 +112,7 @@ module Gitlab def replication_lag_below_threshold? if (lag_time = replication_lag_time) - lag_time <= ::Gitlab::Database::LoadBalancing.max_replication_lag_time + lag_time <= load_balancer.configuration.max_replication_lag_time else false end @@ -125,7 +129,7 @@ module Gitlab # only do this if we haven't replicated in a while so we only need # to connect to the primary when truly necessary. if (lag_size = replication_lag_size) - lag_size <= ::Gitlab::Database::LoadBalancing.max_replication_difference + lag_size <= load_balancer.configuration.max_replication_difference else false end @@ -159,8 +163,6 @@ module Gitlab def primary_write_location load_balancer.primary_write_location - ensure - load_balancer.release_primary_connection end def database_replica_location diff --git a/lib/gitlab/database/load_balancing/host_list.rb b/lib/gitlab/database/load_balancing/host_list.rb index aa731521732..fb3175c7d5d 100644 --- a/lib/gitlab/database/load_balancing/host_list.rb +++ b/lib/gitlab/database/load_balancing/host_list.rb @@ -35,12 +35,6 @@ module Gitlab def hosts=(hosts) @mutex.synchronize do - ::Gitlab::Database::LoadBalancing::Logger.info( - event: :host_list_update, - message: "Updating the host list for service discovery", - host_list_length: hosts.length, - old_host_list_length: @hosts.length - ) @hosts = hosts unsafe_shuffle end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index e3f5d0ac470..9b00b323301 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -12,12 +12,22 @@ module Gitlab REPLICA_SUFFIX = '_replica' - attr_reader :host_list - - # hosts - The hostnames/addresses of the additional databases. - def initialize(hosts = [], model = ActiveRecord::Base) - @model = model - @host_list = HostList.new(hosts.map { |addr| Host.new(addr, self) }) + attr_reader :host_list, :configuration + + # configuration - An instance of `LoadBalancing::Configuration` that + # contains the configuration details (such as the hosts) + # for this load balancer. + # primary_only - If set, the replicas are ignored and the primary is + # always used. + def initialize(configuration, primary_only: false) + @configuration = configuration + @primary_only = primary_only + @host_list = + if primary_only + HostList.new([PrimaryHost.new(self)]) + else + HostList.new(configuration.hosts.map { |addr| Host.new(addr, self) }) + end end def disconnect!(timeout: 120) @@ -169,7 +179,11 @@ module Gitlab when ActiveRecord::StatementInvalid, ActionView::Template::Error # After connecting to the DB Rails will wrap query errors using this # class. - connection_error?(error.cause) + if (cause = error.cause) + connection_error?(cause) + else + false + end when *CONNECTION_ERRORS true else @@ -213,26 +227,26 @@ module Gitlab .establish_connection(replica_db_config) end - private - # ActiveRecord::ConnectionAdapters::ConnectionHandler handles fetching, # and caching for connections pools for each "connection", so we # leverage that. def pool ActiveRecord::Base.connection_handler.retrieve_connection_pool( - @model.connection_specification_name, + @configuration.model.connection_specification_name, role: ActiveRecord::Base.writing_role, shard: ActiveRecord::Base.default_shard ) end + private + def ensure_caching! host.enable_query_cache! unless host.query_cache_enabled end def request_cache - base = RequestStore[:gitlab_load_balancer] ||= {} - base[pool] ||= {} + base = SafeRequestStore[:gitlab_load_balancer] ||= {} + base[self] ||= {} end end end diff --git a/lib/gitlab/database/load_balancing/primary_host.rb b/lib/gitlab/database/load_balancing/primary_host.rb new file mode 100644 index 00000000000..e379652c260 --- /dev/null +++ b/lib/gitlab/database/load_balancing/primary_host.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # A host that wraps the primary database connection. + # + # This class is used to always enable load balancing as if replicas exist, + # without the need for extra database connections. This ensures that code + # using the load balancer doesn't have to handle the case where load + # balancing is enabled, but no replicas have been configured (= the + # default case). + class PrimaryHost + def initialize(load_balancer) + @load_balancer = load_balancer + end + + def release_connection + # no-op as releasing primary connections isn't needed. + nil + end + + def enable_query_cache! + # This could mess up the primary connection, so we make this a no-op + nil + end + + def disable_query_cache! + # This could mess up the primary connection, so we make this a no-op + nil + end + + def query_cache_enabled + @load_balancer.pool.query_cache_enabled + end + + def connection + @load_balancer.pool.connection + end + + def disconnect!(timeout: 120) + nil + end + + def offline! + nil + end + + def online? + true + end + + def primary_write_location + @load_balancer.primary_write_location + end + + def database_replica_location + row = query_and_release(<<-SQL.squish) + SELECT pg_last_wal_replay_lsn()::text AS location + SQL + + row['location'] if row.any? + rescue *Host::CONNECTION_ERRORS + nil + end + + def caught_up?(_location) + true + end + + def query_and_release(sql) + connection.select_all(sql).first || {} + rescue StandardError + {} + ensure + release_connection + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 251961c8246..dfd4892371c 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -13,11 +13,17 @@ module Gitlab # balancer with said hosts. Requests may continue to use the old hosts # until they complete. class ServiceDiscovery + EmptyDnsResponse = Class.new(StandardError) + attr_reader :interval, :record, :record_type, :disconnect_timeout, :load_balancer MAX_SLEEP_ADJUSTMENT = 10 + MAX_DISCOVERY_RETRIES = 3 + + RETRY_DELAY_RANGE = (0.1..0.2).freeze + RECORD_TYPES = { 'A' => Net::DNS::A, 'SRV' => Net::DNS::SRV @@ -43,14 +49,14 @@ module Gitlab # use_tcp - Use TCP instaed of UDP to look up resources # load_balancer - The load balancer instance to use def initialize( + load_balancer, nameserver:, port:, record:, record_type: 'A', interval: 60, disconnect_timeout: 120, - use_tcp: false, - load_balancer: LoadBalancing.proxy.load_balancer + use_tcp: false ) @nameserver = nameserver @port = port @@ -76,15 +82,23 @@ module Gitlab end def perform_service_discovery - refresh_if_necessary - rescue StandardError => error - # Any exceptions that might occur should be reported to - # Sentry, instead of silently terminating this thread. - Gitlab::ErrorTracking.track_exception(error) - - Gitlab::AppLogger.error( - "Service discovery encountered an error: #{error.message}" - ) + MAX_DISCOVERY_RETRIES.times do + return refresh_if_necessary + rescue StandardError => error + # Any exceptions that might occur should be reported to + # Sentry, instead of silently terminating this thread. + Gitlab::ErrorTracking.track_exception(error) + + Gitlab::Database::LoadBalancing::Logger.error( + event: :service_discovery_failure, + message: "Service discovery encountered an error: #{error.message}", + host_list_length: load_balancer.host_list.length + ) + + # Slightly randomize the retry delay so that, in the case of a total + # dns outage, all starting services do not pressure the dns server at the same time. + sleep(rand(RETRY_DELAY_RANGE)) + end interval end @@ -99,7 +113,22 @@ module Gitlab current = addresses_from_load_balancer - replace_hosts(from_dns) if from_dns != current + if from_dns != current + ::Gitlab::Database::LoadBalancing::Logger.info( + event: :host_list_update, + message: "Updating the host list for service discovery", + host_list_length: from_dns.length, + old_host_list_length: current.length + ) + replace_hosts(from_dns) + else + ::Gitlab::Database::LoadBalancing::Logger.info( + event: :host_list_unchanged, + message: "Unchanged host list for service discovery", + host_list_length: from_dns.length, + old_host_list_length: current.length + ) + end interval end @@ -141,6 +170,8 @@ module Gitlab addresses_from_srv_record(response) end + raise EmptyDnsResponse if addresses.empty? + # Addresses are sorted so we can directly compare the old and new # addresses, without having to use any additional data structures. [new_wait_time_for(resources), addresses.sort] diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 0e36ebbc3ee..518a812b406 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -4,13 +4,15 @@ module Gitlab module Database module LoadBalancing class SidekiqClientMiddleware + include Gitlab::Utils::StrongMemoize + def call(worker_class, job, _queue, _redis_pool) # Mailers can't be constantized worker_class = worker_class.to_s.safe_constantize if load_balancing_enabled?(worker_class) job['worker_data_consistency'] = worker_class.get_data_consistency - set_data_consistency_location!(job) unless location_already_provided?(job) + set_data_consistency_locations!(job) unless job['wal_locations'] else job['worker_data_consistency'] = ::WorkerAttributes::DEFAULT_DATA_CONSISTENCY end @@ -27,16 +29,23 @@ module Gitlab worker_class.get_data_consistency_feature_flag_enabled? end - def set_data_consistency_location!(job) - if Session.current.use_primary? - job['database_write_location'] = load_balancer.primary_write_location - else - job['database_replica_location'] = load_balancer.host.database_replica_location - end + def set_data_consistency_locations!(job) + # Once we add support for multiple databases to our load balancer, we would use something like this: + # job['wal_locations'] = Gitlab::Database::DATABASES.transform_values do |connection| + # connection.load_balancer.primary_write_location + # end + # + job['wal_locations'] = { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location end - def location_already_provided?(job) - job['database_replica_location'] || job['database_write_location'] + def wal_location + strong_memoize(:wal_location) do + if Session.current.use_primary? + load_balancer.primary_write_location + else + load_balancer.host.database_replica_location + end + end end def load_balancer diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 0551750568a..15f8f0fb240 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -29,7 +29,7 @@ module Gitlab private def clear - load_balancer.release_host + release_hosts Session.clear_session end @@ -40,10 +40,11 @@ module Gitlab def select_load_balancing_strategy(worker_class, job) return :primary unless load_balancing_available?(worker_class) - location = job['database_write_location'] || job['database_replica_location'] - return :primary_no_wal unless location + wal_locations = get_wal_locations(job) + + return :primary_no_wal unless wal_locations - if replica_caught_up?(location) + if all_databases_has_replica_caught_up?(wal_locations) # Happy case: we can read from a replica. retried_before?(worker_class, job) ? :replica_retried : :replica elsif can_retry?(worker_class, job) @@ -55,6 +56,19 @@ module Gitlab end end + def get_wal_locations(job) + job['dedup_wal_locations'] || job['wal_locations'] || legacy_wal_location(job) + end + + # Already scheduled jobs could still contain legacy database write location. + # TODO: remove this in the next iteration + # https://gitlab.com/gitlab-org/gitlab/-/issues/338213 + def legacy_wal_location(job) + wal_location = job['database_write_location'] || job['database_replica_location'] + + { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location + end + def load_balancing_available?(worker_class) worker_class.include?(::ApplicationWorker) && worker_class.utilizes_load_balancing_capabilities? && @@ -75,12 +89,26 @@ module Gitlab job['retry_count'].nil? end - def load_balancer - LoadBalancing.proxy.load_balancer + def all_databases_has_replica_caught_up?(wal_locations) + wal_locations.all? do |_config_name, location| + # Once we add support for multiple databases to our load balancer, we would use something like this: + # Gitlab::Database::DATABASES[config_name].load_balancer.select_up_to_date_host(location) + load_balancer.select_up_to_date_host(location) + end end - def replica_caught_up?(location) - load_balancer.select_up_to_date_host(location) + def release_hosts + # Once we add support for multiple databases to our load balancer, we would use something like this: + # connection.load_balancer.primary_write_location + # + # Gitlab::Database::DATABASES.values.each do |connection| + # connection.load_balancer.release_host + # end + load_balancer.release_host + end + + def load_balancer + LoadBalancing.proxy.load_balancer end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb new file mode 100644 index 00000000000..b2248b0f4eb --- /dev/null +++ b/lib/gitlab/database/migration.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class Migration + module LockRetriesConcern + extend ActiveSupport::Concern + + class_methods do + def enable_lock_retries! + @enable_lock_retries = true # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def enable_lock_retries? + @enable_lock_retries + end + end + + delegate :enable_lock_retries?, to: :class + end + + # This implements a simple versioning scheme for migration helpers. + # + # We need to be able to version helpers, so we can change their behavior without + # altering the behavior of already existing migrations in incompatible ways. + # + # We can continue to change the behavior of helpers without bumping the version here, + # *if* the change is backwards-compatible. + # + # If not, we would typically override the helper method in a new MigrationHelpers::V[0-9]+ + # class and create a new entry with a bumped version below. + # + # We use major version bumps to indicate significant changes and minor version bumps + # to indicate backwards-compatible or otherwise minor changes (e.g. a Rails version bump). + # However, this hasn't been strictly formalized yet. + MIGRATION_CLASSES = { + 1.0 => Class.new(ActiveRecord::Migration[6.1]) do + include LockRetriesConcern + include Gitlab::Database::MigrationHelpers::V2 + end + }.freeze + + def self.[](version) + MIGRATION_CLASSES[version] || raise(ArgumentError, "Unknown migration version: #{version}") + end + + # The current version to be used in new migrations + def self.current_version + 1.0 + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 23d9b16dc09..9968096b1f6 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -73,6 +73,7 @@ module Gitlab end end + # @deprecated Use `create_table` in V2 instead # # Creates a new table, optionally allowing the caller to add check constraints to the table. # Aside from that addition, this method should behave identically to Rails' `create_table` method. @@ -380,6 +381,8 @@ module Gitlab # The timings can be controlled via the +timing_configuration+ parameter. # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+. # + # Note this helper uses subtransactions when run inside an already open transaction. + # # ==== Examples # # Invoking without parameters # with_lock_retries do @@ -411,7 +414,8 @@ module Gitlab raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) merged_args = { klass: self.class, - logger: Gitlab::BackgroundMigration::Logger + logger: Gitlab::BackgroundMigration::Logger, + allow_savepoints: true }.merge(kwargs) Gitlab::Database::WithLockRetries.new(**merged_args) @@ -600,17 +604,17 @@ module Gitlab # new_column - The name of the new column. # trigger_name - The name of the trigger to use (optional). def install_rename_triggers(table, old, new, trigger_name: nil) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table, connection: connection).create(old, new, trigger_name: trigger_name) end # Removes the triggers used for renaming a column concurrently. def remove_rename_triggers(table, trigger) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table, connection: connection).drop(trigger) end # Returns the (base) name to use for triggers when renaming columns. def rename_trigger_name(table, old, new) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table, connection: connection).name(old, new) end # Changes the type of a column concurrently. @@ -968,42 +972,7 @@ module Gitlab # columns - The name, or array of names, of the column(s) that we want to convert to bigint. # primary_key - The name of the primary key column (most often :id) def initialize_conversion_of_integer_to_bigint(table, columns, primary_key: :id) - unless table_exists?(table) - raise "Table #{table} does not exist" - end - - unless column_exists?(table, primary_key) - raise "Column #{primary_key} does not exist on #{table}" - end - - columns = Array.wrap(columns) - columns.each do |column| - next if column_exists?(table, column) - - raise ArgumentError, "Column #{column} does not exist on #{table}" - end - - check_trigger_permissions!(table) - - conversions = columns.to_h { |column| [column, convert_to_bigint_column(column)] } - - with_lock_retries do - conversions.each do |(source_column, temporary_name)| - column = column_for(table, source_column) - - if (column.name.to_s == primary_key.to_s) || !column.null - # If the column to be converted is either a PK or is defined as NOT NULL, - # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow - # That way, we skip the expensive validation step required to add - # a NOT NULL constraint at the end of the process - add_column(table, temporary_name, :bigint, default: column.default || 0, null: false) - else - add_column(table, temporary_name, :bigint, default: column.default) - end - end - - install_rename_triggers(table, conversions.keys, conversions.values) - end + create_temporary_columns_and_triggers(table, columns, primary_key: primary_key, data_type: :bigint) end # Reverts `initialize_conversion_of_integer_to_bigint` @@ -1019,6 +988,17 @@ module Gitlab temporary_columns.each { |column| remove_column(table, column) } end + alias_method :cleanup_conversion_of_integer_to_bigint, :revert_initialize_conversion_of_integer_to_bigint + + # Reverts `cleanup_conversion_of_integer_to_bigint` + # + # table - The name of the database table containing the columns + # columns - The name, or array of names, of the column(s) that we have converted to bigint. + # primary_key - The name of the primary key column (most often :id) + + def restore_conversion_of_integer_to_bigint(table, columns, primary_key: :id) + create_temporary_columns_and_triggers(table, columns, primary_key: primary_key, data_type: :int) + end # Backfills the new columns used in an integer-to-bigint conversion using background migrations. # @@ -1400,13 +1380,11 @@ into similar problems in the future (e.g. when new tables are created). # validate - Whether to validate the constraint in this call # def add_check_constraint(table, check, constraint_name, validate: true) - validate_check_constraint_name!(constraint_name) - # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. - if transaction_open? - raise 'add_check_constraint can not be run inside a transaction' - end + validate_not_in_transaction!(:add_check_constraint) + + validate_check_constraint_name!(constraint_name) if check_constraint_exists?(table, constraint_name) warning_message = <<~MESSAGE @@ -1451,6 +1429,10 @@ into similar problems in the future (e.g. when new tables are created). end def remove_check_constraint(table, constraint_name) + # This is technically not necessary, but aligned with add_check_constraint + # and allows us to continue use with_lock_retries here + validate_not_in_transaction!(:remove_check_constraint) + validate_check_constraint_name!(constraint_name) # DROP CONSTRAINT requires an EXCLUSIVE lock @@ -1649,6 +1631,45 @@ into similar problems in the future (e.g. when new tables are created). private + def create_temporary_columns_and_triggers(table, columns, primary_key: :id, data_type: :bigint) + unless table_exists?(table) + raise "Table #{table} does not exist" + end + + unless column_exists?(table, primary_key) + raise "Column #{primary_key} does not exist on #{table}" + end + + columns = Array.wrap(columns) + columns.each do |column| + next if column_exists?(table, column) + + raise ArgumentError, "Column #{column} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + conversions = columns.to_h { |column| [column, convert_to_bigint_column(column)] } + + with_lock_retries do + conversions.each do |(source_column, temporary_name)| + column = column_for(table, source_column) + + if (column.name.to_s == primary_key.to_s) || !column.null + # If the column to be converted is either a PK or is defined as NOT NULL, + # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow + # That way, we skip the expensive validation step required to add + # a NOT NULL constraint at the end of the process + add_column(table, temporary_name, data_type, default: column.default || 0, null: false) + else + add_column(table, temporary_name, data_type, default: column.default) + end + end + + install_rename_triggers(table, conversions.keys, conversions.values) + end + end + def validate_check_constraint_name!(constraint_name) if constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters" diff --git a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb index eecf96acb30..d9ef5ab462e 100644 --- a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb +++ b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb @@ -31,10 +31,8 @@ module Gitlab namespace_options = options.merge(null: true, default: nil) - with_lock_retries do - add_column(:namespace_settings, setting_name, type, namespace_options) - add_column(:namespace_settings, lock_column_name, :boolean, default: false, null: false) - end + add_column(:namespace_settings, setting_name, type, namespace_options) + add_column(:namespace_settings, lock_column_name, :boolean, default: false, null: false) add_column(:application_settings, setting_name, type, options) add_column(:application_settings, lock_column_name, :boolean, default: false, null: false) @@ -43,10 +41,8 @@ module Gitlab def remove_cascading_namespace_setting(setting_name) lock_column_name = "lock_#{setting_name}".to_sym - with_lock_retries do - remove_column(:namespace_settings, setting_name) if column_exists?(:namespace_settings, setting_name) - remove_column(:namespace_settings, lock_column_name) if column_exists?(:namespace_settings, lock_column_name) - end + remove_column(:namespace_settings, setting_name) if column_exists?(:namespace_settings, setting_name) + remove_column(:namespace_settings, lock_column_name) if column_exists?(:namespace_settings, lock_column_name) remove_column(:application_settings, setting_name) if column_exists?(:application_settings, setting_name) remove_column(:application_settings, lock_column_name) if column_exists?(:application_settings, lock_column_name) diff --git a/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb b/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb new file mode 100644 index 00000000000..30601bffd7a --- /dev/null +++ b/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module LooseForeignKeyHelpers + include Gitlab::Database::SchemaHelpers + + DELETED_RECORDS_INSERT_FUNCTION_NAME = 'insert_into_loose_foreign_keys_deleted_records' + + def track_record_deletions(table) + execute(<<~SQL) + CREATE TRIGGER #{record_deletion_trigger_name(table)} + AFTER DELETE ON #{table} REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT + EXECUTE FUNCTION #{DELETED_RECORDS_INSERT_FUNCTION_NAME}(); + SQL + end + + def untrack_record_deletions(table) + drop_trigger(table, record_deletion_trigger_name(table)) + end + + private + + def record_deletion_trigger_name(table) + "#{table}_loose_fk_trigger" + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb index f20a9b30fa7..0e7f6075196 100644 --- a/lib/gitlab/database/migration_helpers/v2.rb +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -6,6 +6,118 @@ module Gitlab module V2 include Gitlab::Database::MigrationHelpers + # Superseded by `create_table` override below + def create_table_with_constraints(*_) + raise <<~EOM + #create_table_with_constraints is not supported anymore - use #create_table instead, for example: + + create_table :db_guides do |t| + t.bigint :stars, default: 0, null: false + t.text :title, limit: 128 + t.text :notes, limit: 1024 + + t.check_constraint 'stars > 1000', name: 'so_many_stars' + end + + See https://docs.gitlab.com/ee/development/database/strings_and_the_text_data_type.html + EOM + end + + # Creates a new table, optionally allowing the caller to add text limit constraints to the table. + # This method only extends Rails' `create_table` method + # + # Example: + # + # create_table :db_guides do |t| + # t.bigint :stars, default: 0, null: false + # t.text :title, limit: 128 + # t.text :notes, limit: 1024 + # + # t.check_constraint 'stars > 1000', name: 'so_many_stars' + # end + # + # See Rails' `create_table` for more info on the available arguments. + # + # When adding foreign keys to other tables, consider wrapping the call into a with_lock_retries block + # to avoid traffic stalls. + def create_table(table_name, *args, **kwargs, &block) + helper_context = self + + super do |t| + t.define_singleton_method(:text) do |column_name, **kwargs| + limit = kwargs.delete(:limit) + + super(column_name, **kwargs) + + if limit + # rubocop:disable GitlabSecurity/PublicSend + name = helper_context.send(:text_limit_name, table_name, column_name) + # rubocop:enable GitlabSecurity/PublicSend + + column_name = helper_context.quote_column_name(column_name) + definition = "char_length(#{column_name}) <= #{limit}" + + t.check_constraint(definition, name: name) + end + end + + t.instance_eval(&block) unless block.nil? + end + end + + # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts. + # The timings can be controlled via the +timing_configuration+ parameter. + # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+. + # + # In order to retry the block, the method wraps the block into a transaction. + # + # When called inside an open transaction it will execute the block directly if lock retries are enabled + # with `enable_lock_retries!` at migration level, otherwise it will raise an error. + # + # ==== Examples + # # Invoking without parameters + # with_lock_retries do + # drop_table :my_table + # end + # + # # Invoking with custom +timing_configuration+ + # t = [ + # [1.second, 1.second], + # [2.seconds, 2.seconds] + # ] + # + # with_lock_retries(timing_configuration: t) do + # drop_table :my_table # this will be retried twice + # end + # + # # Disabling the retries using an environment variable + # > export DISABLE_LOCK_RETRIES=true + # + # with_lock_retries do + # drop_table :my_table # one invocation, it will not retry at all + # end + # + # ==== Parameters + # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` + # * +logger+ - [Gitlab::JsonLogger] + # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` + def with_lock_retries(*args, **kwargs, &block) + if transaction_open? + if enable_lock_retries? + Gitlab::AppLogger.warn 'Lock retries already enabled, executing the block directly' + yield + else + raise <<~EOF + #{__callee__} can not be run inside an already open transaction + + Use migration-level lock retries instead, see https://docs.gitlab.com/ee/development/migration_style_guide.html#retry-mechanism-when-acquiring-database-locks + EOF + end + else + super(*args, **kwargs.merge(allow_savepoints: false), &block) + end + end + # Renames a column without requiring downtime. # # Concurrent renames work by using database triggers to ensure both the diff --git a/lib/gitlab/database/migrations/lock_retry_mixin.rb b/lib/gitlab/database/migrations/lock_retry_mixin.rb new file mode 100644 index 00000000000..fff0f35e33c --- /dev/null +++ b/lib/gitlab/database/migrations/lock_retry_mixin.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module LockRetryMixin + module ActiveRecordMigrationProxyLockRetries + def migration_class + migration.class + end + + def enable_lock_retries? + # regular AR migrations don't have this, + # only ones inheriting from Gitlab::Database::Migration have + return false unless migration.respond_to?(:enable_lock_retries?) + + migration.enable_lock_retries? + end + end + + module ActiveRecordMigratorLockRetries + # We patch the original method to start a transaction + # using the WithLockRetries methodology for the whole migration. + def ddl_transaction(migration, &block) + if use_transaction?(migration) && migration.enable_lock_retries? + Gitlab::Database::WithLockRetries.new( + klass: migration.migration_class, + logger: Gitlab::BackgroundMigration::Logger + ).run(raise_on_exhaustion: false, &block) + else + super + end + end + end + + def self.patch! + ActiveRecord::MigrationProxy.prepend(ActiveRecordMigrationProxyLockRetries) + ActiveRecord::Migrator.prepend(ActiveRecordMigratorLockRetries) + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb new file mode 100644 index 00000000000..bbde2063c41 --- /dev/null +++ b/lib/gitlab/database/partitioning.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + def self.register_models(models) + registered_models.merge(models) + end + + def self.registered_models + @registered_models ||= Set.new + end + + def self.sync_partitions(models_to_sync = registered_models) + MultiDatabasePartitionManager.new(models_to_sync).sync_partitions + end + end + end +end diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index 7992c2fdaa7..4cdde5bf2f1 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -4,17 +4,18 @@ module Gitlab module Database module Partitioning class MonthlyStrategy - attr_reader :model, :partitioning_key, :retain_for + attr_reader :model, :partitioning_key, :retain_for, :retain_non_empty_partitions # We create this many partitions in the future HEADROOM = 6.months delegate :table_name, to: :model - def initialize(model, partitioning_key, retain_for: nil) + def initialize(model, partitioning_key, retain_for: nil, retain_non_empty_partitions: false) @model = model @partitioning_key = partitioning_key @retain_for = retain_for + @retain_non_empty_partitions = retain_non_empty_partitions end def current_partitions @@ -29,7 +30,10 @@ module Gitlab end def extra_partitions - current_partitions - desired_partitions + partitions = current_partitions - desired_partitions + partitions.reject!(&:holds_data?) if retain_non_empty_partitions + + partitions end private diff --git a/lib/gitlab/database/partitioning/multi_database_partition_manager.rb b/lib/gitlab/database/partitioning/multi_database_partition_manager.rb new file mode 100644 index 00000000000..5a93e3fb1fb --- /dev/null +++ b/lib/gitlab/database/partitioning/multi_database_partition_manager.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class MultiDatabasePartitionManager + def initialize(models) + @models = models + end + + def sync_partitions + Gitlab::AppLogger.info(message: "Syncing dynamic postgres partitions") + + models.each do |model| + Gitlab::Database::SharedModel.using_connection(model.connection) do + Gitlab::AppLogger.debug(message: "Switched database connection", + connection_name: connection_name, + table_name: model.table_name) + + PartitionManager.new(model).sync_partitions + end + end + + Gitlab::AppLogger.info(message: "Finished sync of dynamic postgres partitions") + end + + private + + attr_reader :models + + def connection_name + Gitlab::Database::SharedModel.connection.pool.db_config.name + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index 7e433ecdd39..8742c0ff166 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -6,60 +6,49 @@ module Gitlab class PartitionManager UnsafeToDetachPartitionError = Class.new(StandardError) - def self.register(model) - raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy) - - models << model - end - - def self.models - @models ||= Set.new - end - LEASE_TIMEOUT = 1.minute MANAGEMENT_LEASE_KEY = 'database_partition_management_%s' RETAIN_DETACHED_PARTITIONS_FOR = 1.week - attr_reader :models - - def initialize(models = self.class.models) - @models = models + def initialize(model) + @model = model end def sync_partitions - Gitlab::AppLogger.info("Checking state of dynamic postgres partitions") + Gitlab::AppLogger.info(message: "Checking state of dynamic postgres partitions", table_name: model.table_name) - models.each do |model| - # Double-checking before getting the lease: - # The prevailing situation is no missing partitions and no extra partitions - next if missing_partitions(model).empty? && extra_partitions(model).empty? + # Double-checking before getting the lease: + # The prevailing situation is no missing partitions and no extra partitions + return if missing_partitions.empty? && extra_partitions.empty? - only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do - partitions_to_create = missing_partitions(model) - create(partitions_to_create) unless partitions_to_create.empty? + only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do + partitions_to_create = missing_partitions + create(partitions_to_create) unless partitions_to_create.empty? - if Feature.enabled?(:partition_pruning, default_enabled: :yaml) - partitions_to_detach = extra_partitions(model) - detach(partitions_to_detach) unless partitions_to_detach.empty? - end + if Feature.enabled?(:partition_pruning, default_enabled: :yaml) + partitions_to_detach = extra_partitions + detach(partitions_to_detach) unless partitions_to_detach.empty? end - rescue StandardError => e - Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)", - table_name: model.table_name, - exception_class: e.class, - exception_message: e.message) end + rescue StandardError => e + Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)", + table_name: model.table_name, + exception_class: e.class, + exception_message: e.message) end private - def missing_partitions(model) + attr_reader :model + delegate :connection, to: :model + + def missing_partitions return [] unless connection.table_exists?(model.table_name) model.partitioning_strategy.missing_partitions end - def extra_partitions(model) + def extra_partitions return [] unless connection.table_exists?(model.table_name) model.partitioning_strategy.extra_partitions @@ -74,8 +63,9 @@ module Gitlab end def create(partitions) - connection.transaction do - with_lock_retries do + # with_lock_retries starts a requires_new transaction most of the time, but not on the last iteration + with_lock_retries do + connection.transaction(requires_new: false) do # so we open a transaction here if not already in progress partitions.each do |partition| connection.execute partition.to_sql @@ -88,8 +78,9 @@ module Gitlab end def detach(partitions) - connection.transaction do - with_lock_retries do + # with_lock_retries starts a requires_new transaction most of the time, but not on the last iteration + with_lock_retries do + connection.transaction(requires_new: false) do # so we open a transaction here if not already in progress partitions.each { |p| detach_one_partition(p) } end end @@ -119,13 +110,10 @@ module Gitlab def with_lock_retries(&block) Gitlab::Database::WithLockRetries.new( klass: self.class, - logger: Gitlab::AppLogger + logger: Gitlab::AppLogger, + connection: connection ).run(&block) end - - def connection - ActiveRecord::Base.connection - end end end end diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb index 6963ecd2cc1..e5b561fc447 100644 --- a/lib/gitlab/database/partitioning/partition_monitoring.rb +++ b/lib/gitlab/database/partitioning/partition_monitoring.rb @@ -6,7 +6,7 @@ module Gitlab class PartitionMonitoring attr_reader :models - def initialize(models = PartitionManager.models) + def initialize(models = Gitlab::Database::Partitioning.registered_models) @models = models end diff --git a/lib/gitlab/database/partitioning/time_partition.rb b/lib/gitlab/database/partitioning/time_partition.rb index 1221f042530..e09ca483549 100644 --- a/lib/gitlab/database/partitioning/time_partition.rb +++ b/lib/gitlab/database/partitioning/time_partition.rb @@ -69,6 +69,10 @@ module Gitlab partition_name <=> other.partition_name end + def holds_data? + conn.execute("SELECT 1 FROM #{fully_qualified_partition} LIMIT 1").ntuples > 0 + end + private def date_or_nil(obj) diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index f1aa7871245..bd8ed677d77 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -6,6 +6,8 @@ module Gitlab module ForeignKeyHelpers include ::Gitlab::Database::SchemaHelpers + ERROR_SCOPE = 'foreign keys' + # Adds a foreign key with only minimal locking on the tables involved. # # In concept it works similarly to add_concurrent_foreign_key, but we have @@ -32,6 +34,8 @@ module Gitlab # name - The name of the foreign key. # def add_concurrent_partitioned_foreign_key(source, target, column:, on_delete: :cascade, name: nil) + assert_not_in_transaction_block(scope: ERROR_SCOPE) + partition_options = { column: column, on_delete: on_delete, diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index c0cc97de276..c9a3b5caf79 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -7,6 +7,8 @@ module Gitlab include Gitlab::Database::MigrationHelpers include Gitlab::Database::SchemaHelpers + ERROR_SCOPE = 'index' + # Concurrently creates a new index on a partitioned table. In concept this works similarly to # `add_concurrent_index`, and won't block reads or writes on the table while the index is being built. # @@ -21,6 +23,8 @@ module Gitlab # # See Rails' `add_index` for more info on the available arguments. def add_concurrent_partitioned_index(table_name, column_names, options = {}) + assert_not_in_transaction_block(scope: ERROR_SCOPE) + raise ArgumentError, 'A name is required for indexes added to partitioned tables' unless options[:name] partitioned_table = find_partitioned_table(table_name) @@ -57,6 +61,8 @@ module Gitlab # # remove_concurrent_partitioned_index_by_name :users, 'index_name_goes_here' def remove_concurrent_partitioned_index_by_name(table_name, index_name) + assert_not_in_transaction_block(scope: ERROR_SCOPE) + find_partitioned_table(table_name) unless index_name_exists?(table_name, index_name) diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 9ccbdc9930e..0dc9f92e4c8 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -431,7 +431,7 @@ module Gitlab replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(original_table_name.to_s, replacement_table_name, replaced_table_name, primary_key_name) - with_lock_retries do + transaction do drop_sync_trigger(original_table_name) replace_table.perform do |sql| diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index 94f74724295..72640f8785d 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class PostgresForeignKey < ApplicationRecord + class PostgresForeignKey < SharedModel self.primary_key = :oid scope :by_referenced_table_identifier, ->(identifier) do diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index 7da60d8375d..eb080904f73 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class PostgresPartition < ActiveRecord::Base + class PostgresPartition < SharedModel self.primary_key = :identifier belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb index 5d2eaa22ee4..3bd342f940f 100644 --- a/lib/gitlab/database/postgres_partitioned_table.rb +++ b/lib/gitlab/database/postgres_partitioned_table.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class PostgresPartitionedTable < ActiveRecord::Base + class PostgresPartitionedTable < SharedModel DYNAMIC_PARTITION_STRATEGIES = %w[range list].freeze self.primary_key = :identifier diff --git a/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb index a2e7f4befab..59ca06b5aca 100644 --- a/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb +++ b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb @@ -7,7 +7,7 @@ module Gitlab extend ActiveSupport::Concern def dump_schema_information # :nodoc: - Gitlab::Database::SchemaMigrations.touch_all(self) + Gitlab::Database::SchemaMigrations.touch_all(self) if Gitlab.dev_or_test_env? nil end diff --git a/lib/gitlab/database/rename_table_helpers.rb b/lib/gitlab/database/rename_table_helpers.rb index 7f5af038c6d..e881c0e5455 100644 --- a/lib/gitlab/database/rename_table_helpers.rb +++ b/lib/gitlab/database/rename_table_helpers.rb @@ -4,27 +4,27 @@ module Gitlab module Database module RenameTableHelpers def rename_table_safely(old_table_name, new_table_name) - with_lock_retries do + transaction do rename_table(old_table_name, new_table_name) execute("CREATE VIEW #{old_table_name} AS SELECT * FROM #{new_table_name}") end end def undo_rename_table_safely(old_table_name, new_table_name) - with_lock_retries do + transaction do execute("DROP VIEW IF EXISTS #{old_table_name}") rename_table(new_table_name, old_table_name) end end def finalize_table_rename(old_table_name, new_table_name) - with_lock_retries do + transaction do execute("DROP VIEW IF EXISTS #{old_table_name}") end end def undo_finalize_table_rename(old_table_name, new_table_name) - with_lock_retries do + transaction do execute("CREATE VIEW #{old_table_name} AS SELECT * FROM #{new_table_name}") end end diff --git a/lib/gitlab/database/schema_migrations/context.rb b/lib/gitlab/database/schema_migrations/context.rb index 35105121bbd..a95f85c6bef 100644 --- a/lib/gitlab/database/schema_migrations/context.rb +++ b/lib/gitlab/database/schema_migrations/context.rb @@ -6,7 +6,7 @@ module Gitlab class Context attr_reader :connection - DEFAULT_SCHEMA_MIGRATIONS_PATH = "db/schema_migrations" + class_attribute :default_schema_migrations_path, default: 'db/schema_migrations' def initialize(connection) @connection = connection @@ -30,7 +30,7 @@ module Gitlab end def database_schema_migrations_path - @connection.pool.db_config.configuration_hash[:schema_migrations_path] || DEFAULT_SCHEMA_MIGRATIONS_PATH + @connection.pool.db_config.configuration_hash[:schema_migrations_path] || self.class.default_schema_migrations_path end end end diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb new file mode 100644 index 00000000000..8f256758961 --- /dev/null +++ b/lib/gitlab/database/shared_model.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class SharedModel < ActiveRecord::Base + self.abstract_class = true + + class << self + def using_connection(connection) + raise 'cannot nest connection overrides for shared models' unless overriding_connection.nil? + + self.overriding_connection = connection + + yield + ensure + self.overriding_connection = nil + end + + def connection + if connection = self.overriding_connection + connection + else + super + end + end + + private + + def overriding_connection + Thread.current[:overriding_connection] + end + + def overriding_connection=(connection) + Thread.current[:overriding_connection] = connection + end + end + end + end +end diff --git a/lib/gitlab/database/transaction/context.rb b/lib/gitlab/database/transaction/context.rb index a50dd30b75b..a902537f02e 100644 --- a/lib/gitlab/database/transaction/context.rb +++ b/lib/gitlab/database/transaction/context.rb @@ -6,9 +6,8 @@ module Gitlab class Context attr_reader :context - LOG_DEPTH_THRESHOLD = 8 - LOG_SAVEPOINTS_THRESHOLD = 32 - LOG_DURATION_S_THRESHOLD = 300 + LOG_SAVEPOINTS_THRESHOLD = 1 # 1 `SAVEPOINT` created in a transaction + LOG_DURATION_S_THRESHOLD = 120 # transaction that is running for 2 minutes or longer LOG_THROTTLE_DURATION = 1 def initialize @@ -19,6 +18,10 @@ module Gitlab @context[:start_time] = current_timestamp end + def set_depth(depth) + @context[:depth] = [@context[:depth].to_i, depth].max + end + def increment_savepoints @context[:savepoints] = @context[:savepoints].to_i + 1 end @@ -31,42 +34,33 @@ module Gitlab @context[:releases] = @context[:releases].to_i + 1 end - def set_depth(depth) - @context[:depth] = [@context[:depth].to_i, depth].max - end - def track_sql(sql) (@context[:queries] ||= []).push(sql) end + def track_backtrace(backtrace) + cleaned_backtrace = Gitlab::BacktraceCleaner.clean_backtrace(backtrace) + (@context[:backtraces] ||= []).push(cleaned_backtrace) + end + def duration return unless @context[:start_time].present? current_timestamp - @context[:start_time] end - def depth_threshold_exceeded? - @context[:depth].to_i > LOG_DEPTH_THRESHOLD - end - def savepoints_threshold_exceeded? - @context[:savepoints].to_i > LOG_SAVEPOINTS_THRESHOLD + @context[:savepoints].to_i >= LOG_SAVEPOINTS_THRESHOLD end def duration_threshold_exceeded? - duration.to_i > LOG_DURATION_S_THRESHOLD - end - - def log_savepoints? - depth_threshold_exceeded? || savepoints_threshold_exceeded? - end - - def log_duration? - duration_threshold_exceeded? + duration.to_i >= LOG_DURATION_S_THRESHOLD end def should_log? - !logged_already? && (log_savepoints? || log_duration?) + return false if logged_already? + + savepoints_threshold_exceeded? || duration_threshold_exceeded? end def commit @@ -77,6 +71,10 @@ module Gitlab log(:rollback) end + def backtraces + @context[:backtraces].to_a + end + private def queries @@ -110,7 +108,8 @@ module Gitlab savepoints_count: @context[:savepoints].to_i, rollbacks_count: @context[:rollbacks].to_i, releases_count: @context[:releases].to_i, - sql: queries + sql: queries, + savepoint_backtraces: backtraces } application_info(attributes) diff --git a/lib/gitlab/database/transaction/observer.rb b/lib/gitlab/database/transaction/observer.rb index 7888f0916e3..ad6886a3d52 100644 --- a/lib/gitlab/database/transaction/observer.rb +++ b/lib/gitlab/database/transaction/observer.rb @@ -21,9 +21,10 @@ module Gitlab context.set_start_time context.set_depth(0) context.track_sql(event.payload[:sql]) - elsif cmd.start_with?('SAVEPOINT ') + elsif cmd.start_with?('SAVEPOINT', 'EXCEPTION') context.set_depth(manager.open_transactions) context.increment_savepoints + context.track_backtrace(caller) elsif cmd.start_with?('ROLLBACK TO SAVEPOINT') context.increment_rollbacks elsif cmd.start_with?('RELEASE SAVEPOINT ') diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index bbf8f133f0f..f9d467ae5cc 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -61,13 +61,15 @@ module Gitlab [10.seconds, 10.minutes] ].freeze - def initialize(logger: NULL_LOGGER, timing_configuration: DEFAULT_TIMING_CONFIGURATION, klass: nil, env: ENV) + def initialize(logger: NULL_LOGGER, allow_savepoints: true, timing_configuration: DEFAULT_TIMING_CONFIGURATION, klass: nil, env: ENV, connection: ActiveRecord::Base.connection) @logger = logger @klass = klass + @allow_savepoints = allow_savepoints @timing_configuration = timing_configuration @env = env @current_iteration = 1 @log_params = { method: 'with_lock_retries', class: klass.to_s } + @connection = connection end # Executes a block of code, retrying it whenever a database lock can't be acquired in time @@ -95,7 +97,7 @@ module Gitlab run_block_with_lock_timeout rescue ActiveRecord::LockWaitTimeout if retry_with_lock_timeout? - disable_idle_in_transaction_timeout if ActiveRecord::Base.connection.transaction_open? + disable_idle_in_transaction_timeout if connection.transaction_open? wait_until_next_retry reset_db_settings @@ -115,14 +117,16 @@ module Gitlab private - attr_reader :logger, :env, :block, :current_iteration, :log_params, :timing_configuration + attr_reader :logger, :env, :block, :current_iteration, :log_params, :timing_configuration, :connection def run_block block.call end def run_block_with_lock_timeout - ActiveRecord::Base.transaction(requires_new: true) do + raise "WithLockRetries should not run inside already open transaction" if connection.transaction_open? && @allow_savepoints.blank? + + connection.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions execute("SET LOCAL lock_timeout TO '#{current_lock_timeout_in_ms}ms'") log(message: 'Lock timeout is set', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms) @@ -149,7 +153,7 @@ module Gitlab log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration) log(message: "Executing the migration without lock timeout", current_iteration: current_iteration) - disable_lock_timeout if ActiveRecord::Base.connection.transaction_open? + disable_lock_timeout if connection.transaction_open? run_block @@ -165,7 +169,7 @@ module Gitlab end def execute(statement) - ActiveRecord::Base.connection.execute(statement) + connection.execute(statement) end def retry_count diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb new file mode 100644 index 00000000000..c5acdb41de5 --- /dev/null +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module WorkItems + module BaseTypeImporter + def self.import + WorkItem::Type::BASE_TYPES.each do |type, attributes| + WorkItem::Type.create!(base_type: type, **attributes.slice(:name, :icon_name)) + end + end + end + end + end +end diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb index eb475307f27..111ea697ec2 100644 --- a/lib/gitlab/devise_failure.rb +++ b/lib/gitlab/devise_failure.rb @@ -2,18 +2,10 @@ module Gitlab class DeviseFailure < Devise::FailureApp - include ::SessionsHelper - # If the request format is not known, send a redirect instead of a 401 # response, since this is the outcome we're most likely to want def http_auth? request_format && super end - - def respond - limit_session_time - - super - end end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index a792eafde79..075027ebdc8 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -132,6 +132,8 @@ module Gitlab diff_file_id, gzip_compress(highlighted_diff_lines_hash.to_json) ) + rescue Encoding::UndefinedConversionError + nil end # HSETs have to have their expiration date manually updated diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index b110d39818d..4b490ae0d26 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -55,7 +55,7 @@ module Gitlab private def create_issue - Issues::CreateService.new( + ::Issues::CreateService.new( project: project, current_user: author, params: { diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 84b55079cea..74c8d0a1fd7 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -71,7 +71,7 @@ module Gitlab end def create_issue! - @issue = Issues::CreateService.new( + @issue = ::Issues::CreateService.new( project: project, current_user: User.support_bot, params: { diff --git a/lib/gitlab/email/message/in_product_marketing/team.rb b/lib/gitlab/email/message/in_product_marketing/team.rb index cf723ad5efd..6a0471ef9c5 100644 --- a/lib/gitlab/email/message/in_product_marketing/team.rb +++ b/lib/gitlab/email/message/in_product_marketing/team.rb @@ -23,7 +23,7 @@ module Gitlab def title [ - s_('InProductMarketing|Team work makes the dream work'), + s_('InProductMarketing|Team work makes the dream work'), s_('InProductMarketing|*GitLab*, noun: a synonym for efficient teams'), s_('InProductMarketing|Find out how your teams are really doing') ][series] diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 50e7631d983..2e0060c7c18 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -40,15 +40,7 @@ module Gitlab def detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN, cache_key: nil) return if data.nil? - if Feature.enabled?(:cached_encoding_detection, type: :development, default_enabled: :yaml) - return CharlockHolmes::EncodingDetector.new(limit).detect(data) unless cache_key.present? - - Rails.cache.fetch([:detect_binary, CharlockHolmes::VERSION, cache_key], expires_in: 1.week) do - CharlockHolmes::EncodingDetector.new(limit).detect(data) - end - else - CharlockHolmes::EncodingDetector.new(limit).detect(data) - end + CharlockHolmes::EncodingDetector.new(limit).detect(data) end def detect_binary?(data, detect = nil) diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 2f78e4e5c0a..c74bd8e75ef 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -5,12 +5,8 @@ # Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. # Experiment options: # - tracking_category (optional, used to set the category when tracking an experiment event) -# - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility -- you likely do not need this, see note in the next paragraph.) # - rollout_strategy: default is `:cookie` based rollout. We may also set it to `:user` based rollout # -# Using the backwards-compatible subject index (use_backwards_compatible_subject_index option): -# This option was added when [the calculation of experimentation_subject_index was changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45733/diffs#41af4a6fa5a10c7068559ce21c5188483751d934_157_173). It is not intended to be used by new experiments, it exists merely for the segmentation integrity of in-flight experiments at the time the change was deployed. That is, we want users who were assigned to the "experimental" group or the "control" group before the change to still be in those same groups after the change. See [the original issue](https://gitlab.com/gitlab-org/gitlab/-/issues/270858) and [this related comment](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48110#note_458223745) for more information. -# # The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), # which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. # @@ -48,14 +44,6 @@ module Gitlab show_trial_status_in_sidebar: { tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar', rollout_strategy: :group - }, - learn_gitlab_a: { - tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA', - rollout_strategy: :user - }, - learn_gitlab_b: { - tracking_category: 'Growth::Activation::Experiment::LearnGitLabB', - rollout_strategy: :user } }.freeze @@ -118,11 +106,7 @@ module Gitlab private def index_for_subject(experiment, subject) - index = if experiment.use_backwards_compatible_subject_index - Digest::SHA1.hexdigest(subject_id(subject)).hex - else - Zlib.crc32("#{experiment.key}#{subject_id(subject)}") - end + index = Zlib.crc32("#{experiment.key}#{subject_id(subject)}") index % 100 end diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 2a43f0d5ca9..7cc29cde45c 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -48,7 +48,7 @@ module Gitlab Experimentation.log_invalid_rollout(experiment_key, subject) - subject ||= fallback_experimentation_subject_index(experiment_key) + subject ||= experimentation_subject_id Experimentation.in_experiment_group?(experiment_key, subject: subject) end @@ -106,16 +106,6 @@ module Gitlab cookies.signed[:experimentation_subject_id] end - def fallback_experimentation_subject_index(experiment_key) - return if experimentation_subject_id.blank? - - if Experimentation.get_experiment(experiment_key).use_backwards_compatible_subject_index - experimentation_subject_id.delete('-') - else - experimentation_subject_id - end - end - def track_experiment_event_for(experiment_key, action, value, subject: nil) return unless Experimentation.active?(experiment_key) @@ -139,7 +129,7 @@ module Gitlab def tracking_group(experiment_key, suffix = nil, subject: nil) return unless Experimentation.active?(experiment_key) - subject ||= fallback_experimentation_subject_index(experiment_key) + subject ||= experimentation_subject_id group = experiment_enabled?(experiment_key, subject: subject) ? GROUP_EXPERIMENTAL : GROUP_CONTROL suffix ? "#{group}#{suffix}" : group diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index 17dda45f5b7..8ba95520638 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -5,12 +5,11 @@ module Gitlab class Experiment FEATURE_FLAG_SUFFIX = "_experiment_percentage" - attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index, :rollout_strategy + attr_reader :key, :tracking_category, :rollout_strategy def initialize(key, **params) @key = key @tracking_category = params[:tracking_category] - @use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index] @rollout_strategy = params[:rollout_strategy] || :cookie end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index d5ced2045f5..a1855132b0c 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -5,7 +5,6 @@ require_dependency 'gitlab/encoding_helper' module Gitlab module Git # The ID of empty tree. - # See http://stackoverflow.com/a/40884093/1856239 and # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' BLANK_SHA = ('0' * 40).freeze diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 7fd4acb4179..6605e896ef1 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -108,20 +108,25 @@ module Gitlab # See also #repository.commits_between # # Ex. - # Commit.between(repo, '29eda46b', 'master') + # Commit.between(repo, '29eda46b', 'master') # all commits, ordered oldest to newest + # Commit.between(repo, '29eda46b', 'master', limit: 100) # 100 newest commits, ordered oldest to newest # - def between(repo, base, head) + def between(repo, base, head, limit: nil) # In either of these cases, we are guaranteed to return no commits, so # shortcut the RPC call return [] if Gitlab::Git.blank_ref?(base) || Gitlab::Git.blank_ref?(head) wrapped_gitaly_errors do - if Feature.enabled?(:between_uses_list_commits, default_enabled: :yaml) - revisions = [head, "^#{base}"] # base..head - - repo.gitaly_commit_client.list_commits(revisions, reverse: true) + revisions = [head, "^#{base}"] # base..head + client = repo.gitaly_commit_client + + # We must return the commits in chronological order but using both + # limit and reverse in the Gitaly RPC would return the oldest N, + # rather than newest N, commits, so reorder in Ruby with limit + if limit + client.list_commits(revisions, pagination_params: { limit: limit }).reverse! else - repo.gitaly_commit_client.between(base, head) + client.list_commits(revisions, reverse: true) end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1ab80fe2454..bc15bd367d8 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -127,6 +127,13 @@ module Gitlab end end + def find_tag(name) + wrapped_gitaly_errors do + gitaly_ref_client.find_tag(name) + end + rescue CommandError + end + def local_branches(sort_by: nil, pagination_params: nil) wrapped_gitaly_errors do gitaly_ref_client.local_branches(sort_by: sort_by, pagination_params: pagination_params) @@ -191,9 +198,9 @@ module Gitlab # Returns an Array of Tags # - def tags + def tags(sort_by: nil) wrapped_gitaly_errors do - gitaly_ref_client.tags + gitaly_ref_client.tags(sort_by: sort_by) end end @@ -360,27 +367,31 @@ module Gitlab end end - def new_blobs(newrev, dynamic_timeout: nil) - return [] if newrev.blank? || newrev == ::Gitlab::Git::BLANK_SHA + def new_blobs(newrevs, dynamic_timeout: nil) + newrevs = Array.wrap(newrevs).reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA } + return [] if newrevs.empty? - strong_memoize("new_blobs_#{newrev}") do - wrapped_gitaly_errors do - gitaly_ref_client.list_new_blobs(newrev, REV_LIST_COMMIT_LIMIT, dynamic_timeout: dynamic_timeout) - end + newrevs = newrevs.uniq.sort + + @new_blobs ||= Hash.new do |h, revs| + h[revs] = blobs(['--not', '--all', '--not'] + newrevs, with_paths: true, dynamic_timeout: dynamic_timeout) end + + @new_blobs[newrevs] end # List blobs reachable via a set of revisions. Supports the # pseudo-revisions `--not` and `--all`. Uses the minimum of # GitalyClient.medium_timeout and dynamic timeout if the dynamic # timeout is set, otherwise it'll always use the medium timeout. - def blobs(revisions, dynamic_timeout: nil) + def blobs(revisions, with_paths: false, dynamic_timeout: nil) revisions = revisions.reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA } return [] if revisions.blank? wrapped_gitaly_errors do - gitaly_blob_client.list_blobs(revisions, limit: REV_LIST_COMMIT_LIMIT, dynamic_timeout: dynamic_timeout) + gitaly_blob_client.list_blobs(revisions, limit: REV_LIST_COMMIT_LIMIT, + with_paths: with_paths, dynamic_timeout: dynamic_timeout) end end @@ -491,13 +502,6 @@ module Gitlab [] end - # Returns a RefName for a given SHA - def ref_name_for_sha(ref_path, sha) - raise ArgumentError, "sha can't be empty" unless sha.present? - - gitaly_ref_client.find_ref_name(sha, ref_path) - end - # Get refs hash which key is the commit id # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch # Note that both inherit from Gitlab::Git::Ref @@ -607,10 +611,6 @@ module Gitlab end end - def find_tag(name) - tags.find { |tag| tag.name == name } - end - def merge_to_ref(user, **kwargs) wrapped_gitaly_errors do gitaly_operation_client.user_merge_to_ref(user, **kwargs) @@ -876,12 +876,6 @@ module Gitlab end end - def squash_in_progress?(squash_id) - wrapped_gitaly_errors do - gitaly_repository_client.squash_in_progress?(squash_id) - end - end - def bundle_to_disk(save_path) wrapped_gitaly_errors do gitaly_repository_client.create_bundle(save_path) @@ -911,17 +905,7 @@ module Gitlab # This guard avoids Gitaly log/error spam raise NoRepository, 'repository does not exist' unless exists? - if Feature.enabled?(:set_full_path) - gitaly_repository_client.set_full_path(full_path) - else - set_config('gitlab.fullpath' => full_path) - end - end - - def set_config(entries) - wrapped_gitaly_errors do - gitaly_repository_client.set_config(entries) - end + gitaly_repository_client.set_full_path(full_path) end def disconnect_alternates diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 5993c8888d3..40c003821b9 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -13,18 +13,53 @@ module Gitlab extend ::Gitlab::Utils::Override include Gitlab::Git::RuggedImpl::UseRugged + TREE_SORT_ORDER = { tree: 0, blob: 1, commit: 2 }.freeze + override :tree_entries def tree_entries(repository, sha, path, recursive, pagination_params = nil) if use_rugged?(repository, :rugged_tree_entries) - [ - execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive), - nil - ] + entries = execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive) + + if pagination_params + paginated_response(entries, pagination_params[:limit], pagination_params[:page_token].to_s) + else + [entries, nil] + end else super end end + # Rugged version of TreePagination in Go: https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3611 + def paginated_response(entries, limit, token) + total_entries = entries.count + + return [[], nil] if limit == 0 || limit.blank? + + entries = Gitlab::Utils.stable_sort_by(entries) { |x| TREE_SORT_ORDER[x.type] } + + if token.blank? + index = 0 + else + index = entries.index { |entry| entry.id == token } + + raise Gitlab::Git::CommandError, "could not find starting OID: #{token}" if index.nil? + + index += 1 + end + + return [entries[index..], nil] if limit < 0 + + last_index = index + limit + result = entries[index...last_index] + + if last_index < total_entries + cursor = Gitaly::PaginationCursor.new(next_cursor: result.last.id) + end + + [result, cursor] + end + def tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) tree_entries_from_rugged(repository, sha, path, recursive).tap do |entries| # This was an optimization to reduce N+1 queries for Gitaly diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index 05ae3391040..0798cc51055 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -6,7 +6,7 @@ module Gitlab attr_reader :username, :name, :email, :gl_id, :timezone def self.from_gitlab(gitlab_user) - new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user), gitlab_user.timezone) + new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email_or_default, Gitlab::GlId.gl_id(gitlab_user), gitlab_user.timezone) end def self.from_gitaly(gitaly_user) diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 362ecbd845d..3b08a833aeb 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -19,12 +19,13 @@ module Gitlab consume_blob_response(response) end - def list_blobs(revisions, limit: 0, bytes_limit: 0, dynamic_timeout: nil) + def list_blobs(revisions, limit: 0, bytes_limit: 0, with_paths: false, dynamic_timeout: nil) request = Gitaly::ListBlobsRequest.new( repository: @gitaly_repo, revisions: Array.wrap(revisions), limit: limit, - bytes_limit: bytes_limit + bytes_limit: bytes_limit, + with_paths: with_paths ) timeout = diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index fa616a252e4..75588ad980c 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -127,6 +127,7 @@ module Gitlab entries = response.flat_map do |message| cursor = message.pagination_cursor if message.pagination_cursor + message.entries.map do |gitaly_tree_entry| Gitlab::Git::Tree.new( id: gitaly_tree_entry.oid, @@ -255,11 +256,12 @@ module Gitlab consume_commits_response(response) end - def list_commits(revisions, reverse: false) + def list_commits(revisions, reverse: false, pagination_params: nil) request = Gitaly::ListCommitsRequest.new( repository: @gitaly_repo, revisions: Array.wrap(revisions), - reverse: reverse + reverse: reverse, + pagination_params: pagination_params ) response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 7097d5bd181..235eef4575e 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -5,6 +5,16 @@ module Gitlab class RefService include Gitlab::EncodingHelper + TAGS_SORT_KEY = { + 'name' => Gitaly::FindAllTagsRequest::SortBy::Key::REFNAME, + 'updated' => Gitaly::FindAllTagsRequest::SortBy::Key::CREATORDATE + }.freeze + + TAGS_SORT_DIRECTION = { + 'asc' => Gitaly::SortDirection::ASCENDING, + 'desc' => Gitaly::SortDirection::DESCENDING + }.freeze + # 'repository' is a Gitlab::Git::Repository def initialize(repository) @repository = repository @@ -52,38 +62,6 @@ module Gitlab consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } end - def find_ref_name(commit_id, ref_prefix) - request = Gitaly::FindRefNameRequest.new( - repository: @gitaly_repo, - commit_id: commit_id, - prefix: ref_prefix - ) - response = GitalyClient.call(@storage, :ref_service, :find_ref_name, request, timeout: GitalyClient.medium_timeout) - encode!(response.name.dup) - end - - def list_new_blobs(newrev, limit = 0, dynamic_timeout: nil) - request = Gitaly::ListNewBlobsRequest.new( - repository: @gitaly_repo, - commit_id: newrev, - limit: limit - ) - - timeout = - if dynamic_timeout - [dynamic_timeout, GitalyClient.medium_timeout].min - else - GitalyClient.medium_timeout - end - - response = GitalyClient.call(@storage, :ref_service, :list_new_blobs, request, timeout: timeout) - response.flat_map do |msg| - # Returns an Array of Gitaly::NewBlobObject objects - # Available methods are: #size, #oid and #path - msg.new_blob_objects - end - end - def count_tag_names tag_names.count end @@ -94,13 +72,15 @@ module Gitlab def local_branches(sort_by: nil, pagination_params: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo, pagination_params: pagination_params) - request.sort_by = sort_by_param(sort_by) if sort_by + request.sort_by = sort_local_branches_by_param(sort_by) if sort_by response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout) consume_find_local_branches_response(response) end - def tags + def tags(sort_by: nil) request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo) + request.sort_by = sort_tags_by_param(sort_by) if sort_by + response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout) consume_tags_response(response) end @@ -127,6 +107,21 @@ module Gitlab Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit) end + def find_tag(tag_name) + return if tag_name.blank? + + request = Gitaly::FindTagRequest.new( + repository: @gitaly_repo, + tag_name: encode_binary(tag_name) + ) + + response = GitalyClient.call(@repository.storage, :ref_service, :find_tag, request, timeout: GitalyClient.medium_timeout) + tag = response.tag + return unless tag + + Gitlab::Git::Tag.new(@repository, tag) + end + def delete_refs(refs: [], except_with_prefixes: []) request = Gitaly::DeleteRefsRequest.new( repository: @gitaly_repo, @@ -211,7 +206,7 @@ module Gitlab response.flat_map { |message| message.names.map { |name| yield(name) } } end - def sort_by_param(sort_by) + def sort_local_branches_by_param(sort_by) sort_by = 'name' if sort_by == 'name_asc' enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) @@ -220,6 +215,17 @@ module Gitlab enum_value end + def sort_tags_by_param(sort_by) + match = sort_by.match(/^(?<key>name|updated)_(?<direction>asc|desc)$/) + + return unless match + + Gitaly::FindAllTagsRequest::SortBy.new( + key: TAGS_SORT_KEY[match[:key]], + direction: TAGS_SORT_DIRECTION[match[:direction]] + ) + end + def consume_find_local_branches_response(response) response.flat_map do |message| message.branches.map do |gitaly_branch| diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 2e26b3341a2..7e7d543d0a5 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -155,23 +155,6 @@ module Gitlab ) end - def squash_in_progress?(squash_id) - request = Gitaly::IsSquashInProgressRequest.new( - repository: @gitaly_repo, - squash_id: squash_id.to_s - ) - - response = GitalyClient.call( - @storage, - :repository_service, - :is_squash_in_progress, - request, - timeout: GitalyClient.fast_timeout - ) - - response.in_progress - end - def fetch_source_branch(source_repository, source_branch, local_ref) request = Gitaly::FetchSourceBranchRequest.new( repository: @gitaly_repo, @@ -281,25 +264,6 @@ module Gitlab nil end - def set_config(entries) - return if entries.empty? - - request = Gitaly::SetConfigRequest.new(repository: @gitaly_repo) - entries.each do |key, value| - request.entries << build_set_config_entry(key, value) - end - - GitalyClient.call( - @storage, - :repository_service, - :set_config, - request, - timeout: GitalyClient.fast_timeout - ) - - nil - end - def license_short_name request = Gitaly::FindLicenseRequest.new(repository: @gitaly_repo) diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index c3cc15e10f7..7ac0d875512 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -11,6 +11,7 @@ module Gitlab Client.new( token_to_use, host: host.presence || self.formatted_import_url(project), + per_page: self.per_page(project), parallel: parallel ) end @@ -33,5 +34,13 @@ module Gitlab url.to_s end end + + def self.per_page(project) + if project.group.present? && Feature.enabled?(:github_importer_lower_per_page_limit, project.group, type: :ops, default_enabled: :yaml) + Gitlab::GithubImport::Client::LOWER_PER_PAGE + else + Gitlab::GithubImport::Client::DEFAULT_PER_PAGE + end + end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 138716b1b53..efa816c5eb0 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -19,6 +19,8 @@ module Gitlab attr_reader :octokit SEARCH_MAX_REQUESTS_PER_MINUTE = 30 + DEFAULT_PER_PAGE = 100 + LOWER_PER_PAGE = 50 # A single page of data and the corresponding page number. Page = Struct.new(:objects, :number) @@ -44,7 +46,7 @@ module Gitlab # this value to `true` for parallel importing is crucial as # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. - def initialize(token, host: nil, per_page: 100, parallel: true) + def initialize(token, host: nil, per_page: DEFAULT_PER_PAGE, parallel: true) @host = host @octokit = ::Octokit::Client.new( access_token: token, diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 9bda066efcc..4cfc920e2e3 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -24,13 +24,14 @@ module Gitlab note_body = MarkdownText.format(note.note, note.author, author_found) attributes = { + discussion_id: Discussion.discussion_id(note), noteable_type: 'MergeRequest', noteable_id: mr_id, project_id: project.id, author_id: author_id, note: note_body, system: false, - commit_id: note.commit_id, + commit_id: note.original_commit_id, line_code: note.line_code, type: 'LegacyDiffNote', created_at: note.created_at, diff --git a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb new file mode 100644 index 00000000000..a2c3d1bd057 --- /dev/null +++ b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# This importer is used when `github_importer_single_endpoint_notes_import` +# feature flag is on and replaces `DiffNotesImporter`. +# +# It fetches 1 PR's diff notes at a time using `pull_request_comments` endpoint, which is +# slower than `NotesImporter` but it makes sure all notes are imported, +# as it can sometimes not be the case for `NotesImporter`, because +# `issues_comments` endpoint it uses can be limited by GitHub API +# to not return all available pages. +module Gitlab + module GithubImport + module Importer + class SingleEndpointDiffNotesImporter + include ParallelScheduling + include SingleEndpointNotesImporting + + def importer_class + DiffNoteImporter + end + + def representation_class + Representation::DiffNote + end + + def sidekiq_worker_class + ImportDiffNoteWorker + end + + def object_type + :diff_note + end + + def collection_method + :pull_request_comments + end + + private + + def noteables + project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + end + + def page_counter_id(merge_request) + "merge_request/#{merge_request.id}/#{collection_method}" + end + + def notes_imported_cache_key + "github-importer/merge_request/diff_notes/already-imported/#{project.id}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb new file mode 100644 index 00000000000..49569ed52d8 --- /dev/null +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# This importer is used when `github_importer_single_endpoint_notes_import` +# feature flag is on and replaces `IssuesImporter` issue notes import. +# +# It fetches 1 issue's comments at a time using `issue_comments` endpoint, which is +# slower than `NotesImporter` but it makes sure all notes are imported, +# as it can sometimes not be the case for `NotesImporter`, because +# `issues_comments` endpoint it uses can be limited by GitHub API +# to not return all available pages. +module Gitlab + module GithubImport + module Importer + class SingleEndpointIssueNotesImporter + include ParallelScheduling + include SingleEndpointNotesImporting + + def importer_class + NoteImporter + end + + def representation_class + Representation::Note + end + + def sidekiq_worker_class + ImportNoteWorker + end + + def object_type + :note + end + + def collection_method + :issue_comments + end + + private + + def noteables + project.issues.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + end + + def page_counter_id(issue) + "issue/#{issue.id}/#{collection_method}" + end + + def notes_imported_cache_key + "github-importer/issue/notes/already-imported/#{project.id}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb new file mode 100644 index 00000000000..d837639c14d --- /dev/null +++ b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# This importer is used when `github_importer_single_endpoint_notes_import` +# feature flag is on and replaces `NotesImporter` MR notes import. +# +# It fetches 1 PR's comments at a time using `issue_comments` endpoint, which is +# slower than `NotesImporter` but it makes sure all notes are imported, +# as it can sometimes not be the case for `NotesImporter`, because +# `issues_comments` endpoint it uses can be limited by GitHub API +# to not return all available pages. +module Gitlab + module GithubImport + module Importer + class SingleEndpointMergeRequestNotesImporter + include ParallelScheduling + include SingleEndpointNotesImporting + + def importer_class + NoteImporter + end + + def representation_class + Representation::Note + end + + def sidekiq_worker_class + ImportNoteWorker + end + + def object_type + :note + end + + def collection_method + :issue_comments + end + + private + + def noteables + project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + end + + def page_counter_id(merge_request) + "merge_request/#{merge_request.id}/#{collection_method}" + end + + def notes_imported_cache_key + "github-importer/merge_request/notes/already-imported/#{project.id}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index 136531505ea..5298a3d81ea 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -23,7 +23,7 @@ module Gitlab # # This method will return `nil` if no ID could be found. def database_id - val = Gitlab::Cache::Import::Caching.read(cache_key) + val = Gitlab::Cache::Import::Caching.read(cache_key, timeout: timeout) val.to_i if val.present? end @@ -32,7 +32,7 @@ module Gitlab # # database_id - The ID of the corresponding database row. def cache_database_id(database_id) - Gitlab::Cache::Import::Caching.write(cache_key, database_id) + Gitlab::Cache::Import::Caching.write(cache_key, database_id, timeout: timeout) end private @@ -76,6 +76,14 @@ module Gitlab ) end end + + def timeout + if project.group.present? && ::Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops, default_enabled: :yaml) + Gitlab::Cache::Import::Caching::LONGER_TIMEOUT + else + Gitlab::Cache::Import::Caching::TIMEOUT + end + end end end end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 8c76f5a9d94..4d0074e43d7 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -44,7 +44,7 @@ module Gitlab # still scheduling duplicates while. Since all work has already been # completed those jobs will just cycle through any remaining pages while # not scheduling anything. - Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i) + Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) info(project.id, message: "importer finished") retval diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index d336b1ba797..d0584cc6255 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -11,7 +11,7 @@ module Gitlab expose_attribute :noteable_type, :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :note, :created_at, :updated_at, - :github_id + :github_id, :original_commit_id NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze @@ -34,6 +34,7 @@ module Gitlab noteable_id: matches[:iid].to_i, file_path: note.path, commit_id: note.commit_id, + original_commit_id: note.original_commit_id, diff_hunk: note.diff_hunk, author: user, note: note.body, diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb new file mode 100644 index 00000000000..43402ecd165 --- /dev/null +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +# This module is used in: +# - SingleEndpointDiffNotesImporter +# - SingleEndpointIssueNotesImporter +# - SingleEndpointMergeRequestNotesImporter +# +# `github_importer_single_endpoint_notes_import` +# feature flag is on. +# +# It fetches 1 PR's associated objects at a time using `issue_comments` or +# `pull_request_comments` endpoint, which is slower than `NotesImporter` +# but it makes sure all notes are imported, as it can sometimes not be +# the case for `NotesImporter`, because `issues_comments` endpoint +# it uses can be limited by GitHub API to not return all available pages. +module Gitlab + module GithubImport + module SingleEndpointNotesImporting + BATCH_SIZE = 100 + + def each_object_to_import + each_notes_page do |page| + page.objects.each do |note| + next if already_imported?(note) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield(note) + + mark_as_imported(note) + end + end + end + + def id_for_already_imported_cache(note) + note.id + end + + private + + def each_notes_page + noteables.each_batch(of: BATCH_SIZE, column: :iid) do |batch| + batch.each do |noteable| + # The page counter needs to be scoped by noteable to avoid skipping + # pages of notes from already imported noteables. + page_counter = PageCounter.new(project, page_counter_id(noteable)) + repo = project.import_source + options = collection_options.merge(page: page_counter.current) + + client.each_page(collection_method, repo, noteable.iid, options) do |page| + next unless page_counter.set(page.number) + + yield page + end + + mark_notes_imported(noteable) + end + end + end + + def mark_notes_imported(noteable) + Gitlab::Cache::Import::Caching.set_add( + notes_imported_cache_key, + noteable.iid + ) + end + + def already_imported_noteables + Gitlab::Cache::Import::Caching.values_from_set(notes_imported_cache_key) + end + + def noteables + NotImplementedError + end + + def notes_imported_cache_key + NotImplementedError + end + + def page_counter_id(noteable) + NotImplementedError + end + end + end +end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index f583ef39d13..93483ee697a 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -106,7 +106,7 @@ module Gitlab unless email user = client.user(username) - email = Gitlab::Cache::Import::Caching.write(cache_key, user.email) if user + email = Gitlab::Cache::Import::Caching.write(cache_key, user.email, timeout: timeout(user.email)) if user end email @@ -171,6 +171,16 @@ module Gitlab # which we couldn't find an ID. [exists, number > 0 ? number : nil] end + + private + + def timeout(email) + if email + Gitlab::Cache::Import::Caching::TIMEOUT + else + Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT + end + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 16a8b5f959e..258c13894fb 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -21,6 +21,8 @@ module Gitlab gon.sentry_environment = Gitlab.config.sentry.environment end + gon.recaptcha_api_server_url = ::Recaptcha.configuration.api_server_url + gon.recaptcha_sitekey = Gitlab::CurrentSettings.recaptcha_site_key gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab.revision gon.feature_category = Gitlab::ApplicationContext.current_context_attribute(:feature_category).presence @@ -35,6 +37,7 @@ module Gitlab gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week gon.time_display_relative = true gon.ee = Gitlab.ee? + gon.jh = Gitlab.jh? gon.dot_com = Gitlab.com? if current_user @@ -51,6 +54,7 @@ module Gitlab push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) + push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/authorize/connection_filter_extension.rb b/lib/gitlab/graphql/authorize/connection_filter_extension.rb index c75510df3e3..889c024ab5e 100644 --- a/lib/gitlab/graphql/authorize/connection_filter_extension.rb +++ b/lib/gitlab/graphql/authorize/connection_filter_extension.rb @@ -7,12 +7,14 @@ module Gitlab class Redactor include ::Gitlab::Graphql::Laziness - def initialize(type, context) + def initialize(type, context, resolver) @type = type @context = context + @resolver = resolver end def redact(nodes) + perform_before_authorize_action(nodes) remove_unauthorized(nodes) nodes @@ -29,6 +31,13 @@ module Gitlab private + def perform_before_authorize_action(nodes) + before_connection_authorization_block = @resolver&.before_connection_authorization_block + return unless before_connection_authorization_block.respond_to?(:call) + + before_connection_authorization_block.call(nodes, @context[:current_user]) + end + def remove_unauthorized(nodes) nodes .map! { |lazy| force(lazy) } @@ -49,14 +58,14 @@ module Gitlab end def redact_connection(conn, context) - redactor = Redactor.new(@field.type.unwrap.node_type, context) + redactor = Redactor.new(@field.type.unwrap.node_type, context, @field.resolver) return unless redactor.active? conn.redactor = redactor if conn.respond_to?(:redactor=) end def redact_list(list, context) - redactor = Redactor.new(@field.type.unwrap, context) + redactor = Redactor.new(@field.type.unwrap, context, @field.resolver) redactor.redact(list) if redactor.active? end end diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb index 26c1ce64a83..7f9013c6e4c 100644 --- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb +++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb @@ -5,19 +5,20 @@ module Gitlab module Loaders # Suitable for use to find resources that expose `where_full_path_in`, # such as Project, Group, Namespace + # full path is always converted to lowercase for case-insensitive results class FullPathModelLoader attr_reader :model_class, :full_path def initialize(model_class, full_path) @model_class = model_class - @full_path = full_path + @full_path = full_path.downcase end def find BatchLoader::GraphQL.for(full_path).batch(key: model_class) do |full_paths, loader, args| # `with_route` avoids an N+1 calculating full_path args[:key].where_full_path_in(full_paths).with_route.each do |model_instance| - loader.call(model_instance.full_path, model_instance) + loader.call(model_instance.full_path.downcase, model_instance) end end end diff --git a/lib/gitlab/graphql/todos_project_permission_preloader/field_extension.rb b/lib/gitlab/graphql/todos_project_permission_preloader/field_extension.rb deleted file mode 100644 index 77f3b1ac71a..00000000000 --- a/lib/gitlab/graphql/todos_project_permission_preloader/field_extension.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module TodosProjectPermissionPreloader - class FieldExtension < ::GraphQL::Schema::FieldExtension - def after_resolve(value:, memo:, **rest) - todos = value.to_a - - Preloaders::UserMaxAccessLevelInProjectsPreloader.new( - todos.map(&:project).compact, - current_user(rest) - ).execute - - value - end - - private - - def current_user(options) - options.dig(:context, :current_user) - end - end - end - end -end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 5f1b9873fee..33f2916345e 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -7,6 +7,7 @@ module Gitlab AVAILABLE_LANGUAGES = { 'bg' => 'Bulgarian - български', 'cs_CZ' => 'Czech - čeština', + 'da_DK' => 'Danish - dansk', 'de' => 'German - Deutsch', 'en' => 'English', 'eo' => 'Esperanto - esperanto', @@ -18,10 +19,12 @@ module Gitlab 'it' => 'Italian - italiano', 'ja' => 'Japanese - 日本語', 'ko' => 'Korean - 한국어', + 'nb_NO' => 'Norwegian (Bokmål) - norsk (bokmål)', 'nl_NL' => 'Dutch - Nederlands', 'pl_PL' => 'Polish - polski', 'pt_BR' => 'Portuguese (Brazil) - português (Brasil)', - 'ru' => 'Russian - Русский', + 'ro_RO' => 'Romanian - română', + 'ru' => 'Russian - русский', 'tr_TR' => 'Turkish - Türkçe', 'uk' => 'Ukrainian - українська', 'zh_CN' => 'Chinese, Simplified - 简体中文', @@ -39,25 +42,28 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab/-/issues/18923 TRANSLATION_LEVELS = { 'bg' => 0, - 'cs_CZ' => 1, + 'cs_CZ' => 0, + 'da_DK' => 25, 'de' => 16, 'en' => 100, 'eo' => 0, - 'es' => 36, + 'es' => 42, 'fil_PH' => 0, - 'fr' => 12, + 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 39, + 'ja' => 38, 'ko' => 12, + 'nb_NO' => 26, 'nl_NL' => 0, 'pl_PL' => 6, - 'pt_BR' => 36, + 'pt_BR' => 45, + 'ro_RO' => 21, 'ru' => 28, 'tr_TR' => 16, 'uk' => 40, - 'zh_CN' => 74, + 'zh_CN' => 94, 'zh_HK' => 2, 'zh_TW' => 3 }.freeze diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index 86f51add504..acd03d9ec20 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -42,6 +42,10 @@ module Gitlab class AttributesPermitter attr_reader :permitted_attributes + # We want to use AttributesCleaner for these relations instead, in the future this should be removed to make sure + # we are using AttributesPermitter for every imported relation. + DISABLED_RELATION_NAMES = %i[user author ci_cd_settings issuable_sla push_rule].freeze + def initialize(config: ImportExport::Config.new.to_h) @config = config @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config) @@ -50,16 +54,20 @@ module Gitlab build_permitted_attributes end - def permit(relation_name, relation_hash) - permitted_attributes = permitted_attributes_for(relation_name) + def permit(relation_sym, relation_hash) + permitted_attributes = permitted_attributes_for(relation_sym) relation_hash.select do |key, _| - permitted_attributes.include?(key) + permitted_attributes.include?(key.to_sym) end end - def permitted_attributes_for(relation_name) - @permitted_attributes[relation_name] || [] + def permitted_attributes_for(relation_sym) + @permitted_attributes[relation_sym] || [] + end + + def permitted_attributes_defined?(relation_sym) + !DISABLED_RELATION_NAMES.include?(relation_sym) && @attributes_finder.included_attributes.key?(relation_sym) end private @@ -88,11 +96,15 @@ module Gitlab end def build_attributes - @attributes_finder.included_attributes.each(&method(:add_permitted_attributes)) + @attributes_finder.included_attributes.each do |model_name, attributes| + add_permitted_attributes(model_name, attributes) + end end def build_methods - @attributes_finder.methods.each(&method(:add_permitted_attributes)) + @attributes_finder.methods.each do |model_name, attributes| + add_permitted_attributes(model_name, attributes) + end end def add_permitted_attributes(model_name, attributes) diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 30cd5ccfbcb..a84efd1d240 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -29,7 +29,7 @@ module Gitlab owner_id ].freeze - TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze + TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook ErrorTracking::ProjectErrorTrackingSetting].freeze def self.create(*args, **kwargs) new(*args, **kwargs).create @@ -45,6 +45,7 @@ module Gitlab end def initialize(relation_sym:, relation_index:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) + @relation_sym = relation_sym @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym @relation_index = relation_index @relation_hash = relation_hash.except('noteable_id') @@ -181,8 +182,17 @@ module Gitlab end def parsed_relation_hash - @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, - relation_class: relation_class) + strong_memoize(:parsed_relation_hash) do + if Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) && attributes_permitter.permitted_attributes_defined?(@relation_sym) + attributes_permitter.permit(@relation_sym, @relation_hash) + else + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, relation_class: relation_class) + end + end + end + + def attributes_permitter + @attributes_permitter ||= Gitlab::ImportExport::AttributesPermitter.new end def existing_or_new_object diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 3ebdcc950c3..fe0974d27a6 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -107,6 +107,7 @@ tree: - lists: - label: - :priorities + - :service_desk_setting group_members: - :user @@ -120,6 +121,41 @@ included_attributes: - :name ci_cd_settings: - :group_runners_enabled + metrics_setting: + - :dashboard_timezone + - :external_dashboard_url + - :project_id + project_badges: + - :created_at + - :group_id + - :image_url + - :link_url + - :name + - :project_id + - :type + - :updated_at + pipeline_schedules: + - :active + - :created_at + - :cron + - :cron_timezone + - :description + - :next_run_at + - :owner_id + - :project_id + - :ref + - :updated_at + error_tracking_setting: + - :api_url + - :organization_name + - :project_id + - :project_name + auto_devops: + - :created_at + - :deploy_strategy + - :enabled + - :project_id + - :updated_at # Do not include the following attributes for the models specified. excluded_attributes: @@ -127,6 +163,7 @@ excluded_attributes: - :name - :path - :namespace_id + - :project_namespace_id - :creator_id - :pool_repository_id - :import_url @@ -297,6 +334,7 @@ excluded_attributes: - :integrated service_desk_setting: - :outgoing_name + - :file_template_project_id priorities: - :label_id events: @@ -331,6 +369,9 @@ excluded_attributes: project_members: - :source_id - :invite_email_success + - :state + group_members: + - :state metrics: - :merge_request_id - :pipeline_id @@ -359,6 +400,7 @@ excluded_attributes: boards: - :milestone_id - :iteration_id + - :iteration_cadence_id lists: - :board_id - :label_id @@ -454,7 +496,6 @@ ee: - :unprotect_access_levels - protected_environments: - :deploy_access_levels - - :service_desk_setting - :security_setting - :push_rule diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index d5f924ae2bd..8d93098a80a 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -61,7 +61,9 @@ module Gitlab # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project/group. def create_relations! - relations.each(&method(:process_relation!)) + relations.each do |relation_key, relation_definition| + process_relation!(relation_key, relation_definition) + end end def process_relation!(relation_key, relation_definition) diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 0f21a16793d..ba25e54ac9f 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -41,7 +41,10 @@ module Gitlab instrumentation_class.add_call_details(duration, args) end - if duration > DURATION_ERROR_THRESHOLD && Feature.enabled?(:report_on_long_redis_durations, default_enabled: :yaml) + if duration > DURATION_ERROR_THRESHOLD && + instrumentation_class == ::Gitlab::Instrumentation::Redis::SharedState && + Feature.enabled?(:report_on_long_redis_durations, default_enabled: :yaml) + Gitlab::ErrorTracking.track_exception(MysteryRedisDurationError.new(caller), command: command_from_args(args), duration: duration, diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index 0fa9f435b5c..91797a7b99b 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -7,7 +7,7 @@ module Gitlab Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker - Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack + Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao )).freeze def self.namespaced_integrations diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 6b33b60e850..f11dd520d2d 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -5,11 +5,14 @@ module Gitlab class IssuablesCountForState # The name of the Gitlab::SafeRequestStore cache key. CACHE_KEY = :issuables_count_for_state + # The expiration time for the Rails cache. + CACHE_EXPIRES_IN = 1.hour + THRESHOLD = 1000 # The state values that can be safely casted to a Symbol. STATES = %w[opened closed merged all].freeze - attr_reader :project + attr_reader :project, :finder def self.declarative_policy_class 'IssuablePolicy' @@ -18,11 +21,12 @@ module Gitlab # finder - The finder class to use for retrieving the issuables. # fast_fail - restrict counting to a shorter period, degrading gracefully on # failure - def initialize(finder, project = nil, fast_fail: false) + def initialize(finder, project = nil, fast_fail: false, store_in_redis_cache: false) @finder = finder @project = project @fast_fail = fast_fail @cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache + @store_in_redis_cache = store_in_redis_cache end def for_state_or_opened(state = nil) @@ -52,7 +56,16 @@ module Gitlab private def cache_for_finder - @cache[@finder] + cached_counts = Rails.cache.read(redis_cache_key, cache_options) if cache_issues_count? + + cached_counts ||= @cache[finder] + return cached_counts if cached_counts.empty? + + if cache_issues_count? && cached_counts.values.all? { |count| count >= THRESHOLD } + Rails.cache.write(redis_cache_key, cached_counts, cache_options) + end + + cached_counts end def cast_state_to_symbol?(state) @@ -108,5 +121,33 @@ module Gitlab "Count of failed calls to IssuableFinder#count_by_state with fast failure" ).increment end + + def cache_issues_count? + @store_in_redis_cache && + finder.instance_of?(IssuesFinder) && + parent_group.present? && + !params_include_filters? + end + + def parent_group + finder.params.group + end + + def redis_cache_key + ['group', parent_group&.id, 'issues'] + end + + def cache_options + { expires_in: CACHE_EXPIRES_IN } + end + + def params_include_filters? + non_filtering_params = %i[ + scope state sort group_id include_subgroups + attempt_group_search_optimizations non_archived issue_types + ] + + finder.params.except(*non_filtering_params).values.any? + end end end diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb new file mode 100644 index 00000000000..dce165a3489 --- /dev/null +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module Gitlab + module Issues + module Rebalancing + class State + REDIS_EXPIRY_TIME = 10.days + MAX_NUMBER_OF_CONCURRENT_REBALANCES = 5 + NAMESPACE = 1 + PROJECT = 2 + + def initialize(root_namespace, projects) + @root_namespace = root_namespace + @projects = projects + @rebalanced_container_type = @root_namespace.is_a?(Group) ? NAMESPACE : PROJECT + @rebalanced_container_id = @rebalanced_container_type == NAMESPACE ? @root_namespace.id : projects.take.id # rubocop:disable CodeReuse/ActiveRecord + end + + def track_new_running_rebalance + with_redis do |redis| + redis.multi do |multi| + # we trigger re-balance for namespaces(groups) or specific user project + value = "#{rebalanced_container_type}/#{rebalanced_container_id}" + multi.sadd(concurrent_running_rebalances_key, value) + multi.expire(concurrent_running_rebalances_key, REDIS_EXPIRY_TIME) + end + end + end + + def concurrent_running_rebalances_count + with_redis { |redis| redis.scard(concurrent_running_rebalances_key).to_i } + end + + def rebalance_in_progress? + all_rebalanced_containers = with_redis { |redis| redis.smembers(concurrent_running_rebalances_key) } + + is_running = case rebalanced_container_type + when NAMESPACE + namespace_ids = all_rebalanced_containers.map {|string| string.split("#{NAMESPACE}/").second.to_i }.compact + namespace_ids.include?(root_namespace.id) + when PROJECT + project_ids = all_rebalanced_containers.map {|string| string.split("#{PROJECT}/").second.to_i }.compact + project_ids.include?(projects.take.id) # rubocop:disable CodeReuse/ActiveRecord + else + false + end + + refresh_keys_expiration if is_running + + is_running + end + + def can_start_rebalance? + rebalance_in_progress? || too_many_rebalances_running? + end + + def cache_issue_ids(issue_ids) + with_redis do |redis| + values = issue_ids.map { |issue| [issue.relative_position, issue.id] } + + redis.multi do |multi| + multi.zadd(issue_ids_key, values) unless values.blank? + multi.expire(issue_ids_key, REDIS_EXPIRY_TIME) + end + end + end + + def get_cached_issue_ids(index, limit) + with_redis do |redis| + redis.zrange(issue_ids_key, index, index + limit - 1) + end + end + + def cache_current_index(index) + with_redis { |redis| redis.set(current_index_key, index, ex: REDIS_EXPIRY_TIME) } + end + + def get_current_index + with_redis { |redis| redis.get(current_index_key).to_i } + end + + def cache_current_project_id(project_id) + with_redis { |redis| redis.set(current_project_key, project_id, ex: REDIS_EXPIRY_TIME) } + end + + def get_current_project_id + with_redis { |redis| redis.get(current_project_key) } + end + + def issue_count + @issue_count ||= with_redis { |redis| redis.zcard(issue_ids_key)} + end + + def remove_current_project_id_cache + with_redis { |redis| redis.del(current_project_key)} + end + + def refresh_keys_expiration + with_redis do |redis| + redis.multi do |multi| + multi.expire(issue_ids_key, REDIS_EXPIRY_TIME) + multi.expire(current_index_key, REDIS_EXPIRY_TIME) + multi.expire(current_project_key, REDIS_EXPIRY_TIME) + multi.expire(concurrent_running_rebalances_key, REDIS_EXPIRY_TIME) + end + end + end + + def cleanup_cache + with_redis do |redis| + redis.multi do |multi| + multi.del(issue_ids_key) + multi.del(current_index_key) + multi.del(current_project_key) + multi.srem(concurrent_running_rebalances_key, "#{rebalanced_container_type}/#{rebalanced_container_id}") + end + end + end + + private + + attr_accessor :root_namespace, :projects, :rebalanced_container_type, :rebalanced_container_id + + def too_many_rebalances_running? + concurrent_running_rebalances_count <= MAX_NUMBER_OF_CONCURRENT_REBALANCES + end + + def redis_key_prefix + "gitlab:issues-position-rebalances" + end + + def issue_ids_key + "#{redis_key_prefix}:#{root_namespace.id}" + end + + def current_index_key + "#{issue_ids_key}:current_index" + end + + def current_project_key + "#{issue_ids_key}:current_project_id" + end + + def concurrent_running_rebalances_key + "#{redis_key_prefix}:running_rebalances" + end + + def with_redis(&blk) + Gitlab::Redis::SharedState.with(&blk) # rubocop: disable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb index 842ee98e4da..753a185344e 100644 --- a/lib/gitlab/kas/client.rb +++ b/lib/gitlab/kas/client.rb @@ -7,6 +7,7 @@ module Gitlab JWT_AUDIENCE = 'gitlab-kas' STUB_CLASSES = { + agent_tracker: Gitlab::Agent::AgentTracker::Rpc::AgentTracker::Stub, configuration_project: Gitlab::Agent::ConfigurationProject::Rpc::ConfigurationProject::Stub }.freeze @@ -17,6 +18,15 @@ module Gitlab raise ConfigurationError, 'KAS internal URL is not configured' unless Gitlab::Kas.internal_url.present? end + def get_connected_agents(project:) + request = Gitlab::Agent::AgentTracker::Rpc::GetConnectedAgentsRequest.new(project_id: project.id) + + stub_for(:agent_tracker) + .get_connected_agents(request, metadata: metadata) + .agents + .to_a + end + def list_agent_config_files(project:) request = Gitlab::Agent::ConfigurationProject::Rpc::ListAgentConfigFilesRequest.new( repository: repository(project), @@ -49,7 +59,7 @@ module Gitlab end def kas_endpoint_url - Gitlab::Kas.internal_url.sub(%r{^grpc://|^grpcs://}, '') + Gitlab::Kas.internal_url.sub(%r{^grpcs?://}, '') end def credentials diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb index e333d3818b9..8a31e068c30 100644 --- a/lib/gitlab/kubernetes/cilium_network_policy.rb +++ b/lib/gitlab/kubernetes/cilium_network_policy.rb @@ -9,6 +9,36 @@ module Gitlab API_VERSION = "cilium.io/v2" KIND = 'CiliumNetworkPolicy' + PREDEFINED_POLICIES = { + 'allow-inbound-http' => <<~YAML.rstrip, + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + metadata: + name: allow-inbound-http + spec: + endpointSelector: + matchLabels: + network-policy.gitlab.com/disabled_by: gitlab + ingress: + - toPorts: + - ports: + - port: '80' + - port: '443' + YAML + 'drop-outbound' => <<~YAML.rstrip + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + metadata: + name: drop-outbound + spec: + endpointSelector: + matchLabels: + network-policy.gitlab.com/disabled_by: gitlab + egress: + - {} + YAML + }.freeze + # We are modeling existing kubernetes resource and don't have # control over amount of parameters. # rubocop:disable Metrics/ParameterLists diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index ee15d3b1812..f635f41ec39 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -41,6 +41,10 @@ module Gitlab def endpoint_id Labkit::Context.current&.get_attribute(:caller_id) end + + def db_config_name + ::Gitlab::Database.db_config_name(marginalia_adapter) + end end end end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 66361529546..ad45a037161 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -153,6 +153,10 @@ module Gitlab method_visibility = method_visibility_for(target, name) + # We silence warnings to avoid such warnings: + # `Skipping set of ruby2_keywords flag for <...> + # (method accepts keywords or method does not accept argument splat)` + # as we apply ruby2_keywords 'blindly' for every instrumented method. proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) if trans = Gitlab::Metrics::Instrumentation.transaction @@ -162,6 +166,7 @@ module Gitlab super end end + silence_warnings { ruby2_keywords(:#{name}) if respond_to?(:ruby2_keywords, true) } #{method_visibility} :#{name} EOF diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index 1c7767f5ca9..ebd0d1634e7 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -20,7 +20,9 @@ module Gitlab :throttle_authenticated_web, :throttle_authenticated_protected_paths_api, :throttle_authenticated_protected_paths_web, - :throttle_authenticated_packages_api + :throttle_authenticated_packages_api, + :throttle_authenticated_git_lfs, + :throttle_authenticated_files_api ].freeze PAYLOAD_KEYS = [ diff --git a/lib/gitlab/middleware/sidekiq_web_static.rb b/lib/gitlab/middleware/sidekiq_web_static.rb new file mode 100644 index 00000000000..61b5fb9e0c6 --- /dev/null +++ b/lib/gitlab/middleware/sidekiq_web_static.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# This module removes the X-Sendfile-Type header for /admin/sidekiq +# assets since Workhorse isn't always guaranteed to have the assets +# present on disk, such as when using Cloud Native GitLab +# containers. These assets are also small and served infrequently so it +# should be fine to do this. +module Gitlab + module Middleware + class SidekiqWebStatic + SIDEKIQ_REGEX = %r{\A/admin/sidekiq/}.freeze + + def initialize(app) + @app = app + end + + def call(env) + env.delete('HTTP_X_SENDFILE_TYPE') if env['PATH_INFO'] =~ SIDEKIQ_REGEX + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 2e54e8bfc1a..693f1470d9d 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -65,8 +65,15 @@ module Gitlab # Note: By default the order is breadth-first # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all) - recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order + cte = base_and_ancestors_cte(upto, hierarchy_order) + + recursive_query = if hierarchy_order + # othewise depth won't be available for outer query + cte.apply_to(unscoped_model.all.select(objects_table[Arel.star])).order(depth: hierarchy_order) + else + cte.apply_to(unscoped_model.all) + end + read_only(recursive_query) end # rubocop: enable CodeReuse/ActiveRecord @@ -78,7 +85,10 @@ module Gitlab # and incremented as we go down the descendant tree # rubocop: disable CodeReuse/ActiveRecord def base_and_descendants(with_depth: false) - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(unscoped_model.all)) + outer_select_relation = unscoped_model.all + outer_select_relation = outer_select_relation.select(objects_table[Arel.star]) if with_depth # Otherwise Active Record will not select `depth` as it's not a table column + + read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(outer_select_relation)) end # rubocop: enable CodeReuse/ActiveRecord @@ -145,7 +155,7 @@ module Gitlab cte = SQL::RecursiveCTE.new(:base_and_ancestors) base_query = ancestors_base.except(:order) - base_query = base_query.select("1 as #{DEPTH_COLUMN}", "ARRAY[#{objects_table.name}.id] AS tree_path", "false AS tree_cycle", objects_table[Arel.star]) if hierarchy_order + base_query = base_query.select("1 as #{DEPTH_COLUMN}", "ARRAY[#{objects_table.name}.id] AS tree_path", "false AS tree_cycle", base_query.default_select_columns) if hierarchy_order cte << base_query @@ -162,7 +172,7 @@ module Gitlab cte.table[DEPTH_COLUMN] + 1, "tree_path || #{quoted_objects_table_name}.id", "#{quoted_objects_table_name}.id = ANY(tree_path)", - objects_table[Arel.star] + parent_query.default_select_columns ).where(cte.table[:tree_cycle].eq(false)) end @@ -178,7 +188,7 @@ module Gitlab cte = SQL::RecursiveCTE.new(:base_and_descendants) base_query = descendants_base.except(:order) - base_query = base_query.select("1 AS #{DEPTH_COLUMN}", "ARRAY[#{objects_table.name}.id] AS tree_path", "false AS tree_cycle", objects_table[Arel.star]) if with_depth + base_query = base_query.select("1 AS #{DEPTH_COLUMN}", "ARRAY[#{objects_table.name}.id] AS tree_path", "false AS tree_cycle", base_query.default_select_columns) if with_depth cte << base_query @@ -195,7 +205,7 @@ module Gitlab cte.table[DEPTH_COLUMN] + 1, "tree_path || #{quoted_objects_table_name}.id", "#{quoted_objects_table_name}.id = ANY(tree_path)", - objects_table[Arel.star] + descendants_query.default_select_columns ).where(cte.table[:tree_cycle].eq(false)) end diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb new file mode 100644 index 00000000000..f19cdf06d9a --- /dev/null +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module CursorBasedKeyset + SUPPORTED_ORDERING = { + Group => { name: :asc } + }.freeze + + def self.available_for_type?(relation) + SUPPORTED_ORDERING.key?(relation.klass) + end + + def self.available?(cursor_based_request_context, relation) + available_for_type?(relation) && + order_satisfied?(relation, cursor_based_request_context) + end + + def self.order_satisfied?(relation, cursor_based_request_context) + order_by_from_request = cursor_based_request_context.order_by + + SUPPORTED_ORDERING[relation.klass] == order_by_from_request + end + private_class_method :order_satisfied? + end + end +end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index b05891066ac..a16bf7a379c 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -14,23 +14,39 @@ module Gitlab # It is expected that the given finder will respond to `execute` method with `gitaly_pagination: true` option # and supports pagination via gitaly. def paginate(finder) - return paginate_via_gitaly(finder) if keyset_pagination_enabled? - return paginate_first_page_via_gitaly(finder) if paginate_first_page? + return paginate_via_gitaly(finder) if keyset_pagination_enabled?(finder) + return paginate_first_page_via_gitaly(finder) if paginate_first_page?(finder) - branches = ::Kaminari.paginate_array(finder.execute) + records = ::Kaminari.paginate_array(finder.execute) Gitlab::Pagination::OffsetPagination .new(request_context) - .paginate(branches) + .paginate(records) end private - def keyset_pagination_enabled? - Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml) && params[:pagination] == 'keyset' + def keyset_pagination_enabled?(finder) + return false unless params[:pagination] == "keyset" + + if finder.is_a?(BranchesFinder) + Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml) + elsif finder.is_a?(::Repositories::TreeFinder) + Feature.enabled?(:repository_tree_gitaly_pagination, project, default_enabled: :yaml) + else + false + end end - def paginate_first_page? - Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml) && (params[:page].blank? || params[:page].to_i == 1) + def paginate_first_page?(finder) + return false unless params[:page].blank? || params[:page].to_i == 1 + + if finder.is_a?(BranchesFinder) + Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml) + elsif finder.is_a?(::Repositories::TreeFinder) + Feature.enabled?(:repository_tree_gitaly_pagination, project, default_enabled: :yaml) + else + false + end end def paginate_via_gitaly(finder) @@ -43,7 +59,7 @@ module Gitlab # Headers are added to immitate offset pagination, while it is the default option def paginate_first_page_via_gitaly(finder) finder.execute(gitaly_pagination: true).tap do |records| - total = project.repository.branch_count + total = finder.total per_page = params[:per_page].presence || Kaminari.config.default_per_page Gitlab::Pagination::OffsetHeaderBuilder.new( diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb index 0755af9587b..2b968c4253f 100644 --- a/lib/gitlab/pagination/keyset/column_order_definition.rb +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -173,6 +173,18 @@ module Gitlab distinct end + def order_direction_as_sql_string + sql_string = ascending_order? ? +'ASC' : +'DESC' + + if nulls_first? + sql_string << ' NULLS FIRST' + elsif nulls_last? + sql_string << ' NULLS LAST' + end + + sql_string + end + private attr_reader :reversed_order_expression, :nullable, :distinct diff --git a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb new file mode 100644 index 00000000000..18390f5b59d --- /dev/null +++ b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class CursorBasedRequestContext + DEFAULT_SORT_DIRECTION = :desc + attr_reader :request_context + delegate :params, to: :request_context + + def initialize(request_context) + @request_context = request_context + end + + def per_page + params[:per_page] + end + + def cursor + params[:cursor] + end + + def apply_headers(cursor_for_next_page) + Gitlab::Pagination::Keyset::HeaderBuilder + .new(request_context) + .add_next_page_header({ cursor: cursor_for_next_page }) + end + + def order_by + { params[:order_by].to_sym => params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION } + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/cursor_pager.rb b/lib/gitlab/pagination/keyset/cursor_pager.rb new file mode 100644 index 00000000000..0b49aa87a02 --- /dev/null +++ b/lib/gitlab/pagination/keyset/cursor_pager.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class CursorPager < Gitlab::Pagination::Base + attr_reader :cursor_based_request_context, :paginator + + def initialize(cursor_based_request_context) + @cursor_based_request_context = cursor_based_request_context + end + + def paginate(relation) + @paginator ||= relation.keyset_paginate( + per_page: cursor_based_request_context.per_page, + cursor: cursor_based_request_context.cursor + ) + + paginator.records + end + + def finalize(_records = []) + # can be called only after executing `paginate(relation)` + apply_headers + end + + private + + def apply_headers + return unless paginator.has_next_page? + + cursor_based_request_context + .apply_headers(paginator.cursor_for_next_page) + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb new file mode 100644 index 00000000000..95afd5a8595 --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + class ArrayScopeColumns + ARRAY_SCOPE_CTE_NAME = 'array_cte' + + def initialize(columns) + validate_columns!(columns) + + array_scope_table = Arel::Table.new(ARRAY_SCOPE_CTE_NAME) + @columns = columns.map do |column| + ColumnData.new(column, "array_scope_#{column}", array_scope_table) + end + end + + def array_scope_cte_name + ARRAY_SCOPE_CTE_NAME + end + + def array_aggregated_columns + columns.map(&:array_aggregated_column) + end + + def array_aggregated_column_names + columns.map(&:array_aggregated_column_name) + end + + def arel_columns + columns.map(&:arel_column) + end + + def array_lookup_expressions_by_position(table_name) + columns.map do |column| + Arel.sql("#{table_name}.#{column.array_aggregated_column_name}[position]") + end + end + + private + + attr_reader :columns + + def validate_columns!(columns) + if columns.blank? + msg = <<~MSG + No array columns were given. + Make sure you explicitly select the columns in the array_scope parameter. + Example: Project.select(:id) + MSG + raise StandardError, msg + end + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb new file mode 100644 index 00000000000..3f620f74eca --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/column_data.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + class ColumnData + attr_reader :original_column_name, :as, :arel_table + + def initialize(original_column_name, as, arel_table) + @original_column_name = original_column_name.to_s + @as = as.to_s + @arel_table = arel_table + end + + def projection + arel_column.as(as) + end + + def arel_column + arel_table[original_column_name] + end + + def arel_column_as + arel_table[as] + end + + def array_aggregated_column_name + "#{arel_table.name}_#{original_column_name}_array" + end + + def array_aggregated_column + Arel::Nodes::NamedFunction.new('ARRAY_AGG', [arel_column]).as(array_aggregated_column_name) + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb new file mode 100644 index 00000000000..d8c69a74e6b --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/order_by_columns.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + class OrderByColumns + include Enumerable + + # This class exposes collection methods for the order by columns + # + # Example: by modelling the `issues.created_at ASC, issues.id ASC` ORDER BY + # SQL clause, this class will receive two ColumnOrderDefinition objects + def initialize(columns, arel_table) + @columns = columns.map do |column| + ColumnData.new(column.attribute_name, "order_by_columns_#{column.attribute_name}", arel_table) + end + end + + def arel_columns + columns.map(&:arel_column) + end + + def array_aggregated_columns + columns.map(&:array_aggregated_column) + end + + def array_aggregated_column_names + columns.map(&:array_aggregated_column_name) + end + + def original_column_names + columns.map(&:original_column_name) + end + + def original_column_names_as_arel_string + columns.map { |c| Arel.sql(c.original_column_name) } + end + + def original_column_names_as_tmp_tamble + temp_table = Arel::Table.new('record') + original_column_names.map { |c| temp_table[c] } + end + + def cursor_values(table_name) + columns.each_with_object({}) do |column, hash| + hash[column.original_column_name] = Arel.sql("#{table_name}.#{column.array_aggregated_column_name}[position]") + end + end + + def array_lookup_expressions_by_position(table_name) + columns.map do |column| + Arel.sql("#{table_name}.#{column.array_aggregated_column_name}[position]") + end + end + + def replace_value_in_array_by_position_expressions + columns.map do |column| + name = "#{QueryBuilder::RECURSIVE_CTE_NAME}.#{column.array_aggregated_column_name}" + new_value = "next_cursor_values.#{column.original_column_name}" + "#{name}[:position_query.position-1]||#{new_value}||#{name}[position_query.position+1:]" + end + end + + def each(&block) + columns.each(&block) + end + + private + + attr_reader :columns + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb new file mode 100644 index 00000000000..39d6e016ac7 --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -0,0 +1,290 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + # rubocop: disable CodeReuse/ActiveRecord + class QueryBuilder + UnsupportedScopeOrder = Class.new(StandardError) + + RECURSIVE_CTE_NAME = 'recursive_keyset_cte' + RECORDS_COLUMN = 'records' + + # This class optimizes slow database queries (PostgreSQL specific) where the + # IN SQL operator is used with sorting. + # + # Arguments: + # scope - ActiveRecord::Relation supporting keyset pagination + # array_scope - ActiveRecord::Relation for the `IN` subselect + # array_mapping_scope - Lambda for connecting scope with array_scope + # finder_query - ActiveRecord::Relation for finding one row by the passed in cursor values + # values - keyset cursor values (optional) + # + # Example ActiveRecord query: Issues in the namespace hierarchy + # > scope = Issue + # > .where(project_id: Group.find(9970).all_projects.select(:id)) + # > .order(:created_at, :id) + # > .limit(20); + # + # Optimized version: + # + # > scope = Issue.where({}).order(:created_at, :id) # base scope + # > array_scope = Group.find(9970).all_projects.select(:id) + # > array_mapping_scope = -> (id_expression) { Issue.where(Issue.arel_table[:project_id].eq(id_expression)) } + # + # # finding the record by id is good enough, we can ignore the created_at_expression + # > finder_query = -> (created_at_expression, id_expression) { Issue.where(Issue.arel_table[:id].eq(id_expression)) } + # + # > Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder.new( + # > scope: scope, + # > array_scope: array_scope, + # > array_mapping_scope: array_mapping_scope, + # > finder_query: finder_query + # > ).execute.limit(20) + def initialize(scope:, array_scope:, array_mapping_scope:, finder_query:, values: {}) + @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) + + unless success + error_message = <<~MSG + The order on the scope does not support keyset pagination. You might need to define a custom Order object.\n + See https://docs.gitlab.com/ee/development/database/keyset_pagination.html#complex-order-configuration\n + Or the Gitlab::Pagination::Keyset::Order class for examples + MSG + raise(UnsupportedScopeOrder, error_message) + end + + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + @array_scope = array_scope + @array_mapping_scope = array_mapping_scope + @finder_query = finder_query + @values = values + @model = @scope.model + @table_name = @model.table_name + @arel_table = @model.arel_table + end + + def execute + selector_cte = Gitlab::SQL::CTE.new(:array_cte, array_scope) + + cte = Gitlab::SQL::RecursiveCTE.new(RECURSIVE_CTE_NAME, union_args: { remove_duplicates: false, remove_order: false }) + cte << initializer_query + cte << data_collector_query + + q = cte + .apply_to(model.where({}) + .with(selector_cte.to_arel)) + .select(result_collector_final_projections) + .where("count <> 0") # filter out the initializer row + + model.from(q.arel.as(table_name)) + end + + private + + attr_reader :array_scope, :scope, :order, :array_mapping_scope, :finder_query, :values, :model, :table_name, :arel_table + + def initializer_query + array_column_names = array_scope_columns.array_aggregated_column_names + order_by_columns.array_aggregated_column_names + + projections = [ + *result_collector_initializer_columns, + *array_column_names, + '0::bigint AS count' + ] + + model.select(projections).from(build_column_arrays_query).limit(1) + end + + # This query finds the first cursor values for each item in the array CTE. + # + # array_cte: + # + # |project_id| + # |----------| + # | 1| + # | 2| + # | 3| + # | 4| + # + # For each project_id, find the first issues row by respecting the created_at, id order. + # + # The `array_mapping_scope` parameter defines how the `array_scope` and the `scope` can be combined. + # + # scope = Issue.where({}) # empty scope + # array_mapping_scope = Issue.where(project_id: X) + # + # scope.merge(array_mapping_scope) # Issue.where(project_id: X) + # + # X will be replaced with a value from the `array_cte` temporary table. + # + # |created_at|id| + # |----------|--| + # |2020-01-15| 2| + # |2020-01-07| 3| + # |2020-01-07| 4| + # |2020-01-10| 5| + def build_column_arrays_query + q = Arel::SelectManager.new + .project(array_scope_columns.array_aggregated_columns + order_by_columns.array_aggregated_columns) + .from(array_cte) + .join(Arel.sql("LEFT JOIN LATERAL (#{initial_keyset_query.to_sql}) #{table_name} ON TRUE")) + + order_by_columns.each { |column| q.where(column.arel_column.not_eq(nil)) } + + q.as('array_scope_lateral_query') + end + + def array_cte + Arel::SelectManager.new + .project(array_scope_columns.arel_columns) + .from(Arel.sql(array_scope_columns.array_scope_cte_name)) + .as(array_scope_columns.array_scope_cte_name) + end + + def initial_keyset_query + keyset_scope = scope.merge(array_mapping_scope.call(*array_scope_columns.arel_columns)) + order + .apply_cursor_conditions(keyset_scope, values, use_union_optimization: true) + .reselect(*order_by_columns.arel_columns) + .limit(1) + end + + def data_collector_query + array_column_list = array_scope_columns.array_aggregated_column_names + + order_column_value_arrays = order_by_columns.replace_value_in_array_by_position_expressions + + select = [ + *result_collector_columns, + *array_column_list, + *order_column_value_arrays, + "#{RECURSIVE_CTE_NAME}.count + 1" + ] + + from = <<~SQL + #{RECURSIVE_CTE_NAME}, + #{array_order_query.lateral.as('position_query').to_sql}, + #{ensure_one_row(next_cursor_values_query).lateral.as('next_cursor_values').to_sql} + SQL + + model.select(select).from(from) + end + + # NULL guard. This method ensures that NULL values are returned when the passed in scope returns 0 rows. + # Example query: returns issues.id or NULL + # + # SELECT issues.id FROM (VALUES (NULL)) nulls (id) + # LEFT JOIN (SELECT id FROM issues WHERE id = 1 LIMIT 1) issues ON TRUE + # LIMIT 1 + def ensure_one_row(query) + q = Arel::SelectManager.new + q.projections = order_by_columns.original_column_names_as_tmp_tamble + + null_values = [nil] * order_by_columns.count + + from = Arel::Nodes::Grouping.new(Arel::Nodes::ValuesList.new([null_values])).as('nulls') + + q.from(from) + q.join(Arel.sql("LEFT JOIN (#{query.to_sql}) record ON TRUE")) + q.limit = 1 + q + end + + # This subquery finds the cursor values for the next record by sorting the generated cursor arrays in memory and taking the first element. + # It combines the cursor arrays (UNNEST) together and sorts them according to the originally defined ORDER BY clause. + # + # Example: issues in the group hierarchy with ORDER BY created_at, id + # + # |project_id| |created_at|id| # 2 arrays combined: issues_created_at_array, issues_id_array + # |----------| |----------|--| + # | 1| |2020-01-15| 2| + # | 2| |2020-01-07| 3| + # | 3| |2020-01-07| 4| + # | 4| |2020-01-10| 5| + # + # The query will return the cursor values: (2020-01-07, 3) and the array position: 1 + # From the position, we can tell that the record belongs to the project with id 2. + def array_order_query + q = Arel::SelectManager.new + .project([*order_by_columns.original_column_names_as_arel_string, Arel.sql('position')]) + .from("UNNEST(#{list(order_by_columns.array_aggregated_column_names)}) WITH ORDINALITY AS u(#{list(order_by_columns.original_column_names)}, position)") + + order_by_columns.each { |column| q.where(Arel.sql(column.original_column_name).not_eq(nil)) } # ignore rows where all columns are NULL + + q.order(Arel.sql(order_by_without_table_references)).take(1) + end + + # This subquery finds the next cursor values after the previously determined position (from array_order_query). + # The current cursor values are passed in as SQL literals since the actual values are encoded into SQL arrays. + # + # Example: issues in the group hierarchy with ORDER BY created_at, id + # + # |project_id| |created_at|id| # 2 arrays combined: issues_created_at_array, issues_id_array + # |----------| |----------|--| + # | 1| |2020-01-15| 2| + # | 2| |2020-01-07| 3| + # | 3| |2020-01-07| 4| + # | 4| |2020-01-10| 5| + # + # Assuming that the determined position is 1, the cursor values will be the following: + # - Filter: project_id = 2 + # - created_at = 2020-01-07 + # - id = 3 + def next_cursor_values_query + cursor_values = order_by_columns.cursor_values(RECURSIVE_CTE_NAME) + array_mapping_scope_columns = array_scope_columns.array_lookup_expressions_by_position(RECURSIVE_CTE_NAME) + + keyset_scope = scope + .reselect(*order_by_columns.arel_columns) + .merge(array_mapping_scope.call(*array_mapping_scope_columns)) + + order + .apply_cursor_conditions(keyset_scope, cursor_values, use_union_optimization: true) + .reselect(*order_by_columns.arel_columns) + .limit(1) + end + + # Generates an ORDER BY clause by using the column position index and the original order clauses. + # This method is used to sort the collected arrays in SQL. + # Example: "issues".created_at DESC , "issues".id ASC => 1 DESC, 2 ASC + def order_by_without_table_references + order.column_definitions.each_with_index.map do |column_definition, i| + "#{i + 1} #{column_definition.order_direction_as_sql_string}" + end.join(", ") + end + + def result_collector_initializer_columns + ["NULL::#{table_name} AS #{RECORDS_COLUMN}"] + end + + def result_collector_columns + query = finder_query + .call(*order_by_columns.array_lookup_expressions_by_position(RECURSIVE_CTE_NAME)) + .select("#{table_name}") + .limit(1) + + ["(#{query.to_sql})"] + end + + def result_collector_final_projections + ["(#{RECORDS_COLUMN}).*"] + end + + def array_scope_columns + @array_scope_columns ||= ArrayScopeColumns.new(array_scope.select_values) + end + + def order_by_columns + @order_by_columns ||= OrderByColumns.new(order.column_definitions, arel_table) + end + + def list(array) + array.join(', ') + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb index c6f0014a0f4..14807fa37c4 100644 --- a/lib/gitlab/pagination/keyset/iterator.rb +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -6,12 +6,13 @@ module Gitlab class Iterator UnsupportedScopeOrder = Class.new(StandardError) - def initialize(scope:, use_union_optimization: true) + def initialize(scope:, use_union_optimization: true, in_operator_optimization_options: nil) @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) - @use_union_optimization = use_union_optimization + @use_union_optimization = in_operator_optimization_options ? false : use_union_optimization + @in_operator_optimization_options = in_operator_optimization_options end # rubocop: disable CodeReuse/ActiveRecord @@ -19,11 +20,10 @@ module Gitlab cursor_attributes = {} loop do - current_scope = scope.dup.limit(of) - relation = order - .apply_cursor_conditions(current_scope, cursor_attributes, { use_union_optimization: @use_union_optimization }) - .reorder(order) - .limit(of) + current_scope = scope.dup + relation = order.apply_cursor_conditions(current_scope, cursor_attributes, keyset_options) + relation = relation.reorder(order) unless @in_operator_optimization_options + relation = relation.limit(of) yield relation @@ -38,6 +38,13 @@ module Gitlab private attr_reader :scope, :order + + def keyset_options + { + use_union_optimization: @use_union_optimization, + in_operator_optimization_options: @in_operator_optimization_options + } + end end end end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index ccfa9334a12..80726fc8efd 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -152,15 +152,24 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def apply_cursor_conditions(scope, values = {}, options = { use_union_optimization: false }) + def apply_cursor_conditions(scope, values = {}, options = { use_union_optimization: false, in_operator_optimization_options: nil }) values ||= {} transformed_values = values.with_indifferent_access - scope = apply_custom_projections(scope) + scope = apply_custom_projections(scope.dup) where_values = build_where_values(transformed_values) if options[:use_union_optimization] && where_values.size > 1 build_union_query(scope, where_values).reorder(self) + elsif options[:in_operator_optimization_options] + opts = options[:in_operator_optimization_options] + + Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder.new( + **{ + scope: scope.reorder(self), + values: values + }.merge(opts) + ).execute else scope.where(build_or_query(where_values)) # rubocop: disable CodeReuse/ActiveRecord end @@ -187,7 +196,7 @@ module Gitlab columns = Arel::Nodes::Grouping.new(column_definitions.map(&:column_expression)) values = Arel::Nodes::Grouping.new(column_definitions.map do |column_definition| value = values[column_definition.attribute_name] - Arel::Nodes.build_quoted(value, column_definition.column_expression) + build_quoted(value, column_definition.column_expression) end) if column_definitions.first.ascending_order? @@ -197,6 +206,12 @@ module Gitlab end end + def build_quoted(value, column_expression) + return value if value.instance_of?(Arel::Nodes::SqlLiteral) + + Arel::Nodes.build_quoted(value, column_expression) + end + # Adds extra columns to the SELECT clause def apply_custom_projections(scope) additional_projections = column_definitions.select(&:add_to_projections).map do |column_definition| @@ -204,7 +219,7 @@ module Gitlab column_definition.column_expression.dup.as(column_definition.attribute_name).to_sql end - scope = scope.select(*scope.arel.projections, *additional_projections) if additional_projections + scope = scope.reselect(*scope.arel.projections, *additional_projections) unless additional_projections.blank? scope end diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 4f8a6ffb2cc..7b5013f137b 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -29,7 +29,7 @@ module Gitlab return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) limited_total_count = pagination_data.total_count_with_limit - if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + if limited_total_count > max_limit # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` # We need to call `reset` because `without_count` relies on `@arel` being unmemoized pagination_data.reset.without_count @@ -38,6 +38,14 @@ module Gitlab end end + def max_limit + if Feature.enabled?(:lower_relation_max_count_limit, type: :ops) + Kaminari::ActiveRecordRelationMethods::MAX_COUNT_NEW_LOWER_LIMIT + else + Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + end + end + def needs_pagination?(relation) return true unless relation.respond_to?(:current_page) return true if params[:page].present? && relation.current_page != params[:page].to_i diff --git a/lib/gitlab/patch/legacy_database_config.rb b/lib/gitlab/patch/legacy_database_config.rb new file mode 100644 index 00000000000..a7d4fdf7490 --- /dev/null +++ b/lib/gitlab/patch/legacy_database_config.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +# The purpose of this code is to transform legacy `database.yml` +# into a `database.yml` containing `main:` as a name of a first database +# +# This should be removed once all places using legacy `database.yml` +# are fixed. The likely moment to remove this check is the %14.0. +# +# This converts the following syntax: +# +# production: +# adapter: postgresql +# database: gitlabhq_production +# username: git +# password: "secure password" +# host: localhost +# +# Into: +# +# production: +# main: +# adapter: postgresql +# database: gitlabhq_production +# username: git +# password: "secure password" +# host: localhost +# + +module Gitlab + module Patch + module LegacyDatabaseConfig + extend ActiveSupport::Concern + + prepended do + attr_reader :uses_legacy_database_config + end + + def database_configuration + @uses_legacy_database_config = false # rubocop:disable Gitlab/ModuleWithInstanceVariables + + super.to_h do |env, configs| + # This check is taken from Rails where the transformation + # of a flat database.yml is done into `primary:` + # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 + if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } + configs = { "main" => configs } + + @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + [env, configs] + end + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 4cb47ffc6d9..c648f4d1fd0 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -184,6 +184,10 @@ module Gitlab @repository_git_route_regex ||= /#{repository_route_regex}\.git/.freeze end + def repository_git_lfs_route_regex + @repository_git_lfs_route_regex ||= %r{#{repository_git_route_regex}\/(info\/lfs|gitlab-lfs)\/}.freeze + end + def repository_wiki_git_route_regex @repository_wiki_git_route_regex ||= /#{full_namespace_route_regex}\.*\.wiki\.git/.freeze end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index ff17ecf8024..c5cf3262039 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -267,7 +267,7 @@ module Gitlab private def zoom_link_service - Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) + ::Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) end end end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 47c76e98e5c..6348a4902f8 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -155,6 +155,20 @@ module Gitlab @execution_message[:approve] = _('Approved the current merge request.') end + desc _('Unapprove a merge request') + explanation _('Unapprove the current merge request.') + types MergeRequest + condition do + quick_action_target.persisted? && quick_action_target.can_be_unapproved_by?(current_user) + end + command :unapprove do + success = MergeRequests::RemoveApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) + + next unless success + + @execution_message[:unapprove] = _('Unapproved the current merge request.') + end + desc do if quick_action_target.allows_multiple_reviewers? _('Assign reviewer(s)') diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index 175f32bd4c6..3f4c0fa45aa 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -82,15 +82,27 @@ module Gitlab end def self.configure_throttles(rack_attack) - throttle_or_track(rack_attack, 'throttle_unauthenticated', Gitlab::Throttle.unauthenticated_options) do |req| - if req.throttle_unauthenticated? - req.ip + # Each of these settings follows the same pattern of specifying separate + # authenticated and unauthenticated rates via settings + Gitlab::Throttle::REGULAR_THROTTLES.each do |throttle| + unauthenticated_options = Gitlab::Throttle.options(throttle, authenticated: false) + throttle_or_track(rack_attack, "throttle_unauthenticated_#{throttle}", unauthenticated_options) do |req| + if req.throttle?(throttle, authenticated: false) + req.ip + end + end + + authenticated_options = Gitlab::Throttle.options(throttle, authenticated: true) + throttle_or_track(rack_attack, "throttle_authenticated_#{throttle}", authenticated_options) do |req| + if req.throttle?(throttle, authenticated: true) + req.throttled_user_id([:api]) + end end end - throttle_or_track(rack_attack, 'throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req| - if req.throttle_authenticated_api? - req.throttled_user_id([:api]) + throttle_or_track(rack_attack, 'throttle_unauthenticated_web', Gitlab::Throttle.unauthenticated_web_options) do |req| + if req.throttle_unauthenticated_web? + req.ip end end @@ -127,14 +139,8 @@ module Gitlab end end - throttle_or_track(rack_attack, 'throttle_unauthenticated_packages_api', Gitlab::Throttle.unauthenticated_packages_api_options) do |req| - if req.throttle_unauthenticated_packages_api? - req.ip - end - end - - throttle_or_track(rack_attack, 'throttle_authenticated_packages_api', Gitlab::Throttle.authenticated_packages_api_options) do |req| - if req.throttle_authenticated_packages_api? + throttle_or_track(rack_attack, 'throttle_authenticated_git_lfs', Gitlab::Throttle.throttle_authenticated_git_lfs_options) do |req| + if req.throttle_authenticated_git_lfs? req.throttled_user_id([:api]) end end @@ -159,7 +165,15 @@ module Gitlab return false if dry_run_config.empty? return true if dry_run_config == '*' - dry_run_config.split(',').map(&:strip).include?(name) + dry_run_throttles = dry_run_config.split(',').map(&:strip) + + # `throttle_unauthenticated` was split into API and web, so to maintain backwards-compatibility + # this throttle name now controls both rate limits. + if dry_run_throttles.include?('throttle_unauthenticated') + dry_run_throttles += %w[throttle_unauthenticated_api throttle_unauthenticated_web] + end + + dry_run_throttles.include?(name) end def self.user_allowlist diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index 7fee6a1b43d..099174842d0 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -3,6 +3,8 @@ module Gitlab module RackAttack module Request + FILES_PATH_REGEX = %r{^/api/v\d+/projects/[^/]+/repository/files/.+}.freeze + def unauthenticated? !(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id) end @@ -58,9 +60,25 @@ module Gitlab path =~ protected_paths_regex end - def throttle_unauthenticated? + def throttle?(throttle, authenticated:) + fragment = Gitlab::Throttle.throttle_fragment!(throttle, authenticated: authenticated) + + __send__("#{fragment}?") # rubocop:disable GitlabSecurity/PublicSend + end + + def throttle_unauthenticated_api? + api_request? && !should_be_skipped? && !throttle_unauthenticated_packages_api? && + !throttle_unauthenticated_files_api? && + Gitlab::Throttle.settings.throttle_unauthenticated_api_enabled && + unauthenticated? + end + + def throttle_unauthenticated_web? + web_request? && + !should_be_skipped? && + # TODO: Column will be renamed in https://gitlab.com/gitlab-org/gitlab/-/issues/340031 Gitlab::Throttle.settings.throttle_unauthenticated_enabled && unauthenticated? end @@ -68,11 +86,13 @@ module Gitlab def throttle_authenticated_api? api_request? && !throttle_authenticated_packages_api? && + !throttle_authenticated_files_api? && Gitlab::Throttle.settings.throttle_authenticated_api_enabled end def throttle_authenticated_web? web_request? && + !throttle_authenticated_git_lfs? && Gitlab::Throttle.settings.throttle_authenticated_web_enabled end @@ -109,6 +129,24 @@ module Gitlab Gitlab::Throttle.settings.throttle_authenticated_packages_api_enabled end + def throttle_authenticated_git_lfs? + git_lfs_path? && + Gitlab::Throttle.settings.throttle_authenticated_git_lfs_enabled + end + + def throttle_unauthenticated_files_api? + files_api_path? && + Feature.enabled?(:files_api_throttling, default_enabled: :yaml) && + Gitlab::Throttle.settings.throttle_unauthenticated_files_api_enabled && + unauthenticated? + end + + def throttle_authenticated_files_api? + files_api_path? && + Feature.enabled?(:files_api_throttling, default_enabled: :yaml) && + Gitlab::Throttle.settings.throttle_authenticated_files_api_enabled + end + private def authenticated_user_id(request_formats) @@ -130,6 +168,14 @@ module Gitlab def packages_api_path? path =~ ::Gitlab::Regex::Packages::API_PATH_REGEX end + + def git_lfs_path? + path =~ Gitlab::PathRegex.repository_git_lfs_route_regex + end + + def files_api_path? + path =~ FILES_PATH_REGEX + end end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 547549361be..f914123a94d 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -6,16 +6,11 @@ module Gitlab REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability).freeze attr_accessor :project, :current_user, :author - # This counter is increased by a number of references filtered out by - # banzai reference exctractor. Note that this counter is stateful and - # not idempotent and is increased whenever you call `references`. - attr_reader :stateful_not_visible_counter def initialize(project, current_user = nil) @project = project @current_user = current_user @references = {} - @stateful_not_visible_counter = 0 super() end @@ -26,14 +21,19 @@ module Gitlab def references(type, ids_only: false) refs = super(type, project, current_user, ids_only: ids_only) - @stateful_not_visible_counter += refs[:not_visible].count + update_visible_nodes_set(refs[:nodes], refs[:visible_nodes]) refs[:visible] end + # this method is stateful, it tracks if all nodes from `references` + # calls are visible or not + def all_visible? + not_visible_nodes.empty? + end + def reset_memoized_values @references = {} - @stateful_not_visible_counter = 0 super() end @@ -76,5 +76,16 @@ module Gitlab @pattern = Regexp.union(patterns.compact) end + + private + + def update_visible_nodes_set(all, visible) + not_visible_nodes.merge(all) + not_visible_nodes.subtract(visible) + end + + def not_visible_nodes + @not_visible_nodes ||= Set.new + end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 698a417283e..a88ef5fe73e 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -21,7 +21,8 @@ module Gitlab end def composer_package_version_regex - @composer_package_version_regex ||= %r{^v?(\d+(\.(\d+|x))*(-.+)?)}.freeze + # see https://github.com/composer/semver/blob/31f3ea725711245195f62e54ffa402d8ef2fdba9/src/VersionParser.php#L215 + @composer_package_version_regex ||= %r{\Av?((\d++)(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?)?\z}.freeze end def composer_dev_version_regex diff --git a/lib/gitlab/repository_cache/preloader.rb b/lib/gitlab/repository_cache/preloader.rb new file mode 100644 index 00000000000..726dde4e0ca --- /dev/null +++ b/lib/gitlab/repository_cache/preloader.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + class RepositoryCache + class Preloader + def initialize(repositories) + @repositories = repositories + end + + def preload(methods) + return if @repositories.empty? + + cache_keys = [] + + sources_by_cache_key = @repositories.each_with_object({}) do |repository, hash| + methods.each do |method| + cache_key = repository.cache.cache_key(method) + + hash[cache_key] = { repository: repository, method: method } + cache_keys << cache_key + end + end + + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + backend.read_multi(*cache_keys).each do |cache_key, value| + source = sources_by_cache_key[cache_key] + + source[:repository].memoize_method_cache_value(source[:method], value) + end + end + end + + private + + def backend + @repositories.first.cache.backend + end + end + end +end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index d0230c035cc..c096c870f2a 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -217,6 +217,10 @@ module Gitlab fallback end + def memoize_method_cache_value(method, value) + strong_memoize(memoizable_name(method)) { value } + end + # Expires the caches of a specific set of methods def expire_method_caches(methods) methods.each do |name| diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb index 8d9d8415cb1..e87c2a0b700 100644 --- a/lib/gitlab/saas.rb +++ b/lib/gitlab/saas.rb @@ -20,6 +20,22 @@ module Gitlab def self.dev_url 'https://dev.gitlab.org' end + + def self.registry_prefix + 'registry.gitlab.com' + end + + def self.customer_support_url + 'https://support.gitlab.com' + end + + def self.customer_license_support_url + 'https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293' + end + + def self.gitlab_com_status_url + 'https://status.gitlab.com' + end end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 90513e346f2..217a48e740d 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -188,7 +188,7 @@ module Gitlab merge_requests = MergeRequestsFinder.new(current_user, issuable_params).execute unless default_project_filter - merge_requests = merge_requests.in_projects(project_ids_relation) + merge_requests = merge_requests.of_projects(project_ids_relation) end apply_sort(merge_requests, scope: 'merge_requests') diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 4158cec9b09..5671fce481f 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -1,17 +1,5 @@ # frozen_string_literal: true -# :nocov: -module DeliverNever - def deliver_later - self - end -end - -module MuteNotifications - def new_note(note) - end -end - module Gitlab class Seeder extend ActionView::Helpers::NumberHelper @@ -82,18 +70,22 @@ module Gitlab Project.include(ProjectSeed) User.include(UserSeed) - mute_notifications - mute_mailer + old_perform_deliveries = ActionMailer::Base.perform_deliveries + ActionMailer::Base.perform_deliveries = false SeedFu.quiet = true without_statement_timeout do - yield + without_new_note_notifications do + yield + end end + puts "\nOK".color(:green) + ensure SeedFu.quiet = false + ActionMailer::Base.perform_deliveries = old_perform_deliveries ActiveRecord::Base.logger = old_logger - puts "\nOK".color(:green) end def self.without_gitaly_timeout @@ -109,12 +101,14 @@ module Gitlab ApplicationSetting.expire end - def self.mute_notifications - NotificationService.prepend(MuteNotifications) - end + def self.without_new_note_notifications + NotificationService.alias_method :original_new_note, :new_note + NotificationService.define_method(:new_note) { |note| } - def self.mute_mailer - ActionMailer::MessageDelivery.prepend(DeliverNever) + yield + ensure + NotificationService.alias_method :new_note, :original_new_note + NotificationService.remove_method :original_new_note end def self.without_statement_timeout diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index 05319ba17a2..3dee257229d 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -57,6 +57,11 @@ module Gitlab worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path) queue_groups = argv.map do |queues_or_query_string| + if queues_or_query_string =~ /[\r\n]/ + raise CommandError, + 'The queue arguments cannot contain newlines' + end + next worker_queues if queues_or_query_string == SidekiqConfig::WorkerMatcher::WILDCARD_MATCH # When using the queue query syntax, we treat each queue group diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 842e53b2ffb..1aebce987fe 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -69,6 +69,7 @@ module Gitlab message = base_message(payload) payload['load_balancing_strategy'] = job['load_balancing_strategy'] if job['load_balancing_strategy'] + payload['dedup_wal_locations'] = job['dedup_wal_locations'] if job['dedup_wal_locations'].present? if job_exception payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 3422cb47516..d084e9e9d7e 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -26,12 +26,14 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::BatchLoader chain.add ::Labkit::Middleware::Sidekiq::Server chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger - chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware if load_balancing_enabled? chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server + # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, + # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server + chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware if load_balancing_enabled? end end @@ -42,13 +44,15 @@ module Gitlab lambda do |chain| chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware chain.add ::Labkit::Middleware::Sidekiq::Client + # Sidekiq Client Middleware should be placed before DuplicateJobs::Client middleware, + # so we can store WAL location before we deduplicate the job. + chain.add ::Gitlab::Database::LoadBalancing::SidekiqClientMiddleware if load_balancing_enabled? chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client chain.add ::Gitlab::SidekiqStatus::ClientMiddleware chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client - # Size limiter should be placed at the bottom, but before the metrics midleware + # Size limiter should be placed at the bottom, but before the metrics middleware chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Client chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics - chain.add ::Gitlab::Database::LoadBalancing::SidekiqClientMiddleware if load_balancing_enabled? end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index c1dc616cbb2..aeb58d7c153 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -17,10 +17,26 @@ module Gitlab # # When new jobs can be scheduled again, the strategy calls `#delete`. class DuplicateJob + include Gitlab::Utils::StrongMemoize + DUPLICATE_KEY_TTL = 6.hours + WAL_LOCATION_TTL = 60.seconds + MAX_REDIS_RETRIES = 5 DEFAULT_STRATEGY = :until_executing STRATEGY_NONE = :none + LUA_SET_WAL_SCRIPT = <<~EOS + local key, wal, offset, ttl = KEYS[1], ARGV[1], tonumber(ARGV[2]), ARGV[3] + local existing_offset = redis.call("LINDEX", key, -1) + if existing_offset == false then + redis.call("RPUSH", key, wal, offset) + redis.call("EXPIRE", key, ttl) + elseif offset > tonumber(existing_offset) then + redis.call("LSET", key, 0, wal) + redis.call("LSET", key, -1, offset) + end + EOS + attr_reader :existing_jid def initialize(job, queue_name) @@ -44,22 +60,59 @@ module Gitlab # This method will return the jid that was set in redis def check!(expiry = DUPLICATE_KEY_TTL) read_jid = nil + read_wal_locations = {} Sidekiq.redis do |redis| redis.multi do |multi| redis.set(idempotency_key, jid, ex: expiry, nx: true) + read_wal_locations = check_existing_wal_locations!(redis, expiry) read_jid = redis.get(idempotency_key) end end job['idempotency_key'] = idempotency_key + # We need to fetch values since the read_wal_locations and read_jid were obtained inside transaction, under redis.multi command. + self.existing_wal_locations = read_wal_locations.transform_values(&:value) self.existing_jid = read_jid.value end + def update_latest_wal_location! + return unless job_wal_locations.present? + + Sidekiq.redis do |redis| + redis.multi do + job_wal_locations.each do |connection_name, location| + redis.eval(LUA_SET_WAL_SCRIPT, keys: [wal_location_key(connection_name)], argv: [location, pg_wal_lsn_diff(connection_name).to_i, WAL_LOCATION_TTL]) + end + end + end + end + + def latest_wal_locations + return {} unless job_wal_locations.present? + + strong_memoize(:latest_wal_locations) do + read_wal_locations = {} + + Sidekiq.redis do |redis| + redis.multi do + job_wal_locations.keys.each do |connection_name| + read_wal_locations[connection_name] = redis.lindex(wal_location_key(connection_name), 0) + end + end + end + + read_wal_locations.transform_values(&:value).compact + end + end + def delete! Sidekiq.redis do |redis| - redis.del(idempotency_key) + redis.multi do |multi| + redis.del(idempotency_key) + delete_wal_locations!(redis) + end end end @@ -93,6 +146,7 @@ module Gitlab private + attr_accessor :existing_wal_locations attr_reader :queue_name, :job attr_writer :existing_jid @@ -100,6 +154,10 @@ module Gitlab @worker_klass ||= worker_class_name.to_s.safe_constantize end + def pg_wal_lsn_diff(connection_name) + Gitlab::Database::DATABASES[connection_name].pg_wal_lsn_diff(job_wal_locations[connection_name], existing_wal_locations[connection_name]) + end + def strategy return DEFAULT_STRATEGY unless worker_klass return DEFAULT_STRATEGY unless worker_klass.respond_to?(:idempotent?) @@ -120,6 +178,20 @@ module Gitlab job['jid'] end + def job_wal_locations + return {} unless preserve_wal_location? + + job['wal_locations'] || {} + end + + def existing_wal_location_key(connection_name) + "#{idempotency_key}:#{connection_name}:existing_wal_location" + end + + def wal_location_key(connection_name) + "#{idempotency_key}:#{connection_name}:wal_location" + end + def idempotency_key @idempotency_key ||= job['idempotency_key'] || "#{namespace}:#{idempotency_hash}" end @@ -135,6 +207,29 @@ module Gitlab def idempotency_string "#{worker_class_name}:#{Sidekiq.dump_json(arguments)}" end + + def delete_wal_locations!(redis) + job_wal_locations.keys.each do |connection_name| + redis.del(wal_location_key(connection_name)) + redis.del(existing_wal_location_key(connection_name)) + end + end + + def check_existing_wal_locations!(redis, expiry) + read_wal_locations = {} + + job_wal_locations.each do |connection_name, location| + key = existing_wal_location_key(connection_name) + redis.set(key, location, ex: expiry, nx: true) + read_wal_locations[connection_name] = redis.get(key) + end + + read_wal_locations + end + + def preserve_wal_location? + Feature.enabled?(:preserve_latest_wal_locations_for_idempotent_jobs, default_enabled: :yaml) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb index 469033a5e52..fc58d4f5323 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb @@ -14,6 +14,8 @@ module Gitlab job['duplicate-of'] = duplicate_job.existing_jid if duplicate_job.idempotent? + duplicate_job.update_latest_wal_location! + Gitlab::SidekiqLogging::DeduplicationLogger.instance.log( job, "dropped #{strategy_name}", duplicate_job.options) return false @@ -23,8 +25,16 @@ module Gitlab yield end + def perform(job) + update_job_wal_location!(job) + end + private + def update_job_wal_location!(job) + job['dedup_wal_locations'] = duplicate_job.latest_wal_locations if duplicate_job.latest_wal_locations.present? + end + def deduplicatable_job? !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled] end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb index 738efa36fc8..5164b994267 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb @@ -8,9 +8,14 @@ module Gitlab # removes the lock after the job has executed preventing a new job to be queued # while a job is still executing. class UntilExecuted < Base + extend ::Gitlab::Utils::Override + include DeduplicatesWhenScheduling - def perform(_job) + override :perform + def perform(job) + super + yield duplicate_job.delete! diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb index 68d66383b2b..1f7e3a4ea30 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb @@ -8,9 +8,13 @@ module Gitlab # removes the lock before the job starts allowing a new job to be queued # while a job is still executing. class UntilExecuting < Base + extend ::Gitlab::Utils::Override + include DeduplicatesWhenScheduling - def perform(_job) + override :perform + def perform(job) + super duplicate_job.delete! yield diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index b37eeb8bad1..a83522a489a 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -4,12 +4,12 @@ module Gitlab module SidekiqMiddleware module SizeLimiter # Handle a Sidekiq job payload limit based on current configuration. - # This validator pulls the configuration from the environment variables: - # - GITLAB_SIDEKIQ_SIZE_LIMITER_MODE: the current mode of the size - # limiter. This must be either `track` or `compress`. - # - GITLAB_SIDEKIQ_SIZE_LIMITER_COMPRESSION_THRESHOLD_BYTES: the - # threshold before the input job payload is compressed. - # - GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES: the size limit in bytes. + # This validator pulls the configuration from application settings: + # - limiter_mode: the current mode of the size + # limiter. This must be either `track` or `compress`. + # - compression_threshold_bytes: the threshold before the input job + # payload is compressed. + # - limit_bytes: the size limit in bytes. # # In track mode, if a job payload limit exceeds the size limit, an # event is sent to Sentry and the job is scheduled like normal. @@ -18,12 +18,29 @@ module Gitlab # then compressed. If the compressed payload still exceeds the limit, the # job is discarded, and a ExceedLimitError exception is raised. class Validator - def self.validate!(worker_class, job) - new(worker_class, job).validate! + # Avoid limiting the size of jobs for `BackgroundMigrationWorker` classes. + # We can't read the configuration from `ApplicationSetting` for those jobs + # when migrating a path that modifies the `application_settings` table. + # Reading the application settings through `ApplicationSetting#current` + # causes a `SELECT` with a list of column names, but that list of column + # names might not match what the table currently looks like causing + # an error when scheduling background migrations. + # + # The worker classes aren't constants here, because that would force + # Application Settings to be loaded earlier causing failures loading + # the environmant in rake tasks + EXEMPT_WORKER_NAMES = ["BackgroundMigrationWorker", "Database::BatchedBackgroundMigrationWorker"].to_set + + class << self + def validate!(worker_class, job) + return if EXEMPT_WORKER_NAMES.include?(worker_class.to_s) + + new(worker_class, job).validate! + end end DEFAULT_SIZE_LIMIT = 0 - DEFAULT_COMPRESION_THRESHOLD_BYTES = 100_000 # 100kb + DEFAULT_COMPRESSION_THRESHOLD_BYTES = 100_000 # 100kb MODES = [ TRACK_MODE = 'track', @@ -34,9 +51,9 @@ module Gitlab def initialize( worker_class, job, - mode: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_MODE'], - compression_threshold: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_COMPRESSION_THRESHOLD_BYTES'], - size_limit: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES'] + mode: Gitlab::CurrentSettings.sidekiq_job_limiter_mode, + compression_threshold: Gitlab::CurrentSettings.sidekiq_job_limiter_compression_threshold_bytes, + size_limit: Gitlab::CurrentSettings.sidekiq_job_limiter_limit_bytes ) @worker_class = worker_class @job = job @@ -47,11 +64,11 @@ module Gitlab end def validate! - return unless @size_limit > 0 - return if allow_big_payload? - job_args = compress_if_necessary(::Sidekiq.dump_json(@job['args'])) + + return if @size_limit == 0 return if job_args.bytesize <= @size_limit + return if allow_big_payload? exception = exceed_limit_error(job_args) if compress_mode? @@ -72,10 +89,10 @@ module Gitlab end def set_compression_threshold(compression_threshold) - @compression_threshold = (compression_threshold || DEFAULT_COMPRESION_THRESHOLD_BYTES).to_i + @compression_threshold = (compression_threshold || DEFAULT_COMPRESSION_THRESHOLD_BYTES).to_i if @compression_threshold <= 0 ::Sidekiq.logger.warn "Invalid Sidekiq size limiter compression threshold: #{@compression_threshold}" - @compression_threshold = DEFAULT_COMPRESION_THRESHOLD_BYTES + @compression_threshold = DEFAULT_COMPRESSION_THRESHOLD_BYTES end end @@ -83,6 +100,7 @@ module Gitlab @size_limit = (size_limit || DEFAULT_SIZE_LIMIT).to_i if @size_limit < 0 ::Sidekiq.logger.warn "Invalid Sidekiq size limiter limit: #{@size_limit}" + @size_limit = DEFAULT_SIZE_LIMIT end end diff --git a/lib/gitlab/sidekiq_queue.rb b/lib/gitlab/sidekiq_queue.rb index eb3a8e3d497..67a9d8120d8 100644 --- a/lib/gitlab/sidekiq_queue.rb +++ b/lib/gitlab/sidekiq_queue.rb @@ -7,6 +7,9 @@ module Gitlab NoMetadataError = Class.new(StandardError) InvalidQueueError = Class.new(StandardError) + WORKER_KEY = 'worker_class' + ALLOWED_KEYS = Gitlab::ApplicationContext::KNOWN_KEYS + [WORKER_KEY] + attr_reader :queue_name def initialize(queue_name) @@ -21,8 +24,8 @@ module Gitlab job_search_metadata = search_metadata .stringify_keys - .slice(*Gitlab::ApplicationContext::KNOWN_KEYS) - .transform_keys { |key| "meta.#{key}" } + .slice(*ALLOWED_KEYS) + .transform_keys(&method(:transform_key)) .compact raise NoMetadataError if job_search_metadata.empty? @@ -49,6 +52,14 @@ module Gitlab private + def transform_key(key) + if Gitlab::ApplicationContext::KNOWN_KEYS.include?(key) + "meta.#{key}" + elsif key == WORKER_KEY + 'class' + end + end + def queue strong_memoize(:queue) do # Sidekiq::Queue.new always returns a queue, even if it doesn't diff --git a/lib/gitlab/signed_tag.rb b/lib/gitlab/signed_tag.rb index 3b22cb7622d..49194300a39 100644 --- a/lib/gitlab/signed_tag.rb +++ b/lib/gitlab/signed_tag.rb @@ -7,12 +7,7 @@ module Gitlab def initialize(repository, tag) @repository = repository @tag = tag - - if Feature.enabled?(:get_tag_signatures) - @signature_data = Gitlab::Git::Tag.extract_signature_lazily(repository, tag.id) if repository - else - @signature_data = [signature_text_of_message.b, signed_text_of_message.b] - end + @signature_data = Gitlab::Git::Tag.extract_signature_lazily(repository, tag.id) if repository end def signature @@ -26,22 +21,5 @@ module Gitlab def signed_text @signature_data&.fetch(1) end - - private - - def signature_text_of_message - @tag.message.slice(@tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1) - rescue StandardError - nil - end - - def signed_text_of_message - %{object #{@tag.target_commit.id} -type commit -tag #{@tag.name} -tagger #{@tag.tagger.name} <#{@tag.tagger.email}> #{@tag.tagger.date.seconds} #{@tag.tagger.timezone} - -#{@tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}} - end end end diff --git a/lib/gitlab/slash_commands/issue_close.rb b/lib/gitlab/slash_commands/issue_close.rb index 3dad7216983..5d33f2fe62d 100644 --- a/lib/gitlab/slash_commands/issue_close.rb +++ b/lib/gitlab/slash_commands/issue_close.rb @@ -29,7 +29,7 @@ module Gitlab private def close_issue(issue:) - Issues::CloseService.new(project: project, current_user: current_user).execute(issue) + ::Issues::CloseService.new(project: project, current_user: current_user).execute(issue) end def presenter(issue) diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb index 0612663017c..9f10da247d7 100644 --- a/lib/gitlab/slash_commands/issue_move.rb +++ b/lib/gitlab/slash_commands/issue_move.rb @@ -29,11 +29,11 @@ module Gitlab return Gitlab::SlashCommands::Presenters::Access.new.not_found end - new_issue = Issues::MoveService.new(project: project, current_user: current_user) + new_issue = ::Issues::MoveService.new(project: project, current_user: current_user) .execute(old_issue, target_project) presenter(new_issue).present(old_issue) - rescue Issues::MoveService::MoveError => e + rescue ::Issues::MoveService::MoveError => e presenter(old_issue).display_move_error(e.message) end diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index fab016d2e1b..ef368767689 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -33,7 +33,7 @@ module Gitlab private def create_issue(title:, description:) - Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute + ::Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute end def presenter(issue) diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index ab2e1404cd2..78fa5009bc4 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -13,6 +13,38 @@ module Gitlab def self.payment_form_url "#{self.subscriptions_url}/payment_forms/cc_validation" end + + def self.subscriptions_comparison_url + 'https://about.gitlab.com/pricing/gitlab-com/feature-comparison' + end + + def self.subscriptions_graphql_url + "#{self.subscriptions_url}/graphql" + end + + def self.subscriptions_more_minutes_url + "#{self.subscriptions_url}/buy_pipeline_minutes" + end + + def self.subscriptions_more_storage_url + "#{self.subscriptions_url}/buy_storage" + end + + def self.subscriptions_manage_url + "#{self.subscriptions_url}/subscriptions" + end + + def self.subscriptions_plans_url + "#{self.subscriptions_url}/plans" + end + + def self.subscription_portal_admin_email + ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_EMAIL', 'gl_com_api@gitlab.com') + end + + def self.subscription_portal_admin_token + ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_TOKEN', 'customer_admin_token') + end end end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index da925f0f83a..263483ba54b 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -7,6 +7,7 @@ module Gitlab TEMPLATES_WITH_LATEST_VERSION = { 'Jobs/Browser-Performance-Testing' => true, + 'Jobs/Build' => true, 'Security/API-Fuzzing' => true, 'Security/DAST' => true, 'Terraform' => true diff --git a/lib/gitlab/throttle.rb b/lib/gitlab/throttle.rb index 8f045021088..622dc7d9ed0 100644 --- a/lib/gitlab/throttle.rb +++ b/lib/gitlab/throttle.rb @@ -4,6 +4,11 @@ module Gitlab class Throttle DEFAULT_RATE_LIMITING_RESPONSE_TEXT = 'Retry later' + # Each of these settings follows the same pattern of specifying separate + # authenticated and unauthenticated rates via settings. New throttles should + # ideally be regular as well. + REGULAR_THROTTLES = [:api, :packages_api, :files_api].freeze + def self.settings Gitlab::CurrentSettings.current_application_settings end @@ -24,21 +29,38 @@ module Gitlab "HTTP_#{env_value.upcase.tr('-', '_')}" end - def self.unauthenticated_options + class << self + def options(throttle, authenticated:) + fragment = throttle_fragment!(throttle, authenticated: authenticated) + + # rubocop:disable GitlabSecurity/PublicSend + limit_proc = proc { |req| settings.public_send("#{fragment}_requests_per_period") } + period_proc = proc { |req| settings.public_send("#{fragment}_period_in_seconds").seconds } + # rubocop:enable GitlabSecurity/PublicSend + + { limit: limit_proc, period: period_proc } + end + + def throttle_fragment!(throttle, authenticated:) + raise("Unknown throttle: #{throttle}") unless REGULAR_THROTTLES.include?(throttle) + + "throttle_#{'un' unless authenticated}authenticated_#{throttle}" + end + end + + def self.unauthenticated_web_options + # TODO: Columns will be renamed in https://gitlab.com/gitlab-org/gitlab/-/issues/340031 + # Once this is done, web can be made into a regular throttle limit_proc = proc { |req| settings.throttle_unauthenticated_requests_per_period } period_proc = proc { |req| settings.throttle_unauthenticated_period_in_seconds.seconds } - { limit: limit_proc, period: period_proc } - end - def self.authenticated_api_options - limit_proc = proc { |req| settings.throttle_authenticated_api_requests_per_period } - period_proc = proc { |req| settings.throttle_authenticated_api_period_in_seconds.seconds } { limit: limit_proc, period: period_proc } end def self.authenticated_web_options limit_proc = proc { |req| settings.throttle_authenticated_web_requests_per_period } period_proc = proc { |req| settings.throttle_authenticated_web_period_in_seconds.seconds } + { limit: limit_proc, period: period_proc } end @@ -49,16 +71,9 @@ module Gitlab { limit: limit_proc, period: period_proc } end - def self.unauthenticated_packages_api_options - limit_proc = proc { |req| settings.throttle_unauthenticated_packages_api_requests_per_period } - period_proc = proc { |req| settings.throttle_unauthenticated_packages_api_period_in_seconds.seconds } - - { limit: limit_proc, period: period_proc } - end - - def self.authenticated_packages_api_options - limit_proc = proc { |req| settings.throttle_authenticated_packages_api_requests_per_period } - period_proc = proc { |req| settings.throttle_authenticated_packages_api_period_in_seconds.seconds } + def self.throttle_authenticated_git_lfs_options + limit_proc = proc { |req| settings.throttle_authenticated_git_lfs_requests_per_period } + period_proc = proc { |req| settings.throttle_authenticated_git_lfs_period_in_seconds.seconds } { limit: limit_proc, period: period_proc } end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index 4f974c98c71..b72d33113dd 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -6,7 +6,8 @@ module Gitlab original_record_timestamps = model.record_timestamps model.record_timestamps = false - if block.arity.abs == 1 + # negative arity means arguments are optional + if block.arity == 1 || block.arity < 0 block.call(model) else block.call diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 5fb360296b7..f4fbea50515 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -18,7 +18,7 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end - def snowplow_options(group) + def options(group) additional_features = Feature.enabled?(:additional_snowplow_tracking, group) { namespace: SNOWPLOW_NAMESPACE, diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 7902f96dfa6..fe5669be014 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -8,7 +8,8 @@ module Gitlab def initialize(namespace: nil, project: nil, user: nil, **extra) @namespace = namespace - @plan = @namespace&.actual_plan_name + @plan = namespace&.actual_plan_name + @project = project @extra = extra end @@ -34,14 +35,29 @@ module Gitlab private + attr_accessor :namespace, :project, :extra, :plan + def to_h { environment: environment, source: source, - plan: @plan, - extra: @extra + plan: plan, + extra: extra + }.merge(project_and_namespace) + end + + def project_and_namespace + return {} unless ::Feature.enabled?(:add_namespace_and_project_to_snowplow_tracking, default_enabled: :yaml) + + { + namespace_id: namespace&.id, + project_id: project_id } end + + def project_id + project.is_a?(Integer) ? project : project&.id + end end end end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index a242f718b16..c00a0e1bcb4 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -24,6 +24,8 @@ module Gitlab instance.project_job_url(object.project, object, **options) when Commit commit_url(object, **options) + when Compare + compare_url(object, **options) when Group instance.group_canonical_url(object, **options) when Issue @@ -68,6 +70,12 @@ module Gitlab instance.commit_url(commit, **options) end + def compare_url(compare, **options) + return '' unless compare.project + + instance.project_compare_url(compare.project, **options.merge(compare.to_param)) + end + def note_url(note, **options) if note.for_commit? return '' unless note.project diff --git a/lib/gitlab/usage/metrics/instrumentations/service_ping_features_metric.rb b/lib/gitlab/usage/metrics/instrumentations/service_ping_features_metric.rb new file mode 100644 index 00000000000..dcf528c8136 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/service_ping_features_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ServicePingFeaturesMetric < GenericMetric + value do + Gitlab::CurrentSettings.usage_ping_features_enabled + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 910c8397f20..854242031be 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -259,7 +259,8 @@ module Gitlab smtp_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::SmtpConfig.encrypted_secrets.active? }, operating_system: alt_usage_data(fallback: nil) { operating_system }, gitaly_apdex: alt_usage_data { gitaly_apdex }, - collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none') + collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none'), + service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none') } } end @@ -918,7 +919,7 @@ module Gitlab jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord fogbugz: projects_imported_count('fogbugz', time_period), phabricator: projects_imported_count('phabricator', time_period), - csv: count(Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord + csv: count(::Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord } end @@ -934,7 +935,7 @@ module Gitlab project_imports = distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id) bulk_imports = distinct_count(::BulkImport.where(time_period), :user_id) jira_issue_imports = distinct_count(::JiraImportState.where(time_period), :user_id) - csv_issue_imports = distinct_count(Issues::CsvImport.where(time_period), :user_id) + csv_issue_imports = distinct_count(::Issues::CsvImport.where(time_period), :user_id) group_imports = distinct_count(::GroupImportState.where(time_period), :user_id) add(project_imports, bulk_imports, jira_issue_imports, csv_issue_imports, group_imports) diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index c9106d7c6b8..e5a50c92329 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -3,6 +3,7 @@ module Gitlab::UsageDataCounters class CiTemplateUniqueCounter REDIS_SLOT = 'ci_templates' + KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__) # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_` TEMPLATE_TO_EVENT = { @@ -20,19 +21,24 @@ module Gitlab::UsageDataCounters class << self def track_unique_project_event(project_id:, template:, config_source:) - if event = unique_project_event(template, config_source) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id) - end + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(ci_template_event_name(template, config_source), values: project_id) end - private + def ci_templates(relative_base = 'lib/gitlab/ci/templates') + Dir.glob('**/*.gitlab-ci.yml', base: Rails.root.join(relative_base)) + end + + def ci_template_event_name(template_name, config_source) + prefix = 'implicit_' if config_source.to_s == 'auto_devops_source' + template_event_name = TEMPLATE_TO_EVENT[template_name] || template_to_event_name(template_name) - def unique_project_event(template, config_source) - if name = TEMPLATE_TO_EVENT[template] - prefix = 'implicit_' if config_source.to_s == 'auto_devops_source' + "p_#{REDIS_SLOT}_#{prefix}#{template_event_name}" + end + + private - "p_#{REDIS_SLOT}_#{prefix}#{name}" - end + def template_to_event_name(template) + ActiveSupport::Inflector.parameterize(template.chomp('.gitlab-ci.yml'), separator: '_').underscore end end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 96562a44391..8fc8bb5d344 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -18,6 +18,24 @@ module Gitlab KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze + CATEGORIES_FOR_TOTALS = %w[ + analytics + code_review + compliance + deploy_token_packages + ecosystem + epic_boards_usage + epics_usage + ide_edit + incident_management + issues_edit + pipeline_authoring + quickactions + search + testing + user_packages + ].freeze + # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # @@ -90,7 +108,7 @@ module Gitlab hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event])) end - if eligible_for_totals?(events_names) + if eligible_for_totals?(events_names) && CATEGORIES_FOR_TOTALS.include?(category) event_results["#{category}_total_unique_counts_weekly"] = unique_events(**weekly_time_range.merge(event_names: events_names)) event_results["#{category}_total_unique_counts_monthly"] = unique_events(**monthly_time_range.merge(event_names: events_names)) end diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index e4f20b61901..261bdeb9bfa 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -2,84 +2,67 @@ category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: i_analytics_dev_ops_adoption category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: i_analytics_dev_ops_score category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: p_analytics_merge_request category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: i_analytics_instance_statistics category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: g_analytics_contribution category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: g_analytics_insights category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: g_analytics_issues category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: g_analytics_productivity category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: g_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: p_analytics_pipelines category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: p_analytics_code_reviews category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: p_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: p_analytics_insights category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: p_analytics_issues category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: p_analytics_repo category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits - name: i_analytics_cohorts category: analytics redis_slot: analytics aggregation: weekly - feature_flag: track_unique_visits diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 3c692f2b1af..cf790767f17 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -4,73 +4,590 @@ redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_auto_devops_build +# Explicit include:template pipeline events +- name: p_ci_templates_5_min_production_app category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_auto_devops_deploy +- name: p_ci_templates_aws_cf_deploy_ec2 category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_sast +- name: p_ci_templates_auto_devops_build category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_secret_detection +- name: p_ci_templates_auto_devops_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly -# Explicit include:template pipeline events -- name: p_ci_templates_5_min_production_app +- name: p_ci_templates_auto_devops_deploy_latest category: ci_templates redis_slot: ci_templates aggregation: weekly +# This part of the file is generated automatically by +# bin/rake gitlab:usage_data:generate_ci_template_events +# +# Do not edit it manually! +# +# The section above this should be removed once we roll out tracking all ci +# templates +# https://gitlab.com/gitlab-org/gitlab/-/issues/339684 + +- name: p_ci_templates_terraform_base_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_terraform_base + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_dotnet + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_nodejs + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_openshift + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_auto_devops category: ci_templates redis_slot: ci_templates aggregation: weekly - -- name: p_ci_templates_aws_cf_deploy_ec2 +- name: p_ci_templates_bash category: ci_templates redis_slot: ci_templates aggregation: weekly - -- name: p_ci_templates_aws_deploy_ecs +- name: p_ci_templates_rust category: ci_templates redis_slot: ci_templates aggregation: weekly - -- name: p_ci_templates_auto_devops_build +- name: p_ci_templates_elixir category: ci_templates redis_slot: ci_templates aggregation: weekly - -- name: p_ci_templates_auto_devops_deploy +- name: p_ci_templates_clojure category: ci_templates redis_slot: ci_templates aggregation: weekly - -- name: p_ci_templates_auto_devops_deploy_latest +- name: p_ci_templates_crystal + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_getting_started + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_code_quality + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_verify_load_performance_testing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_verify_accessibility + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_verify_failfast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_verify_browser_performance + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_verify_browser_performance_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_grails category: ci_templates redis_slot: ci_templates aggregation: weekly - - name: p_ci_templates_security_sast category: ci_templates redis_slot: ci_templates aggregation: weekly - +- name: p_ci_templates_security_dast_runner_validation + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_dast_on_demand_scan + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly - -- name: p_ci_templates_terraform_base_latest +- name: p_ci_templates_security_license_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_coverage_fuzzing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_api_fuzzing_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_secure_binaries + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_dast_api + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_container_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_dast_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_dependency_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_api_fuzzing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_dast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_cluster_image_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_ios_fastlane + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_composer + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_c + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_python + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_android_fastlane + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_android_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_django + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_maven + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_flutter + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_workflows_branch_pipelines + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_workflows_mergerequest_pipelines + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_laravel + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_managed_cluster_applications + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_php + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_packer + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_terraform + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_mono + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_serverless + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_go + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_scala + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_latex + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_android + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_indeni_cloudrail + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_deploy_ecs + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_aws_cf_provision_and_deploy_ec2 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_aws_deploy_ecs + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_gradle + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_chef + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_dast_default_branch_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_load_performance_testing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_helm_2to3 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_code_intelligence + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_code_quality + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_deploy_ecs + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_deploy_ec2 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_build + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_browser_performance_testing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_test + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_deploy_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_browser_performance_testing_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_cf_provision + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_build_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_terraform_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_swift + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_jekyll + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_harp + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_octopress + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_brunch + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_doxygen + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_hyde + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_lektor + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_jbake + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_hexo + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_middleman + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_hugo + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_pelican + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_nanoc + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_swaggerui + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_jigsaw + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_metalsmith + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_gatsby + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_pages_html + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_dart + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_docker + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_julia + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_npm + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_dotnet_core + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_5_minute_production_app + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_ruby + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_load_performance_testing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_helm_2to3 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_code_intelligence + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_code_quality + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_deploy_ecs + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_deploy_ec2 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_auto_devops_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_auto_devops_build + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_browser_performance_testing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_test + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_auto_devops_deploy_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_browser_performance_testing_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_cf_provision + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_build_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_dast_runner_validation + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_dast_on_demand_scan + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_license_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_coverage_fuzzing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_api_fuzzing_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_secure_binaries + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_dast_api + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_container_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_dast_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_dependency_scanning + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_api_fuzzing + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_dast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_security_cluster_image_scanning category: ci_templates redis_slot: ci_templates aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 7ad51bfe832..d4a818f8fe0 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -237,3 +237,15 @@ category: code_review aggregation: weekly feature_flag: diff_searching_usage_data +- name: i_code_review_total_suggestions_applied + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_total_suggestions_added + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_resolve_thread_in_issue + redis_slot: code_review + category: code_review + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 3db0482d38e..261d3b37783 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -4,27 +4,22 @@ redis_slot: compliance category: compliance aggregation: weekly - feature_flag: track_unique_visits - name: g_compliance_audit_events category: compliance redis_slot: compliance aggregation: weekly - feature_flag: track_unique_visits - name: i_compliance_audit_events category: compliance redis_slot: compliance aggregation: weekly - feature_flag: track_unique_visits - name: i_compliance_credential_inventory category: compliance redis_slot: compliance aggregation: weekly - feature_flag: track_unique_visits - name: a_compliance_audit_events_api category: compliance redis_slot: compliance aggregation: weekly - feature_flag: track_unique_visits - name: g_edit_by_web_ide category: ide_edit redis_slot: edit @@ -67,7 +62,6 @@ - name: design_action category: source_code aggregation: daily - feature_flag: usage_data_design_action - name: project_action category: source_code aggregation: daily @@ -160,7 +154,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_metrics_report_widget_total - name: i_testing_group_code_coverage_visit_total category: testing redis_slot: testing @@ -174,7 +167,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_web_performance_widget_total - name: i_testing_group_code_coverage_project_click_total category: testing redis_slot: testing @@ -183,7 +175,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_load_performance_widget_total - name: i_testing_metrics_report_artifact_uploaders category: testing redis_slot: testing diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 7df351859fb..7f77fa8ee02 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -3,6 +3,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_unapprove + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_assign_single category: quickactions redis_slot: quickactions diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 0d6f4b93aee..0fadd68aeab 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -20,8 +20,10 @@ module Gitlab MR_CREATE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_create_multiline_mr_comment' MR_EDIT_MULTILINE_COMMENT_ACTION = 'i_code_review_user_edit_multiline_mr_comment' MR_REMOVE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_remove_multiline_mr_comment' - MR_ADD_SUGGESTION_ACTION = 'i_code_review_user_add_suggestion' - MR_APPLY_SUGGESTION_ACTION = 'i_code_review_user_apply_suggestion' + MR_USER_ADD_SUGGESTION_ACTION = 'i_code_review_user_add_suggestion' + MR_TOTAL_ADD_SUGGESTION_ACTION = 'i_code_review_total_suggestions_added' + MR_USER_APPLY_SUGGESTION_ACTION = 'i_code_review_user_apply_suggestion' + MR_TOTAL_APPLY_SUGGESTION_ACTION = 'i_code_review_total_suggestions_applied' MR_MARKED_AS_DRAFT_ACTION = 'i_code_review_user_marked_as_draft' MR_UNMARKED_AS_DRAFT_ACTION = 'i_code_review_user_unmarked_as_draft' MR_RESOLVE_THREAD_ACTION = 'i_code_review_user_resolve_thread' @@ -46,6 +48,7 @@ module Gitlab MR_LABELS_CHANGED_ACTION = 'i_code_review_user_labels_changed' MR_LOAD_CONFLICT_UI_ACTION = 'i_code_review_user_load_conflict_ui' MR_RESOLVE_CONFLICT_ACTION = 'i_code_review_user_resolve_conflict' + MR_RESOLVE_THREAD_IN_ISSUE_ACTION = 'i_code_review_user_resolve_thread_in_issue' class << self def track_mr_diffs_action(merge_request:) @@ -112,8 +115,9 @@ module Gitlab track_unique_action_by_user(MR_PUBLISH_REVIEW_ACTION, user) end - def track_add_suggestion_action(user:) - track_unique_action_by_user(MR_ADD_SUGGESTION_ACTION, user) + def track_add_suggestion_action(note:) + track_unique_action_by_user(MR_USER_ADD_SUGGESTION_ACTION, note.author) + track_unique_action_by_objects(MR_TOTAL_ADD_SUGGESTION_ACTION, note.suggestions) end def track_marked_as_draft_action(user:) @@ -124,16 +128,17 @@ module Gitlab track_unique_action_by_user(MR_UNMARKED_AS_DRAFT_ACTION, user) end - def track_apply_suggestion_action(user:) - track_unique_action_by_user(MR_APPLY_SUGGESTION_ACTION, user) + def track_apply_suggestion_action(user:, suggestions:) + track_unique_action_by_user(MR_USER_APPLY_SUGGESTION_ACTION, user) + track_unique_action_by_objects(MR_TOTAL_APPLY_SUGGESTION_ACTION, suggestions) end def track_users_assigned_to_mr(users:) - track_unique_action_by_users(MR_ASSIGNED_USERS_ACTION, users) + track_unique_action_by_objects(MR_ASSIGNED_USERS_ACTION, users) end def track_users_review_requested(users:) - track_unique_action_by_users(MR_REVIEW_REQUESTED_USERS_ACTION, users) + track_unique_action_by_objects(MR_REVIEW_REQUESTED_USERS_ACTION, users) end def track_title_edit_action(user:) @@ -210,6 +215,10 @@ module Gitlab track_unique_action_by_user(MR_RESOLVE_CONFLICT_ACTION, user) end + def track_resolve_thread_in_issue_action(user:) + track_unique_action_by_user(MR_RESOLVE_THREAD_IN_ISSUE_ACTION, user) + end + private def track_unique_action_by_merge_request(action, merge_request) @@ -222,10 +231,10 @@ module Gitlab track_unique_action(action, user.id) end - def track_unique_action_by_users(action, users) - return if users.blank? + def track_unique_action_by_objects(action, objects) + return if objects.blank? - track_unique_action(action, users.map(&:id)) + track_unique_action(action, objects.map(&:id)) end def track_unique_action(action, value) diff --git a/lib/gitlab/zentao/client.rb b/lib/gitlab/zentao/client.rb new file mode 100644 index 00000000000..bdfa4b3a308 --- /dev/null +++ b/lib/gitlab/zentao/client.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Zentao + class Client + Error = Class.new(StandardError) + ConfigError = Class.new(Error) + + attr_reader :integration + + def initialize(integration) + raise ConfigError, 'Please check your integration configuration.' unless integration + + @integration = integration + end + + def ping + response = fetch_product(zentao_product_xid) + + active = response.fetch('deleted') == '0' rescue false + + if active + { success: true } + else + { success: false, message: 'Not Found' } + end + end + + def fetch_product(product_id) + get("products/#{product_id}") + end + + def fetch_issues(params = {}) + get("products/#{zentao_product_xid}/issues", + params.reverse_merge(page: 1, limit: 20)) + end + + def fetch_issue(issue_id) + get("issues/#{issue_id}") + end + + private + + def get(path, params = {}) + options = { headers: headers, query: params } + response = Gitlab::HTTP.get(url(path), options) + + return {} unless response.success? + + Gitlab::Json.parse(response.body) + rescue JSON::ParserError + {} + end + + def url(path) + host = integration.api_url.presence || integration.url + + URI.join(host, '/api.php/v1/', path) + end + + def headers + { + 'Content-Type': 'application/json', + 'Token': integration.api_token + } + end + + def zentao_product_xid + integration.zentao_product_xid + end + end + end +end diff --git a/lib/object_storage/config.rb b/lib/object_storage/config.rb index f933d4e4866..82d9fc07043 100644 --- a/lib/object_storage/config.rb +++ b/lib/object_storage/config.rb @@ -84,13 +84,16 @@ module ObjectStorage def fog_attributes @fog_attributes ||= begin - return {} unless enabled? && aws? - return {} unless server_side_encryption.present? + return {} unless aws_server_side_encryption_enabled? aws_server_side_encryption_headers.compact end end + def aws_server_side_encryption_enabled? + aws? && server_side_encryption.present? + end + private # This returns a Hash of HTTP encryption headers to send along to S3. diff --git a/lib/sidebars/concerns/has_pill.rb b/lib/sidebars/concerns/has_pill.rb index 4bbf69bf16b..0a2e1f12008 100644 --- a/lib/sidebars/concerns/has_pill.rb +++ b/lib/sidebars/concerns/has_pill.rb @@ -21,8 +21,8 @@ module Sidebars {} end - def format_cached_count(count_service, count) - if count > count_service::CACHED_COUNT_THRESHOLD + def format_cached_count(threshold, count) + if count > threshold number_to_human( count, units: { thousand: 'k', million: 'm' }, precision: 1, significant: false, format: '%n%u' diff --git a/lib/sidebars/groups/menus/ci_cd_menu.rb b/lib/sidebars/groups/menus/ci_cd_menu.rb index e870bbf5ebc..f5bce57f496 100644 --- a/lib/sidebars/groups/menus/ci_cd_menu.rb +++ b/lib/sidebars/groups/menus/ci_cd_menu.rb @@ -11,11 +11,6 @@ module Sidebars true end - override :link - def link - renderable_items.first.link - end - override :title def title _('CI/CD') diff --git a/lib/sidebars/groups/menus/group_information_menu.rb b/lib/sidebars/groups/menus/group_information_menu.rb index b28cb927ad2..9656811455e 100644 --- a/lib/sidebars/groups/menus/group_information_menu.rb +++ b/lib/sidebars/groups/menus/group_information_menu.rb @@ -13,11 +13,6 @@ module Sidebars true end - override :link - def link - renderable_items.first.link - end - override :title def title context.group.subgroup? ? _('Subgroup information') : _('Group information') diff --git a/lib/sidebars/groups/menus/issues_menu.rb b/lib/sidebars/groups/menus/issues_menu.rb index 95641c09076..4044cb1c716 100644 --- a/lib/sidebars/groups/menus/issues_menu.rb +++ b/lib/sidebars/groups/menus/issues_menu.rb @@ -17,11 +17,6 @@ module Sidebars true end - override :link - def link - issues_group_path(context.group) - end - override :title def title _('Issues') @@ -43,7 +38,7 @@ module Sidebars count_service = ::Groups::OpenIssuesCountService count = count_service.new(context.group, context.current_user).count - format_cached_count(count_service, count) + format_cached_count(count_service::CACHED_COUNT_THRESHOLD, count) end end diff --git a/lib/sidebars/groups/menus/merge_requests_menu.rb b/lib/sidebars/groups/menus/merge_requests_menu.rb index 7faf50305c6..050cba07641 100644 --- a/lib/sidebars/groups/menus/merge_requests_menu.rb +++ b/lib/sidebars/groups/menus/merge_requests_menu.rb @@ -37,7 +37,7 @@ module Sidebars count_service = ::Groups::MergeRequestsCountService count = count_service.new(context.group, context.current_user).count - format_cached_count(count_service, count) + format_cached_count(count_service::CACHED_COUNT_THRESHOLD, count) end end diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb index e46e2820c04..e81e9355e7e 100644 --- a/lib/sidebars/groups/menus/packages_registries_menu.rb +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -13,11 +13,6 @@ module Sidebars true end - override :link - def link - renderable_items.first.link - end - override :title def title _('Packages & Registries') diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index 8bc6077d302..f0239ca6a1a 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -19,11 +19,6 @@ module Sidebars true end - override :link - def link - edit_group_path(context.group) - end - override :title def title _('Settings') diff --git a/lib/sidebars/groups/panel.rb b/lib/sidebars/groups/panel.rb index 73b943c5655..6efe89d496a 100644 --- a/lib/sidebars/groups/panel.rb +++ b/lib/sidebars/groups/panel.rb @@ -16,11 +16,6 @@ module Sidebars add_menu(Sidebars::Groups::Menus::SettingsMenu.new(context)) end - override :render_raw_menus_partial - def render_raw_menus_partial - 'layouts/nav/sidebar/group_menus' - end - override :aria_label def aria_label context.group.subgroup? ? _('Subgroup navigation') : _('Group navigation') diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb index 3b8872fd572..1af3d024291 100644 --- a/lib/sidebars/menu.rb +++ b/lib/sidebars/menu.rb @@ -33,10 +33,9 @@ module Sidebars has_renderable_items? || menu_with_partial? end - # Menus might have or not a link override :link def link - nil + renderable_items.first&.link end # This method normalizes the information retrieved from the submenus and this menu diff --git a/lib/sidebars/projects/menus/analytics_menu.rb b/lib/sidebars/projects/menus/analytics_menu.rb index 29fd0609596..b13b25d1cfe 100644 --- a/lib/sidebars/projects/menus/analytics_menu.rb +++ b/lib/sidebars/projects/menus/analytics_menu.rb @@ -21,7 +21,7 @@ module Sidebars def link return cycle_analytics_menu_item.link if cycle_analytics_menu_item.render? - renderable_items.first.link + super end override :extra_container_html_options diff --git a/lib/sidebars/projects/menus/ci_cd_menu.rb b/lib/sidebars/projects/menus/ci_cd_menu.rb index f85a9faacd3..67e4209c382 100644 --- a/lib/sidebars/projects/menus/ci_cd_menu.rb +++ b/lib/sidebars/projects/menus/ci_cd_menu.rb @@ -15,11 +15,6 @@ module Sidebars add_item(pipeline_schedules_menu_item) end - override :link - def link - project_pipelines_path(context.project) - end - override :extra_container_html_options def extra_container_html_options { diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb index fa6482562e8..110d78367b9 100644 --- a/lib/sidebars/projects/menus/deployments_menu.rb +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -13,11 +13,6 @@ module Sidebars true end - override :link - def link - renderable_items.first.link - end - override :extra_container_html_options def extra_container_html_options { diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index aad1ce60d0e..e26bb2237e6 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -15,11 +15,6 @@ module Sidebars true end - override :link - def link - renderable_items.first.link - end - override :extra_container_html_options def extra_container_html_options { diff --git a/lib/sidebars/projects/menus/issues_menu.rb b/lib/sidebars/projects/menus/issues_menu.rb index fd57f21db88..3774bec2f13 100644 --- a/lib/sidebars/projects/menus/issues_menu.rb +++ b/lib/sidebars/projects/menus/issues_menu.rb @@ -18,11 +18,6 @@ module Sidebars true end - override :link - def link - project_issues_path(context.project) - end - override :extra_container_html_options def extra_container_html_options { diff --git a/lib/sidebars/projects/menus/learn_gitlab_menu.rb b/lib/sidebars/projects/menus/learn_gitlab_menu.rb index f29f4a6eed6..16335f5b076 100644 --- a/lib/sidebars/projects/menus/learn_gitlab_menu.rb +++ b/lib/sidebars/projects/menus/learn_gitlab_menu.rb @@ -23,7 +23,7 @@ module Sidebars override :has_pill? def has_pill? - context.learn_gitlab_experiment_enabled + context.learn_gitlab_enabled end override :pill_count @@ -40,8 +40,7 @@ module Sidebars { class: 'home', data: { - track_label: 'learn_gitlab', - track_property: context.learn_gitlab_experiment_tracking_category + track_label: 'learn_gitlab' } } end @@ -53,7 +52,7 @@ module Sidebars override :render? def render? - context.learn_gitlab_experiment_enabled + context.learn_gitlab_enabled end end end diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index 0d7e0776d5b..59554726263 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -19,11 +19,6 @@ module Sidebars true end - override :link - def link - renderable_items.first&.link - end - override :extra_container_html_options def extra_container_html_options { diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index d49bb680853..f5f0da2992e 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -13,11 +13,6 @@ module Sidebars true end - override :link - def link - renderable_items.first.link - end - override :title def title _('Packages & Registries') diff --git a/lib/sidebars/projects/menus/project_information_menu.rb b/lib/sidebars/projects/menus/project_information_menu.rb index a5f06ebea20..44b94ee3522 100644 --- a/lib/sidebars/projects/menus/project_information_menu.rb +++ b/lib/sidebars/projects/menus/project_information_menu.rb @@ -13,11 +13,6 @@ module Sidebars true end - override :link - def link - renderable_items.first.link - end - override :extra_container_html_options def extra_container_html_options { class: 'shortcuts-project-information' } diff --git a/lib/sidebars/projects/menus/repository_menu.rb b/lib/sidebars/projects/menus/repository_menu.rb index a784aecc3dc..0a295f0f618 100644 --- a/lib/sidebars/projects/menus/repository_menu.rb +++ b/lib/sidebars/projects/menus/repository_menu.rb @@ -20,11 +20,6 @@ module Sidebars true end - override :link - def link - project_tree_path(context.project) - end - override :extra_container_html_options def extra_container_html_options { diff --git a/lib/sidebars/projects/menus/security_compliance_menu.rb b/lib/sidebars/projects/menus/security_compliance_menu.rb index 5616b466560..9367514cdca 100644 --- a/lib/sidebars/projects/menus/security_compliance_menu.rb +++ b/lib/sidebars/projects/menus/security_compliance_menu.rb @@ -15,11 +15,6 @@ module Sidebars true end - override :link - def link - renderable_items.first&.link - end - override :title def title _('Security & Compliance') diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 250143df649..6439c97d0bc 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -17,15 +17,11 @@ module Sidebars add_item(monitor_menu_item) add_item(pages_menu_item) add_item(packages_and_registries_menu_item) + add_item(usage_quotas_menu_item) true end - override :link - def link - edit_project_path(context.project) - end - override :title def title _('Settings') @@ -146,6 +142,19 @@ module Sidebars item_id: :packages_and_registries ) end + + def usage_quotas_menu_item + unless Feature.enabled?(:project_storage_ui, context.project&.group, default_enabled: :yaml) + return ::Sidebars::NilMenuItem.new(item_id: :usage_quotas) + end + + ::Sidebars::MenuItem.new( + title: s_('UsageQuota|Usage Quotas'), + link: project_usage_quotas_path(context.project), + active_routes: { path: 'usage_quotas#index' }, + item_id: :usage_quotas + ) + end end end end diff --git a/lib/support/logrotate/gitlab b/lib/support/logrotate/gitlab index c34db47e214..43fa8eb963c 100644 --- a/lib/support/logrotate/gitlab +++ b/lib/support/logrotate/gitlab @@ -1,5 +1,4 @@ # GitLab logrotate settings -# based on: http://stackoverflow.com/a/4883967 /home/git/gitlab/log/*.log { su git git diff --git a/lib/system_check/incoming_email_check.rb b/lib/system_check/incoming_email_check.rb index e0e1147711c..84033ada710 100644 --- a/lib/system_check/incoming_email_check.rb +++ b/lib/system_check/incoming_email_check.rb @@ -7,9 +7,11 @@ module SystemCheck def multi_check if Gitlab.config.incoming_email.enabled - checks = [ - SystemCheck::IncomingEmail::ImapAuthenticationCheck - ] + checks = [] + + if Gitlab.config.incoming_email.inbox_method == 'imap' + checks << SystemCheck::IncomingEmail::ImapAuthenticationCheck + end if Rails.env.production? checks << SystemCheck::IncomingEmail::InitdConfiguredCheck diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 51f15f5a56a..a6738b01f18 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -118,7 +118,7 @@ namespace :gitlab do desc 'Create missing dynamic database partitions' task create_dynamic_partitions: :environment do - Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions + Gitlab::Database::Partitioning.sync_partitions end # This is targeted towards deploys and upgrades of GitLab. @@ -151,6 +151,12 @@ namespace :gitlab do # initializers here as the application can continue to run while # a rake task reloads the database schema. Rake::Task['db:test:load'].enhance do + # Due to bug in `db:test:load` if many DBs are used + # the `ActiveRecord::Base.connection` might be switched to another one + # This is due to `if should_reconnect`: + # https://github.com/rails/rails/blob/a81aeb63a007ede2fe606c50539417dada9030c7/activerecord/lib/active_record/railties/databases.rake#L622 + ActiveRecord::Base.establish_connection :main + Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end diff --git a/lib/tasks/gitlab/docs/compile_deprecations.rake b/lib/tasks/gitlab/docs/compile_deprecations.rake new file mode 100644 index 00000000000..0fd43775015 --- /dev/null +++ b/lib/tasks/gitlab/docs/compile_deprecations.rake @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :docs do + desc "Generate deprecation list from individual files" + task :compile_deprecations do + require_relative '../../../../tooling/deprecations/docs' + + File.write(Deprecations::Docs.path, Deprecations::Docs.render) + + puts "Deprecations compiled to #{Deprecations::Docs.path}" + end + + desc "Check that the deprecation doc is up to date" + task :check_deprecations do + require_relative '../../../../tooling/deprecations/docs' + + contents = Deprecations::Docs.render + doc = File.read(Deprecations::Docs.path) + + if doc == contents + puts "Deprecations doc is up to date." + else + format_output('Deprecations doc is outdated! Please update it by running `bundle exec rake gitlab:docs:compile_deprecations`.') + abort + end + end + end +end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 6675439e430..ef58c9339f1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -15,8 +15,7 @@ namespace :gitlab do gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly')) # Our test setup expects a git repo, so clone rather than copy - version = Gitlab::GitalyClient.expected_server_version - checkout_or_clone_version(version: version, repo: gdk_gitaly_dir, target_dir: args.dir, clone_opts: %w[--depth 1]) + clone_repo(gdk_gitaly_dir, args.dir, clone_opts: %w[--depth 1]) unless Dir.exist?(args.dir) # We assume the GDK gitaly already compiled binaries build_dir = File.join(gdk_gitaly_dir, '_build') @@ -31,7 +30,7 @@ namespace :gitlab do FileUtils.cp_r(ruby_bundle_file, args.dir) gitaly_binary = File.join(build_dir, 'bin', 'gitaly') - warn_gitaly_out_of_date!(gitaly_binary, version) + warn_gitaly_out_of_date!(gitaly_binary, Gitlab::GitalyClient.expected_server_version) rescue Errno::ENOENT => e puts "Could not copy files, did you run `gdk update`? Error: #{e.message}" diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index 52c5c680292..b9137aa0d4c 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -111,7 +111,7 @@ namespace :gitlab do desc 'GitLab | GraphQL | Generate GraphQL docs' task compile_docs: [:environment, :enable_feature_flags] do - renderer = Tooling::Graphql::Docs::Renderer.new(GitlabSchema, render_options) + renderer = Tooling::Graphql::Docs::Renderer.new(GitlabSchema, **render_options) renderer.write @@ -120,7 +120,7 @@ namespace :gitlab do desc 'GitLab | GraphQL | Check if GraphQL docs are up to date' task check_docs: [:environment, :enable_feature_flags] do - renderer = Tooling::Graphql::Docs::Renderer.new(GitlabSchema, render_options) + renderer = Tooling::Graphql::Docs::Renderer.new(GitlabSchema, **render_options) doc = File.read(Rails.root.join(OUTPUT_DIR, 'index.md')) diff --git a/lib/tasks/gitlab/product_intelligence.rake b/lib/tasks/gitlab/product_intelligence.rake deleted file mode 100644 index 329cd9c8c2a..00000000000 --- a/lib/tasks/gitlab/product_intelligence.rake +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -namespace :gitlab do - namespace :product_intelligence do - # @example - # bundle exec rake gitlab:product_intelligence:activate_metrics MILESTONE=14.0 - - desc 'GitLab | Product Intelligence | Update milestone metrics status to data_available' - task activate_metrics: :environment do - milestone = ENV['MILESTONE'] - raise "Please supply the MILESTONE env var".color(:red) unless milestone.present? - - Gitlab::Usage::MetricDefinition.definitions.values.each do |metric| - next if metric.attributes[:milestone] != milestone || metric.attributes[:status] != 'implemented' - - metric.attributes[:status] = 'data_available' - path = metric.path - File.open(path, "w") { |file| file << metric.to_h.deep_stringify_keys.to_yaml } - end - - puts "Task completed successfully" - end - end -end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index d3060d92e88..90ed91221ae 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -86,9 +86,8 @@ namespace :gitlab do # 3: high priority # 5: _super_ high priority, this should only be used for _very_ important queues # - # As per http://stackoverflow.com/a/21241357/290102 the formula for calculating - # the likelihood of a job being popped off a queue (given all queues have work - # to perform) is: + # The formula for calculating the likelihood of a job being popped off a queue + # (given all queues have work to perform) is: # # chance = (queue weight / total weight of all queues) * 100 BANNER diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index ddd3424acda..35ddc627389 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -28,5 +28,33 @@ namespace :gitlab do task generate_from_yaml: :environment do puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data) end + + desc 'GitLab | UsageDataMetrics | Generate known_events/ci_templates.yml based on template definitions' + task generate_ci_template_events: :environment do + banner = <<~BANNER + # This file is generated automatically by + # bin/rake gitlab:usage_data:generate_ci_template_events + # + # Do not edit it manually! + BANNER + + repository_includes = ci_template_includes_hash(:repository_source) + auto_devops_jobs_includes = ci_template_includes_hash(:auto_devops_source, 'Jobs') + auto_devops_security_includes = ci_template_includes_hash(:auto_devops_source, 'Security') + all_includes = [*repository_includes, *auto_devops_jobs_includes, *auto_devops_security_includes] + + File.write(Gitlab::UsageDataCounters::CiTemplateUniqueCounter::KNOWN_EVENTS_FILE_PATH, banner + YAML.dump(all_includes).gsub(/ *$/m, '')) + end + + def ci_template_includes_hash(source, template_directory = nil) + Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_templates("lib/gitlab/ci/templates/#{template_directory}").map do |template| + { + 'name' => Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name("#{template_directory}/#{template}", source), + 'category' => 'ci_templates', + 'redis_slot' => Gitlab::UsageDataCounters::CiTemplateUniqueCounter::REDIS_SLOT, + 'aggregation' => 'weekly' + } + end + end end end diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake deleted file mode 100644 index fa3f8805159..00000000000 --- a/lib/tasks/karma.rake +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -unless Rails.env.production? - namespace :karma do - # alias exists for legacy reasons - desc 'GitLab | Karma | Generate fixtures for JavaScript tests' - task fixtures: ['frontend:fixtures'] - - desc 'GitLab | Karma | Run JavaScript tests' - task tests: ['yarn:check'] do - sh "yarn run karma" do |ok, res| - abort('rake karma:tests failed') unless ok - end - end - end - - desc 'GitLab | Karma | Shortcut for karma:fixtures and karma:tests' - task karma: ['karma:fixtures', 'karma:tests'] -end diff --git a/lib/tasks/pngquant.rake b/lib/tasks/pngquant.rake deleted file mode 100644 index 45c0288cadf..00000000000 --- a/lib/tasks/pngquant.rake +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -return if Rails.env.production? - -require 'png_quantizator' -require 'parallel' -require_relative '../../tooling/lib/tooling/images' - -# The amount of variance (in bytes) allowed in -# file size when testing for compression size - -namespace :pngquant do - # Returns an array of all images eligible for compression - def doc_images - Dir.glob('doc/**/*.png', File::FNM_CASEFOLD) - end - - desc 'GitLab | Pngquant | Compress all documentation PNG images using pngquant' - task :compress do - files = doc_images - puts "Compressing #{files.size} PNG files in doc/**" - - Parallel.each(files) do |file| - was_uncompressed, savings = Tooling::Image.compress_image(file) - - if was_uncompressed - puts "#{file} was reduced by #{savings} bytes" - end - end - end - - desc 'GitLab | Pngquant | Checks that all documentation PNG images have been compressed with pngquant' - task :lint do - files = doc_images - puts "Checking #{files.size} PNG files in doc/**" - - uncompressed_files = Parallel.map(files) do |file| - is_uncompressed, _ = Tooling::Image.compress_image(file, true) - if is_uncompressed - puts "Uncompressed file detected: ".color(:red) + file - file - end - end.compact - - if uncompressed_files.empty? - puts "All documentation images are optimally compressed!".color(:green) - else - warn( - "The #{uncompressed_files.size} image(s) above have not been optimally compressed using pngquant.".color(:red), - 'Please run "bin/rake pngquant:compress" and commit the result.' - ) - abort - end - end -end diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index f5d16835347..a4147ae1bba 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -4,4 +4,21 @@ unless Rails.env.production? require 'rubocop/rake_task' RuboCop::RakeTask.new + + namespace :rubocop do + namespace :todo do + desc 'Generate RuboCop todos' + task :generate do + require 'rubocop' + + options = %w[ + --auto-gen-config + --auto-gen-only-exclude + --exclude-limit=100000 + ] + + RuboCop::CLI.new.run(options) + end + end + end end |