diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-01 21:15:19 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-01 21:15:19 +0300 |
commit | 5fc2d78fb96b0fd50dfb737190fd411033b3c3ab (patch) | |
tree | 24cb469e61661c923a1398505b2bb928612f80d4 /app | |
parent | 66629d156e2420269ed53eff3dca0912cfe848e2 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
32 files changed, 218 insertions, 168 deletions
diff --git a/app/assets/javascripts/issuable_bulk_update_sidebar/issuable_bulk_update_sidebar.js b/app/assets/javascripts/issuable_bulk_update_sidebar/issuable_bulk_update_sidebar.js index a9d4548f8cf..50d1cb95896 100644 --- a/app/assets/javascripts/issuable_bulk_update_sidebar/issuable_bulk_update_sidebar.js +++ b/app/assets/javascripts/issuable_bulk_update_sidebar/issuable_bulk_update_sidebar.js @@ -5,7 +5,7 @@ import { property } from 'lodash'; import issueableEventHub from '~/issues_list/eventhub'; import LabelsSelect from '~/labels_select'; -import MilestoneSelect from '~/milestone_select'; +import MilestoneSelect from '~/milestones/milestone_select'; import initIssueStatusSelect from './init_issue_status_select'; import IssuableBulkUpdateActions from './issuable_bulk_update_actions'; import subscriptionSelect from './subscription_select'; diff --git a/app/assets/javascripts/pages/milestones/shared/components/delete_milestone_modal.vue b/app/assets/javascripts/milestones/components/delete_milestone_modal.vue index 34f9fe778ea..34f9fe778ea 100644 --- a/app/assets/javascripts/pages/milestones/shared/components/delete_milestone_modal.vue +++ b/app/assets/javascripts/milestones/components/delete_milestone_modal.vue diff --git a/app/assets/javascripts/pages/milestones/shared/components/promote_milestone_modal.vue b/app/assets/javascripts/milestones/components/promote_milestone_modal.vue index b41611001ab..b41611001ab 100644 --- a/app/assets/javascripts/pages/milestones/shared/components/promote_milestone_modal.vue +++ b/app/assets/javascripts/milestones/components/promote_milestone_modal.vue diff --git a/app/assets/javascripts/pages/milestones/shared/delete_milestone_modal_init.js b/app/assets/javascripts/milestones/delete_milestone_modal_init.js index 3aeff2db2e0..3aeff2db2e0 100644 --- a/app/assets/javascripts/pages/milestones/shared/delete_milestone_modal_init.js +++ b/app/assets/javascripts/milestones/delete_milestone_modal_init.js diff --git a/app/assets/javascripts/pages/milestones/shared/event_hub.js b/app/assets/javascripts/milestones/event_hub.js index e31806ad199..e31806ad199 100644 --- a/app/assets/javascripts/pages/milestones/shared/event_hub.js +++ b/app/assets/javascripts/milestones/event_hub.js diff --git a/app/assets/javascripts/shared/milestones/form.js b/app/assets/javascripts/milestones/form.js index 3ca9288b156..40d45d7deb8 100644 --- a/app/assets/javascripts/shared/milestones/form.js +++ b/app/assets/javascripts/milestones/form.js @@ -1,7 +1,7 @@ import $ from 'jquery'; import initDatePicker from '~/behaviors/date_picker'; -import GLForm from '../../gl_form'; -import ZenMode from '../../zen_mode'; +import GLForm from '~/gl_form'; +import ZenMode from '~/zen_mode'; export default (initGFM = true) => { new ZenMode(); // eslint-disable-line no-new diff --git a/app/assets/javascripts/pages/milestones/shared/init_milestones_show.js b/app/assets/javascripts/milestones/init_milestones_show.js index b2a896a3265..8939e1535c1 100644 --- a/app/assets/javascripts/pages/milestones/shared/init_milestones_show.js +++ b/app/assets/javascripts/milestones/init_milestones_show.js @@ -1,6 +1,6 @@ /* eslint-disable no-new */ -import Milestone from '~/milestone'; +import Milestone from '~/milestones/milestone'; import Sidebar from '~/right_sidebar'; import MountMilestoneSidebar from '~/sidebar/mount_milestone_sidebar'; diff --git a/app/assets/javascripts/milestone.js b/app/assets/javascripts/milestones/milestone.js index b4e53c1fab6..2c43bed412e 100644 --- a/app/assets/javascripts/milestone.js +++ b/app/assets/javascripts/milestones/milestone.js @@ -1,7 +1,7 @@ import $ from 'jquery'; -import createFlash from './flash'; -import axios from './lib/utils/axios_utils'; -import { __ } from './locale'; +import createFlash from '~/flash'; +import axios from '~/lib/utils/axios_utils'; +import { __ } from '~/locale'; export default class Milestone { constructor() { diff --git a/app/assets/javascripts/milestone_select.js b/app/assets/javascripts/milestones/milestone_select.js index aa8a40b6a87..91780d5ee01 100644 --- a/app/assets/javascripts/milestone_select.js +++ b/app/assets/javascripts/milestones/milestone_select.js @@ -7,8 +7,8 @@ import Api from '~/api'; import initDeprecatedJQueryDropdown from '~/deprecated_jquery_dropdown'; import { __, sprintf } from '~/locale'; import { sortMilestonesByDueDate } from '~/milestones/milestone_utils'; -import axios from './lib/utils/axios_utils'; -import { timeFor, parsePikadayDate, dateInWords } from './lib/utils/datetime_utility'; +import axios from '~/lib/utils/axios_utils'; +import { timeFor, parsePikadayDate, dateInWords } from '~/lib/utils/datetime_utility'; export default class MilestoneSelect { constructor(currentProject, els, options = {}) { diff --git a/app/assets/javascripts/pages/milestones/shared/promote_milestone_modal_init.js b/app/assets/javascripts/milestones/promote_milestone_modal_init.js index 5472b8c684f..5472b8c684f 100644 --- a/app/assets/javascripts/pages/milestones/shared/promote_milestone_modal_init.js +++ b/app/assets/javascripts/milestones/promote_milestone_modal_init.js diff --git a/app/assets/javascripts/pages/dashboard/milestones/show/index.js b/app/assets/javascripts/pages/dashboard/milestones/show/index.js index 1f3e458fe17..d1ff7ec336c 100644 --- a/app/assets/javascripts/pages/dashboard/milestones/show/index.js +++ b/app/assets/javascripts/pages/dashboard/milestones/show/index.js @@ -1,4 +1,4 @@ -import Milestone from '~/milestone'; +import Milestone from '~/milestones/milestone'; import Sidebar from '~/right_sidebar'; import MountMilestoneSidebar from '~/sidebar/mount_milestone_sidebar'; diff --git a/app/assets/javascripts/pages/groups/milestones/edit/index.js b/app/assets/javascripts/pages/groups/milestones/edit/index.js index 4f8514a9a1d..6bf73c2563c 100644 --- a/app/assets/javascripts/pages/groups/milestones/edit/index.js +++ b/app/assets/javascripts/pages/groups/milestones/edit/index.js @@ -1,3 +1,3 @@ -import initForm from '~/shared/milestones/form'; +import initForm from '~/milestones/form'; initForm(); diff --git a/app/assets/javascripts/pages/groups/milestones/new/index.js b/app/assets/javascripts/pages/groups/milestones/new/index.js index 4f8514a9a1d..6bf73c2563c 100644 --- a/app/assets/javascripts/pages/groups/milestones/new/index.js +++ b/app/assets/javascripts/pages/groups/milestones/new/index.js @@ -1,3 +1,3 @@ -import initForm from '~/shared/milestones/form'; +import initForm from '~/milestones/form'; initForm(); diff --git a/app/assets/javascripts/pages/groups/milestones/show/index.js b/app/assets/javascripts/pages/groups/milestones/show/index.js index 914e2831185..5eec1015447 100644 --- a/app/assets/javascripts/pages/groups/milestones/show/index.js +++ b/app/assets/javascripts/pages/groups/milestones/show/index.js @@ -1,5 +1,5 @@ -import initDeleteMilestoneModal from '~/pages/milestones/shared/delete_milestone_modal_init'; -import initMilestonesShow from '~/pages/milestones/shared/init_milestones_show'; +import initDeleteMilestoneModal from '~/milestones/delete_milestone_modal_init'; +import initMilestonesShow from '~/milestones/init_milestones_show'; initMilestonesShow(); initDeleteMilestoneModal(); diff --git a/app/assets/javascripts/pages/milestones/shared/index.js b/app/assets/javascripts/pages/milestones/shared/index.js deleted file mode 100644 index dabfe32848b..00000000000 --- a/app/assets/javascripts/pages/milestones/shared/index.js +++ /dev/null @@ -1,7 +0,0 @@ -import initDeleteMilestoneModal from './delete_milestone_modal_init'; -import initPromoteMilestoneModal from './promote_milestone_modal_init'; - -export default () => { - initDeleteMilestoneModal(); - initPromoteMilestoneModal(); -}; diff --git a/app/assets/javascripts/pages/projects/issues/form.js b/app/assets/javascripts/pages/projects/issues/form.js index c0da0069a99..07d6943e287 100644 --- a/app/assets/javascripts/pages/projects/issues/form.js +++ b/app/assets/javascripts/pages/projects/issues/form.js @@ -7,7 +7,7 @@ import GLForm from '~/gl_form'; import initSuggestions from '~/issuable_suggestions'; import initIssuableTypeSelector from '~/issuable_type_selector'; import LabelsSelect from '~/labels_select'; -import MilestoneSelect from '~/milestone_select'; +import MilestoneSelect from '~/milestones/milestone_select'; import IssuableTemplateSelectors from '~/templates/issuable_template_selectors'; export default () => { diff --git a/app/assets/javascripts/pages/projects/merge_requests/init_merge_request.js b/app/assets/javascripts/pages/projects/merge_requests/init_merge_request.js index 7d5719cf8a8..f2777434029 100644 --- a/app/assets/javascripts/pages/projects/merge_requests/init_merge_request.js +++ b/app/assets/javascripts/pages/projects/merge_requests/init_merge_request.js @@ -6,7 +6,7 @@ import ShortcutsNavigation from '~/behaviors/shortcuts/shortcuts_navigation'; import Diff from '~/diff'; import GLForm from '~/gl_form'; import LabelsSelect from '~/labels_select'; -import MilestoneSelect from '~/milestone_select'; +import MilestoneSelect from '~/milestones/milestone_select'; import IssuableTemplateSelectors from '~/templates/issuable_template_selectors'; export default () => { diff --git a/app/assets/javascripts/pages/projects/milestones/edit/index.js b/app/assets/javascripts/pages/projects/milestones/edit/index.js index 4f8514a9a1d..6bf73c2563c 100644 --- a/app/assets/javascripts/pages/projects/milestones/edit/index.js +++ b/app/assets/javascripts/pages/projects/milestones/edit/index.js @@ -1,3 +1,3 @@ -import initForm from '~/shared/milestones/form'; +import initForm from '~/milestones/form'; initForm(); diff --git a/app/assets/javascripts/pages/projects/milestones/index/index.js b/app/assets/javascripts/pages/projects/milestones/index/index.js index 150b506b121..6912ab9f8ba 100644 --- a/app/assets/javascripts/pages/projects/milestones/index/index.js +++ b/app/assets/javascripts/pages/projects/milestones/index/index.js @@ -1,3 +1,5 @@ -import milestones from '~/pages/milestones/shared'; +import initDeleteMilestoneModal from '~/milestones/delete_milestone_modal_init'; +import initPromoteMilestoneModal from '~/milestones/promote_milestone_modal_init'; -milestones(); +initDeleteMilestoneModal(); +initPromoteMilestoneModal(); diff --git a/app/assets/javascripts/pages/projects/milestones/new/index.js b/app/assets/javascripts/pages/projects/milestones/new/index.js index 4f8514a9a1d..6bf73c2563c 100644 --- a/app/assets/javascripts/pages/projects/milestones/new/index.js +++ b/app/assets/javascripts/pages/projects/milestones/new/index.js @@ -1,3 +1,3 @@ -import initForm from '~/shared/milestones/form'; +import initForm from '~/milestones/form'; initForm(); diff --git a/app/assets/javascripts/pages/projects/milestones/show/index.js b/app/assets/javascripts/pages/projects/milestones/show/index.js index 3c755e9b98c..098b3a5f391 100644 --- a/app/assets/javascripts/pages/projects/milestones/show/index.js +++ b/app/assets/javascripts/pages/projects/milestones/show/index.js @@ -1,5 +1,7 @@ -import milestones from '~/pages/milestones/shared'; -import initMilestonesShow from '~/pages/milestones/shared/init_milestones_show'; +import initMilestonesShow from '~/milestones/init_milestones_show'; +import initDeleteMilestoneModal from '~/milestones/delete_milestone_modal_init'; +import initPromoteMilestoneModal from '~/milestones/promote_milestone_modal_init'; initMilestonesShow(); -milestones(); +initDeleteMilestoneModal(); +initPromoteMilestoneModal(); diff --git a/app/controllers/projects/google_cloud/base_controller.rb b/app/controllers/projects/google_cloud/base_controller.rb index 2a9e89a445b..8bfe5c9c5f3 100644 --- a/app/controllers/projects/google_cloud/base_controller.rb +++ b/app/controllers/projects/google_cloud/base_controller.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class Projects::GoogleCloud::BaseController < Projects::ApplicationController - feature_category :google_cloud + feature_category :five_minute_production_app before_action :admin_project_google_cloud! before_action :google_oauth2_enabled! diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index aca556b18cb..3bec7928058 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -73,11 +73,8 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo :show, :toggle_award_emoji, :toggle_subscription, :update ] - feature_category :code_testing, [ - :test_reports, :coverage_reports, :codequality_reports, - :codequality_mr_diff_reports - ] - + feature_category :code_testing, [:test_reports, :coverage_reports] + feature_category :code_quality, [:codequality_reports, :codequality_mr_diff_reports] feature_category :accessibility_testing, [:accessibility_reports] feature_category :infrastructure_as_code, [:terraform_reports] feature_category :continuous_integration, [:pipeline_status, :pipelines, :exposed_artifacts] diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb new file mode 100644 index 00000000000..a497d2cabe5 --- /dev/null +++ b/app/models/ci/namespace_mirror.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Ci + # This model represents a record in a shadow table of the main database's namespaces table. + # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN. + class NamespaceMirror < ApplicationRecord + # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517 + end +end diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb new file mode 100644 index 00000000000..c6e3101fb3a --- /dev/null +++ b/app/models/ci/project_mirror.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Ci + # This model represents a shadow table of the main database's projects table. + # It allows us to navigate the project and namespace hierarchy on the ci database. + class ProjectMirror < ApplicationRecord + # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517 + end +end diff --git a/app/models/postgresql/replication_slot.rb b/app/models/postgresql/replication_slot.rb index 1a4d3bd5794..1c38edcca61 100644 --- a/app/models/postgresql/replication_slot.rb +++ b/app/models/postgresql/replication_slot.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Postgresql - class ReplicationSlot < ApplicationRecord + class ReplicationSlot < Gitlab::Database::SharedModel self.table_name = 'pg_replication_slots' # Returns true if there are any replication slots in use. diff --git a/app/views/projects/issues/_related_branches.html.haml b/app/views/projects/issues/_related_branches.html.haml index c47257eec4a..310a0c1a61e 100644 --- a/app/views/projects/issues/_related_branches.html.haml +++ b/app/views/projects/issues/_related_branches.html.haml @@ -1,7 +1,7 @@ - if @related_branches.any? %h2.gl-font-lg = pluralize(@related_branches.size, 'Related Branch') - %ul.unstyled-list.related-merge-requests + %ul.related-merge-requests.gl-pl-0 - @related_branches.each do |branch| %li.gl-display-flex.gl-align-items-center - if branch[:pipeline_status].present? diff --git a/app/views/projects/merge_requests/invalid.html.haml b/app/views/projects/merge_requests/invalid.html.haml index fd1b2328a98..eb8de425f61 100644 --- a/app/views/projects/merge_requests/invalid.html.haml +++ b/app/views/projects/merge_requests/invalid.html.haml @@ -1,7 +1,12 @@ - page_title "#{@merge_request.title} (#{@merge_request.to_reference}", _("Merge requests") -- badge_css_classes = "badge gl-text-white" -- badge_info_css_classes = "#{badge_css_classes} badge-info" -- badge_inverse_css_classes = "#{badge_css_classes} badge-inverse" + +- badge_start = '<span class="badge badge-pill gl-badge sm badge-info">'.html_safe +- badge_end = '</span>'.html_safe + +- err_fork_project_removed = s_("MergeRequest|Can't show this merge request because the fork project was deleted.") +- err_source_branch = s_("MergeRequest|Can't show this merge request because the source branch %{badge_start}%{source_branch}%{badge_end} is missing from project %{badge_start}%{project_path}%{badge_end}. Close this merge request or update the source branch.") +- err_target_branch = s_("MergeRequest|Can't show this merge request because the target branch %{badge_start}%{target_branch}%{badge_end} is missing from project %{badge_start}%{project_path}%{badge_end}. Close this merge request or update the target branch.") +- err_internal = s_("MergeRequest|Can't show this merge request because of an internal error. Contact your administrator.") .merge-request = render "projects/merge_requests/mr_title" @@ -11,20 +16,12 @@ .gl-alert-container = sprite_icon('error', size: 16, css_class: 'gl-icon gl-alert-icon gl-alert-icon-no-title') .gl-alert-content{ role: 'alert' } - %p - We cannot render this merge request properly because + .gl-alert-body - if @merge_request.for_fork? && !@merge_request.source_project - fork project was removed + = err_fork_project_removed - elsif !@merge_request.source_branch_exists? - %span{ class: badge_inverse_css_classes }= @merge_request.source_branch - does not exist in - %span{ class: badge_info_css_classes }= @merge_request.source_project_path + = err_source_branch.html_safe % { badge_start: badge_start, badge_end: badge_end, source_branch: @merge_request.source_branch, project_path: @merge_request.source_project_path } - elsif !@merge_request.target_branch_exists? - %span{ class: badge_inverse_css_classes }= @merge_request.target_branch - does not exist in - %span{ class: badge_info_css_classes }= @merge_request.target_project_path + = err_target_branch.html_safe % { badge_start: badge_start, badge_end: badge_end, target_branch: @merge_request.target_branch, project_path: @merge_request.source_project_path } - else - of internal error - - %strong - Please close merge request or change branches with existing one + = err_internal diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index fbec46dbc2f..89a4d9dc7cf 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1476,7 +1476,7 @@ :tags: [] - :name: pipeline_background:ci_pipeline_artifacts_create_quality_report :worker_name: Ci::PipelineArtifacts::CreateQualityReportWorker - :feature_category: :code_testing + :feature_category: :code_quality :has_external_dependencies: :urgency: :low :resource_boundary: :unknown diff --git a/app/workers/background_migration/single_database_worker.rb b/app/workers/background_migration/single_database_worker.rb new file mode 100644 index 00000000000..b6661d4fd14 --- /dev/null +++ b/app/workers/background_migration/single_database_worker.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +module BackgroundMigration + module SingleDatabaseWorker + extend ActiveSupport::Concern + + include ApplicationWorker + + MAX_LEASE_ATTEMPTS = 5 + + included do + data_consistency :always + + sidekiq_options retry: 3 + + feature_category :database + urgency :throttled + loggable_arguments 0, 1 + end + + class_methods do + # The minimum amount of time between processing two jobs of the same migration + # class. + # + # This interval is set to 2 or 5 minutes so autovacuuming and other + # maintenance related tasks have plenty of time to clean up after a migration + # has been performed. + def minimum_interval + 2.minutes.to_i + end + + def tracking_database + raise NotImplementedError, "#{self.name} does not implement #{__method__}" + end + + def unhealthy_metric_name + raise NotImplementedError, "#{self.name} does not implement #{__method__}" + end + end + + # Performs the background migration. + # + # See Gitlab::BackgroundMigration.perform for more information. + # + # class_name - The class name of the background migration to run. + # arguments - The arguments to pass to the migration class. + # lease_attempts - The number of times we will try to obtain an exclusive + # lease on the class before giving up. See MR for more discussion. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45298#note_434304956 + def perform(class_name, arguments = [], lease_attempts = MAX_LEASE_ATTEMPTS) + job_coordinator.with_shared_connection do + perform_with_connection(class_name, arguments, lease_attempts) + end + end + + private + + def job_coordinator + @job_coordinator ||= Gitlab::BackgroundMigration.coordinator_for_database(self.class.tracking_database) + end + + def perform_with_connection(class_name, arguments, lease_attempts) + with_context(caller_id: class_name.to_s) do + retried = lease_attempts != MAX_LEASE_ATTEMPTS + attempts_left = lease_attempts - 1 + should_perform, ttl = perform_and_ttl(class_name, attempts_left, retried) + + break if should_perform.nil? + + if should_perform + job_coordinator.perform(class_name, arguments) + else + # If the lease could not be obtained this means either another process is + # running a migration of this class or we ran one recently. In this case + # we'll reschedule the job in such a way that it is picked up again around + # the time the lease expires. + self.class + .perform_in(ttl || self.class.minimum_interval, class_name, arguments, attempts_left) + end + end + end + + def perform_and_ttl(class_name, attempts_left, retried) + # In test environments `perform_in` will run right away. This can then + # lead to stack level errors in the above `#perform`. To work around this + # we'll just perform the migration right away in the test environment. + return [true, nil] if always_perform? + + lease = lease_for(class_name, retried) + lease_obtained = !!lease.try_obtain + healthy_db = healthy_database? + perform = lease_obtained && healthy_db + + database_unhealthy_counter.increment if lease_obtained && !healthy_db + + # When the DB is unhealthy or the lease can't be obtained after several tries, + # then give up on the job and log a warning. Otherwise we could end up in + # an infinite rescheduling loop. Jobs can be tracked in the database with the + # use of Gitlab::Database::BackgroundMigrationJob + if !perform && attempts_left < 0 + msg = if !lease_obtained + 'Job could not get an exclusive lease after several tries. Giving up.' + else + 'Database was unhealthy after several tries. Giving up.' + end + + Sidekiq.logger.warn(class: class_name, message: msg, job_id: jid) + + return [nil, nil] + end + + [perform, lease.ttl] + end + + def lease_for(class_name, retried) + Gitlab::ExclusiveLease + .new(lease_key_for(class_name, retried), timeout: self.class.minimum_interval) + end + + def lease_key_for(class_name, retried) + key = "#{self.class.name}:#{class_name}" + # We use a different exclusive lock key for retried jobs to allow them running concurrently with the scheduled jobs. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68763 for more information. + key += ":retried" if retried + key + end + + def always_perform? + Rails.env.test? + end + + # Returns true if the database is healthy enough to allow the migration to be + # performed. + # + # class_name - The name of the background migration that we might want to + # run. + def healthy_database? + !Postgresql::ReplicationSlot.lag_too_great? + end + + def database_unhealthy_counter + Gitlab::Metrics.counter( + self.class.unhealthy_metric_name, + 'The number of times a background migration is rescheduled because the database is unhealthy.' + ) + end + end +end diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index b771ab4d4e7..6489aad3173 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -1,120 +1,13 @@ # frozen_string_literal: true class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker + include BackgroundMigration::SingleDatabaseWorker - MAX_LEASE_ATTEMPTS = 5 - - data_consistency :always - - sidekiq_options retry: 3 - - feature_category :database - urgency :throttled - loggable_arguments 0, 1 - - # The minimum amount of time between processing two jobs of the same migration - # class. - # - # This interval is set to 2 or 5 minutes so autovacuuming and other - # maintenance related tasks have plenty of time to clean up after a migration - # has been performed. - def self.minimum_interval - 2.minutes.to_i - end - - # Performs the background migration. - # - # See Gitlab::BackgroundMigration.perform for more information. - # - # class_name - The class name of the background migration to run. - # arguments - The arguments to pass to the migration class. - # lease_attempts - The number of times we will try to obtain an exclusive - # lease on the class before giving up. See MR for more discussion. - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45298#note_434304956 - def perform(class_name, arguments = [], lease_attempts = MAX_LEASE_ATTEMPTS) - with_context(caller_id: class_name.to_s) do - retried = lease_attempts != MAX_LEASE_ATTEMPTS - attempts_left = lease_attempts - 1 - should_perform, ttl = perform_and_ttl(class_name, attempts_left, retried) - - break if should_perform.nil? - - if should_perform - Gitlab::BackgroundMigration.perform(class_name, arguments) - else - # If the lease could not be obtained this means either another process is - # running a migration of this class or we ran one recently. In this case - # we'll reschedule the job in such a way that it is picked up again around - # the time the lease expires. - self.class - .perform_in(ttl || self.class.minimum_interval, class_name, arguments, attempts_left) - end - end - end - - def perform_and_ttl(class_name, attempts_left, retried) - # In test environments `perform_in` will run right away. This can then - # lead to stack level errors in the above `#perform`. To work around this - # we'll just perform the migration right away in the test environment. - return [true, nil] if always_perform? - - lease = lease_for(class_name, retried) - lease_obtained = !!lease.try_obtain - healthy_db = healthy_database? - perform = lease_obtained && healthy_db - - database_unhealthy_counter.increment if lease_obtained && !healthy_db - - # When the DB is unhealthy or the lease can't be obtained after several tries, - # then give up on the job and log a warning. Otherwise we could end up in - # an infinite rescheduling loop. Jobs can be tracked in the database with the - # use of Gitlab::Database::BackgroundMigrationJob - if !perform && attempts_left < 0 - msg = if !lease_obtained - 'Job could not get an exclusive lease after several tries. Giving up.' - else - 'Database was unhealthy after several tries. Giving up.' - end - - Sidekiq.logger.warn(class: class_name, message: msg, job_id: jid) - - return [nil, nil] - end - - [perform, lease.ttl] - end - - def lease_for(class_name, retried) - Gitlab::ExclusiveLease - .new(lease_key_for(class_name, retried), timeout: self.class.minimum_interval) - end - - def lease_key_for(class_name, retried) - key = "#{self.class.name}:#{class_name}" - # We use a different exclusive lock key for retried jobs to allow them running concurrently with the scheduled jobs. - # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68763 for more information. - key += ":retried" if retried - key - end - - def always_perform? - Rails.env.test? - end - - # Returns true if the database is healthy enough to allow the migration to be - # performed. - # - # class_name - The name of the background migration that we might want to - # run. - def healthy_database? - !Postgresql::ReplicationSlot.lag_too_great? + def self.tracking_database + @tracking_database ||= Gitlab::Database::MAIN_DATABASE_NAME.to_sym end - def database_unhealthy_counter - Gitlab::Metrics.counter( - :background_migration_database_health_reschedules, - 'The number of times a background migration is rescheduled because the database is unhealthy.' - ) + def self.unhealthy_metric_name + @unhealthy_metric_name ||= :background_migration_database_health_reschedules end end diff --git a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb index bb0a81a0a17..dc7e8f888c6 100644 --- a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb +++ b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb @@ -10,7 +10,7 @@ module Ci sidekiq_options retry: 3 queue_namespace :pipeline_background - feature_category :code_testing + feature_category :code_quality idempotent! |