diff options
88 files changed, 843 insertions, 597 deletions
diff --git a/.rubocop_todo/database/multiple_databases.yml b/.rubocop_todo/database/multiple_databases.yml index a2e9a7e1e7e..e555c2f912b 100644 --- a/.rubocop_todo/database/multiple_databases.yml +++ b/.rubocop_todo/database/multiple_databases.yml @@ -10,7 +10,6 @@ Database/MultipleDatabases: - ee/lib/pseudonymizer/pager.rb - ee/lib/system_check/geo/geo_database_configured_check.rb - ee/spec/lib/pseudonymizer/dumper_spec.rb - - ee/spec/models/pg_replication_slot_spec.rb - ee/spec/services/ee/merge_requests/update_service_spec.rb - lib/backup/database.rb - lib/backup/manager.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index efd54c94ed4..82dc7691ff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -540,6 +540,17 @@ entry. - [Add pipeline artifacts and uploads sizes to project REST API](gitlab-org/gitlab@58d66f28faf42ae98ca11ff1ba0bdd9180e988ad) by @guillaume.chauvel ([merge request](gitlab-org/gitlab!72075)) - [Remove not used parameter from epics finder](gitlab-org/gitlab@49fce172b57b2f376a114726b1dd1900fe36a238) ([merge request](gitlab-org/gitlab!72285)) **GitLab Enterprise Edition** +## 14.4.3 (2021-12-01) + +### Fixed (6 changes) + +- [Check validation only if new record of license](gitlab-org/gitlab@5e0834a921dad1b1e07119de629ea44eb0ad5733) ([merge request](gitlab-org/gitlab!75421)) **GitLab Enterprise Edition** +- [Fix for hexadecimal branch deletion](gitlab-org/gitlab@fc3c2f211d5a2f190032c4d0109e2bcb31050b4d) ([merge request](gitlab-org/gitlab!75421)) +- [Geo - Fix no repo error message for group-level wikis](gitlab-org/gitlab@bdf3a712a4bfe245dfa7e7a90c24f2fdb482e309) ([merge request](gitlab-org/gitlab!75421)) **GitLab Enterprise Edition** +- [Prevent Git operations from checking replication lag on non-Geo-secondary sites](gitlab-org/gitlab@c158c01027f61aadd1c72f0817731d368d0d58cc) ([merge request](gitlab-org/gitlab!75421)) **GitLab Enterprise Edition** +- [Allow SSO callbacks through maintenance mode](gitlab-org/gitlab@1acae9807b1808ac360a4be098a50c547c9540b9) by @dzaporozhets ([merge request](gitlab-org/gitlab!75421)) **GitLab Enterprise Edition** +- [Fix 2FA setup for LDAP users](gitlab-org/gitlab@9b9a7230aed3ffeef3e8f608dd1a569397c71684) ([merge request](gitlab-org/gitlab!75421)) + ## 14.4.2 (2021-11-08) ### Fixed (3 changes) diff --git a/app/assets/javascripts/issuable_bulk_update_sidebar/issuable_bulk_update_sidebar.js b/app/assets/javascripts/issuable_bulk_update_sidebar/issuable_bulk_update_sidebar.js index a9d4548f8cf..50d1cb95896 100644 --- a/app/assets/javascripts/issuable_bulk_update_sidebar/issuable_bulk_update_sidebar.js +++ b/app/assets/javascripts/issuable_bulk_update_sidebar/issuable_bulk_update_sidebar.js @@ -5,7 +5,7 @@ import { property } from 'lodash'; import issueableEventHub from '~/issues_list/eventhub'; import LabelsSelect from '~/labels_select'; -import MilestoneSelect from '~/milestone_select'; +import MilestoneSelect from '~/milestones/milestone_select'; import initIssueStatusSelect from './init_issue_status_select'; import IssuableBulkUpdateActions from './issuable_bulk_update_actions'; import subscriptionSelect from './subscription_select'; diff --git a/app/assets/javascripts/pages/milestones/shared/components/delete_milestone_modal.vue b/app/assets/javascripts/milestones/components/delete_milestone_modal.vue index 34f9fe778ea..34f9fe778ea 100644 --- a/app/assets/javascripts/pages/milestones/shared/components/delete_milestone_modal.vue +++ b/app/assets/javascripts/milestones/components/delete_milestone_modal.vue diff --git a/app/assets/javascripts/pages/milestones/shared/components/promote_milestone_modal.vue b/app/assets/javascripts/milestones/components/promote_milestone_modal.vue index b41611001ab..b41611001ab 100644 --- a/app/assets/javascripts/pages/milestones/shared/components/promote_milestone_modal.vue +++ b/app/assets/javascripts/milestones/components/promote_milestone_modal.vue diff --git a/app/assets/javascripts/pages/milestones/shared/delete_milestone_modal_init.js b/app/assets/javascripts/milestones/delete_milestone_modal_init.js index 3aeff2db2e0..3aeff2db2e0 100644 --- a/app/assets/javascripts/pages/milestones/shared/delete_milestone_modal_init.js +++ b/app/assets/javascripts/milestones/delete_milestone_modal_init.js diff --git a/app/assets/javascripts/pages/milestones/shared/event_hub.js b/app/assets/javascripts/milestones/event_hub.js index e31806ad199..e31806ad199 100644 --- a/app/assets/javascripts/pages/milestones/shared/event_hub.js +++ b/app/assets/javascripts/milestones/event_hub.js diff --git a/app/assets/javascripts/shared/milestones/form.js b/app/assets/javascripts/milestones/form.js index 3ca9288b156..40d45d7deb8 100644 --- a/app/assets/javascripts/shared/milestones/form.js +++ b/app/assets/javascripts/milestones/form.js @@ -1,7 +1,7 @@ import $ from 'jquery'; import initDatePicker from '~/behaviors/date_picker'; -import GLForm from '../../gl_form'; -import ZenMode from '../../zen_mode'; +import GLForm from '~/gl_form'; +import ZenMode from '~/zen_mode'; export default (initGFM = true) => { new ZenMode(); // eslint-disable-line no-new diff --git a/app/assets/javascripts/pages/milestones/shared/init_milestones_show.js b/app/assets/javascripts/milestones/init_milestones_show.js index b2a896a3265..8939e1535c1 100644 --- a/app/assets/javascripts/pages/milestones/shared/init_milestones_show.js +++ b/app/assets/javascripts/milestones/init_milestones_show.js @@ -1,6 +1,6 @@ /* eslint-disable no-new */ -import Milestone from '~/milestone'; +import Milestone from '~/milestones/milestone'; import Sidebar from '~/right_sidebar'; import MountMilestoneSidebar from '~/sidebar/mount_milestone_sidebar'; diff --git a/app/assets/javascripts/milestone.js b/app/assets/javascripts/milestones/milestone.js index b4e53c1fab6..2c43bed412e 100644 --- a/app/assets/javascripts/milestone.js +++ b/app/assets/javascripts/milestones/milestone.js @@ -1,7 +1,7 @@ import $ from 'jquery'; -import createFlash from './flash'; -import axios from './lib/utils/axios_utils'; -import { __ } from './locale'; +import createFlash from '~/flash'; +import axios from '~/lib/utils/axios_utils'; +import { __ } from '~/locale'; export default class Milestone { constructor() { diff --git a/app/assets/javascripts/milestone_select.js b/app/assets/javascripts/milestones/milestone_select.js index aa8a40b6a87..91780d5ee01 100644 --- a/app/assets/javascripts/milestone_select.js +++ b/app/assets/javascripts/milestones/milestone_select.js @@ -7,8 +7,8 @@ import Api from '~/api'; import initDeprecatedJQueryDropdown from '~/deprecated_jquery_dropdown'; import { __, sprintf } from '~/locale'; import { sortMilestonesByDueDate } from '~/milestones/milestone_utils'; -import axios from './lib/utils/axios_utils'; -import { timeFor, parsePikadayDate, dateInWords } from './lib/utils/datetime_utility'; +import axios from '~/lib/utils/axios_utils'; +import { timeFor, parsePikadayDate, dateInWords } from '~/lib/utils/datetime_utility'; export default class MilestoneSelect { constructor(currentProject, els, options = {}) { diff --git a/app/assets/javascripts/pages/milestones/shared/promote_milestone_modal_init.js b/app/assets/javascripts/milestones/promote_milestone_modal_init.js index 5472b8c684f..5472b8c684f 100644 --- a/app/assets/javascripts/pages/milestones/shared/promote_milestone_modal_init.js +++ b/app/assets/javascripts/milestones/promote_milestone_modal_init.js diff --git a/app/assets/javascripts/pages/dashboard/milestones/show/index.js b/app/assets/javascripts/pages/dashboard/milestones/show/index.js index 1f3e458fe17..d1ff7ec336c 100644 --- a/app/assets/javascripts/pages/dashboard/milestones/show/index.js +++ b/app/assets/javascripts/pages/dashboard/milestones/show/index.js @@ -1,4 +1,4 @@ -import Milestone from '~/milestone'; +import Milestone from '~/milestones/milestone'; import Sidebar from '~/right_sidebar'; import MountMilestoneSidebar from '~/sidebar/mount_milestone_sidebar'; diff --git a/app/assets/javascripts/pages/groups/milestones/edit/index.js b/app/assets/javascripts/pages/groups/milestones/edit/index.js index 4f8514a9a1d..6bf73c2563c 100644 --- a/app/assets/javascripts/pages/groups/milestones/edit/index.js +++ b/app/assets/javascripts/pages/groups/milestones/edit/index.js @@ -1,3 +1,3 @@ -import initForm from '~/shared/milestones/form'; +import initForm from '~/milestones/form'; initForm(); diff --git a/app/assets/javascripts/pages/groups/milestones/new/index.js b/app/assets/javascripts/pages/groups/milestones/new/index.js index 4f8514a9a1d..6bf73c2563c 100644 --- a/app/assets/javascripts/pages/groups/milestones/new/index.js +++ b/app/assets/javascripts/pages/groups/milestones/new/index.js @@ -1,3 +1,3 @@ -import initForm from '~/shared/milestones/form'; +import initForm from '~/milestones/form'; initForm(); diff --git a/app/assets/javascripts/pages/groups/milestones/show/index.js b/app/assets/javascripts/pages/groups/milestones/show/index.js index 914e2831185..5eec1015447 100644 --- a/app/assets/javascripts/pages/groups/milestones/show/index.js +++ b/app/assets/javascripts/pages/groups/milestones/show/index.js @@ -1,5 +1,5 @@ -import initDeleteMilestoneModal from '~/pages/milestones/shared/delete_milestone_modal_init'; -import initMilestonesShow from '~/pages/milestones/shared/init_milestones_show'; +import initDeleteMilestoneModal from '~/milestones/delete_milestone_modal_init'; +import initMilestonesShow from '~/milestones/init_milestones_show'; initMilestonesShow(); initDeleteMilestoneModal(); diff --git a/app/assets/javascripts/pages/milestones/shared/index.js b/app/assets/javascripts/pages/milestones/shared/index.js deleted file mode 100644 index dabfe32848b..00000000000 --- a/app/assets/javascripts/pages/milestones/shared/index.js +++ /dev/null @@ -1,7 +0,0 @@ -import initDeleteMilestoneModal from './delete_milestone_modal_init'; -import initPromoteMilestoneModal from './promote_milestone_modal_init'; - -export default () => { - initDeleteMilestoneModal(); - initPromoteMilestoneModal(); -}; diff --git a/app/assets/javascripts/pages/projects/issues/form.js b/app/assets/javascripts/pages/projects/issues/form.js index c0da0069a99..07d6943e287 100644 --- a/app/assets/javascripts/pages/projects/issues/form.js +++ b/app/assets/javascripts/pages/projects/issues/form.js @@ -7,7 +7,7 @@ import GLForm from '~/gl_form'; import initSuggestions from '~/issuable_suggestions'; import initIssuableTypeSelector from '~/issuable_type_selector'; import LabelsSelect from '~/labels_select'; -import MilestoneSelect from '~/milestone_select'; +import MilestoneSelect from '~/milestones/milestone_select'; import IssuableTemplateSelectors from '~/templates/issuable_template_selectors'; export default () => { diff --git a/app/assets/javascripts/pages/projects/merge_requests/init_merge_request.js b/app/assets/javascripts/pages/projects/merge_requests/init_merge_request.js index 7d5719cf8a8..f2777434029 100644 --- a/app/assets/javascripts/pages/projects/merge_requests/init_merge_request.js +++ b/app/assets/javascripts/pages/projects/merge_requests/init_merge_request.js @@ -6,7 +6,7 @@ import ShortcutsNavigation from '~/behaviors/shortcuts/shortcuts_navigation'; import Diff from '~/diff'; import GLForm from '~/gl_form'; import LabelsSelect from '~/labels_select'; -import MilestoneSelect from '~/milestone_select'; +import MilestoneSelect from '~/milestones/milestone_select'; import IssuableTemplateSelectors from '~/templates/issuable_template_selectors'; export default () => { diff --git a/app/assets/javascripts/pages/projects/milestones/edit/index.js b/app/assets/javascripts/pages/projects/milestones/edit/index.js index 4f8514a9a1d..6bf73c2563c 100644 --- a/app/assets/javascripts/pages/projects/milestones/edit/index.js +++ b/app/assets/javascripts/pages/projects/milestones/edit/index.js @@ -1,3 +1,3 @@ -import initForm from '~/shared/milestones/form'; +import initForm from '~/milestones/form'; initForm(); diff --git a/app/assets/javascripts/pages/projects/milestones/index/index.js b/app/assets/javascripts/pages/projects/milestones/index/index.js index 150b506b121..6912ab9f8ba 100644 --- a/app/assets/javascripts/pages/projects/milestones/index/index.js +++ b/app/assets/javascripts/pages/projects/milestones/index/index.js @@ -1,3 +1,5 @@ -import milestones from '~/pages/milestones/shared'; +import initDeleteMilestoneModal from '~/milestones/delete_milestone_modal_init'; +import initPromoteMilestoneModal from '~/milestones/promote_milestone_modal_init'; -milestones(); +initDeleteMilestoneModal(); +initPromoteMilestoneModal(); diff --git a/app/assets/javascripts/pages/projects/milestones/new/index.js b/app/assets/javascripts/pages/projects/milestones/new/index.js index 4f8514a9a1d..6bf73c2563c 100644 --- a/app/assets/javascripts/pages/projects/milestones/new/index.js +++ b/app/assets/javascripts/pages/projects/milestones/new/index.js @@ -1,3 +1,3 @@ -import initForm from '~/shared/milestones/form'; +import initForm from '~/milestones/form'; initForm(); diff --git a/app/assets/javascripts/pages/projects/milestones/show/index.js b/app/assets/javascripts/pages/projects/milestones/show/index.js index 3c755e9b98c..098b3a5f391 100644 --- a/app/assets/javascripts/pages/projects/milestones/show/index.js +++ b/app/assets/javascripts/pages/projects/milestones/show/index.js @@ -1,5 +1,7 @@ -import milestones from '~/pages/milestones/shared'; -import initMilestonesShow from '~/pages/milestones/shared/init_milestones_show'; +import initMilestonesShow from '~/milestones/init_milestones_show'; +import initDeleteMilestoneModal from '~/milestones/delete_milestone_modal_init'; +import initPromoteMilestoneModal from '~/milestones/promote_milestone_modal_init'; initMilestonesShow(); -milestones(); +initDeleteMilestoneModal(); +initPromoteMilestoneModal(); diff --git a/app/controllers/projects/google_cloud/base_controller.rb b/app/controllers/projects/google_cloud/base_controller.rb index 2a9e89a445b..8bfe5c9c5f3 100644 --- a/app/controllers/projects/google_cloud/base_controller.rb +++ b/app/controllers/projects/google_cloud/base_controller.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class Projects::GoogleCloud::BaseController < Projects::ApplicationController - feature_category :google_cloud + feature_category :five_minute_production_app before_action :admin_project_google_cloud! before_action :google_oauth2_enabled! diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index aca556b18cb..3bec7928058 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -73,11 +73,8 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo :show, :toggle_award_emoji, :toggle_subscription, :update ] - feature_category :code_testing, [ - :test_reports, :coverage_reports, :codequality_reports, - :codequality_mr_diff_reports - ] - + feature_category :code_testing, [:test_reports, :coverage_reports] + feature_category :code_quality, [:codequality_reports, :codequality_mr_diff_reports] feature_category :accessibility_testing, [:accessibility_reports] feature_category :infrastructure_as_code, [:terraform_reports] feature_category :continuous_integration, [:pipeline_status, :pipelines, :exposed_artifacts] diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb new file mode 100644 index 00000000000..a497d2cabe5 --- /dev/null +++ b/app/models/ci/namespace_mirror.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Ci + # This model represents a record in a shadow table of the main database's namespaces table. + # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN. + class NamespaceMirror < ApplicationRecord + # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517 + end +end diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb new file mode 100644 index 00000000000..c6e3101fb3a --- /dev/null +++ b/app/models/ci/project_mirror.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Ci + # This model represents a shadow table of the main database's projects table. + # It allows us to navigate the project and namespace hierarchy on the ci database. + class ProjectMirror < ApplicationRecord + # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517 + end +end diff --git a/app/models/postgresql/replication_slot.rb b/app/models/postgresql/replication_slot.rb index 1a4d3bd5794..1c38edcca61 100644 --- a/app/models/postgresql/replication_slot.rb +++ b/app/models/postgresql/replication_slot.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Postgresql - class ReplicationSlot < ApplicationRecord + class ReplicationSlot < Gitlab::Database::SharedModel self.table_name = 'pg_replication_slots' # Returns true if there are any replication slots in use. diff --git a/app/views/projects/issues/_related_branches.html.haml b/app/views/projects/issues/_related_branches.html.haml index c47257eec4a..310a0c1a61e 100644 --- a/app/views/projects/issues/_related_branches.html.haml +++ b/app/views/projects/issues/_related_branches.html.haml @@ -1,7 +1,7 @@ - if @related_branches.any? %h2.gl-font-lg = pluralize(@related_branches.size, 'Related Branch') - %ul.unstyled-list.related-merge-requests + %ul.related-merge-requests.gl-pl-0 - @related_branches.each do |branch| %li.gl-display-flex.gl-align-items-center - if branch[:pipeline_status].present? diff --git a/app/views/projects/merge_requests/invalid.html.haml b/app/views/projects/merge_requests/invalid.html.haml index fd1b2328a98..eb8de425f61 100644 --- a/app/views/projects/merge_requests/invalid.html.haml +++ b/app/views/projects/merge_requests/invalid.html.haml @@ -1,7 +1,12 @@ - page_title "#{@merge_request.title} (#{@merge_request.to_reference}", _("Merge requests") -- badge_css_classes = "badge gl-text-white" -- badge_info_css_classes = "#{badge_css_classes} badge-info" -- badge_inverse_css_classes = "#{badge_css_classes} badge-inverse" + +- badge_start = '<span class="badge badge-pill gl-badge sm badge-info">'.html_safe +- badge_end = '</span>'.html_safe + +- err_fork_project_removed = s_("MergeRequest|Can't show this merge request because the fork project was deleted.") +- err_source_branch = s_("MergeRequest|Can't show this merge request because the source branch %{badge_start}%{source_branch}%{badge_end} is missing from project %{badge_start}%{project_path}%{badge_end}. Close this merge request or update the source branch.") +- err_target_branch = s_("MergeRequest|Can't show this merge request because the target branch %{badge_start}%{target_branch}%{badge_end} is missing from project %{badge_start}%{project_path}%{badge_end}. Close this merge request or update the target branch.") +- err_internal = s_("MergeRequest|Can't show this merge request because of an internal error. Contact your administrator.") .merge-request = render "projects/merge_requests/mr_title" @@ -11,20 +16,12 @@ .gl-alert-container = sprite_icon('error', size: 16, css_class: 'gl-icon gl-alert-icon gl-alert-icon-no-title') .gl-alert-content{ role: 'alert' } - %p - We cannot render this merge request properly because + .gl-alert-body - if @merge_request.for_fork? && !@merge_request.source_project - fork project was removed + = err_fork_project_removed - elsif !@merge_request.source_branch_exists? - %span{ class: badge_inverse_css_classes }= @merge_request.source_branch - does not exist in - %span{ class: badge_info_css_classes }= @merge_request.source_project_path + = err_source_branch.html_safe % { badge_start: badge_start, badge_end: badge_end, source_branch: @merge_request.source_branch, project_path: @merge_request.source_project_path } - elsif !@merge_request.target_branch_exists? - %span{ class: badge_inverse_css_classes }= @merge_request.target_branch - does not exist in - %span{ class: badge_info_css_classes }= @merge_request.target_project_path + = err_target_branch.html_safe % { badge_start: badge_start, badge_end: badge_end, target_branch: @merge_request.target_branch, project_path: @merge_request.source_project_path } - else - of internal error - - %strong - Please close merge request or change branches with existing one + = err_internal diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index fbec46dbc2f..89a4d9dc7cf 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1476,7 +1476,7 @@ :tags: [] - :name: pipeline_background:ci_pipeline_artifacts_create_quality_report :worker_name: Ci::PipelineArtifacts::CreateQualityReportWorker - :feature_category: :code_testing + :feature_category: :code_quality :has_external_dependencies: :urgency: :low :resource_boundary: :unknown diff --git a/app/workers/background_migration/single_database_worker.rb b/app/workers/background_migration/single_database_worker.rb new file mode 100644 index 00000000000..b6661d4fd14 --- /dev/null +++ b/app/workers/background_migration/single_database_worker.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +module BackgroundMigration + module SingleDatabaseWorker + extend ActiveSupport::Concern + + include ApplicationWorker + + MAX_LEASE_ATTEMPTS = 5 + + included do + data_consistency :always + + sidekiq_options retry: 3 + + feature_category :database + urgency :throttled + loggable_arguments 0, 1 + end + + class_methods do + # The minimum amount of time between processing two jobs of the same migration + # class. + # + # This interval is set to 2 or 5 minutes so autovacuuming and other + # maintenance related tasks have plenty of time to clean up after a migration + # has been performed. + def minimum_interval + 2.minutes.to_i + end + + def tracking_database + raise NotImplementedError, "#{self.name} does not implement #{__method__}" + end + + def unhealthy_metric_name + raise NotImplementedError, "#{self.name} does not implement #{__method__}" + end + end + + # Performs the background migration. + # + # See Gitlab::BackgroundMigration.perform for more information. + # + # class_name - The class name of the background migration to run. + # arguments - The arguments to pass to the migration class. + # lease_attempts - The number of times we will try to obtain an exclusive + # lease on the class before giving up. See MR for more discussion. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45298#note_434304956 + def perform(class_name, arguments = [], lease_attempts = MAX_LEASE_ATTEMPTS) + job_coordinator.with_shared_connection do + perform_with_connection(class_name, arguments, lease_attempts) + end + end + + private + + def job_coordinator + @job_coordinator ||= Gitlab::BackgroundMigration.coordinator_for_database(self.class.tracking_database) + end + + def perform_with_connection(class_name, arguments, lease_attempts) + with_context(caller_id: class_name.to_s) do + retried = lease_attempts != MAX_LEASE_ATTEMPTS + attempts_left = lease_attempts - 1 + should_perform, ttl = perform_and_ttl(class_name, attempts_left, retried) + + break if should_perform.nil? + + if should_perform + job_coordinator.perform(class_name, arguments) + else + # If the lease could not be obtained this means either another process is + # running a migration of this class or we ran one recently. In this case + # we'll reschedule the job in such a way that it is picked up again around + # the time the lease expires. + self.class + .perform_in(ttl || self.class.minimum_interval, class_name, arguments, attempts_left) + end + end + end + + def perform_and_ttl(class_name, attempts_left, retried) + # In test environments `perform_in` will run right away. This can then + # lead to stack level errors in the above `#perform`. To work around this + # we'll just perform the migration right away in the test environment. + return [true, nil] if always_perform? + + lease = lease_for(class_name, retried) + lease_obtained = !!lease.try_obtain + healthy_db = healthy_database? + perform = lease_obtained && healthy_db + + database_unhealthy_counter.increment if lease_obtained && !healthy_db + + # When the DB is unhealthy or the lease can't be obtained after several tries, + # then give up on the job and log a warning. Otherwise we could end up in + # an infinite rescheduling loop. Jobs can be tracked in the database with the + # use of Gitlab::Database::BackgroundMigrationJob + if !perform && attempts_left < 0 + msg = if !lease_obtained + 'Job could not get an exclusive lease after several tries. Giving up.' + else + 'Database was unhealthy after several tries. Giving up.' + end + + Sidekiq.logger.warn(class: class_name, message: msg, job_id: jid) + + return [nil, nil] + end + + [perform, lease.ttl] + end + + def lease_for(class_name, retried) + Gitlab::ExclusiveLease + .new(lease_key_for(class_name, retried), timeout: self.class.minimum_interval) + end + + def lease_key_for(class_name, retried) + key = "#{self.class.name}:#{class_name}" + # We use a different exclusive lock key for retried jobs to allow them running concurrently with the scheduled jobs. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68763 for more information. + key += ":retried" if retried + key + end + + def always_perform? + Rails.env.test? + end + + # Returns true if the database is healthy enough to allow the migration to be + # performed. + # + # class_name - The name of the background migration that we might want to + # run. + def healthy_database? + !Postgresql::ReplicationSlot.lag_too_great? + end + + def database_unhealthy_counter + Gitlab::Metrics.counter( + self.class.unhealthy_metric_name, + 'The number of times a background migration is rescheduled because the database is unhealthy.' + ) + end + end +end diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb index b771ab4d4e7..6489aad3173 100644 --- a/app/workers/background_migration_worker.rb +++ b/app/workers/background_migration_worker.rb @@ -1,120 +1,13 @@ # frozen_string_literal: true class BackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker + include BackgroundMigration::SingleDatabaseWorker - MAX_LEASE_ATTEMPTS = 5 - - data_consistency :always - - sidekiq_options retry: 3 - - feature_category :database - urgency :throttled - loggable_arguments 0, 1 - - # The minimum amount of time between processing two jobs of the same migration - # class. - # - # This interval is set to 2 or 5 minutes so autovacuuming and other - # maintenance related tasks have plenty of time to clean up after a migration - # has been performed. - def self.minimum_interval - 2.minutes.to_i - end - - # Performs the background migration. - # - # See Gitlab::BackgroundMigration.perform for more information. - # - # class_name - The class name of the background migration to run. - # arguments - The arguments to pass to the migration class. - # lease_attempts - The number of times we will try to obtain an exclusive - # lease on the class before giving up. See MR for more discussion. - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45298#note_434304956 - def perform(class_name, arguments = [], lease_attempts = MAX_LEASE_ATTEMPTS) - with_context(caller_id: class_name.to_s) do - retried = lease_attempts != MAX_LEASE_ATTEMPTS - attempts_left = lease_attempts - 1 - should_perform, ttl = perform_and_ttl(class_name, attempts_left, retried) - - break if should_perform.nil? - - if should_perform - Gitlab::BackgroundMigration.perform(class_name, arguments) - else - # If the lease could not be obtained this means either another process is - # running a migration of this class or we ran one recently. In this case - # we'll reschedule the job in such a way that it is picked up again around - # the time the lease expires. - self.class - .perform_in(ttl || self.class.minimum_interval, class_name, arguments, attempts_left) - end - end - end - - def perform_and_ttl(class_name, attempts_left, retried) - # In test environments `perform_in` will run right away. This can then - # lead to stack level errors in the above `#perform`. To work around this - # we'll just perform the migration right away in the test environment. - return [true, nil] if always_perform? - - lease = lease_for(class_name, retried) - lease_obtained = !!lease.try_obtain - healthy_db = healthy_database? - perform = lease_obtained && healthy_db - - database_unhealthy_counter.increment if lease_obtained && !healthy_db - - # When the DB is unhealthy or the lease can't be obtained after several tries, - # then give up on the job and log a warning. Otherwise we could end up in - # an infinite rescheduling loop. Jobs can be tracked in the database with the - # use of Gitlab::Database::BackgroundMigrationJob - if !perform && attempts_left < 0 - msg = if !lease_obtained - 'Job could not get an exclusive lease after several tries. Giving up.' - else - 'Database was unhealthy after several tries. Giving up.' - end - - Sidekiq.logger.warn(class: class_name, message: msg, job_id: jid) - - return [nil, nil] - end - - [perform, lease.ttl] - end - - def lease_for(class_name, retried) - Gitlab::ExclusiveLease - .new(lease_key_for(class_name, retried), timeout: self.class.minimum_interval) - end - - def lease_key_for(class_name, retried) - key = "#{self.class.name}:#{class_name}" - # We use a different exclusive lock key for retried jobs to allow them running concurrently with the scheduled jobs. - # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68763 for more information. - key += ":retried" if retried - key - end - - def always_perform? - Rails.env.test? - end - - # Returns true if the database is healthy enough to allow the migration to be - # performed. - # - # class_name - The name of the background migration that we might want to - # run. - def healthy_database? - !Postgresql::ReplicationSlot.lag_too_great? + def self.tracking_database + @tracking_database ||= Gitlab::Database::MAIN_DATABASE_NAME.to_sym end - def database_unhealthy_counter - Gitlab::Metrics.counter( - :background_migration_database_health_reschedules, - 'The number of times a background migration is rescheduled because the database is unhealthy.' - ) + def self.unhealthy_metric_name + @unhealthy_metric_name ||= :background_migration_database_health_reschedules end end diff --git a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb index bb0a81a0a17..dc7e8f888c6 100644 --- a/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb +++ b/app/workers/ci/pipeline_artifacts/create_quality_report_worker.rb @@ -10,7 +10,7 @@ module Ci sidekiq_options retry: 3 queue_namespace :pipeline_background - feature_category :code_testing + feature_category :code_quality idempotent! diff --git a/config/feature_categories.yml b/config/feature_categories.yml index 04804e373ab..edc6541db8c 100644 --- a/config/feature_categories.yml +++ b/config/feature_categories.yml @@ -9,12 +9,14 @@ --- - accessibility_testing - advanced_deployments +- api_security - attack_emulation - audit_events - audit_reports - authentication_and_authorization - auto_devops - backup_restore +- build - build_artifacts - chatops - cloud_native_installation @@ -33,16 +35,18 @@ - database - dataops - delivery -- delivery_management - dependency_firewall - dependency_proxy - dependency_scanning +- deployment_management - design_management +- design_system - devops_reports - disaster_recovery - dynamic_application_security_testing - editor_extension - environment_management +- error_budgets - error_tracking - experimentation_activation - experimentation_adoption @@ -50,20 +54,19 @@ - experimentation_expansion - feature_flags - five_minute_production_app -- foundations - fuzz_testing - geo_replication - git_lfs - gitaly - gitlab_docs - global_search -- google_cloud - helm_chart_registry - horse - importers - incident_management - infrastructure_as_code - insider_threat +- instance_resiliency - integrations - intel_code_security - interactive_application_security_testing @@ -86,7 +89,6 @@ - package_registry - pages - performance_testing -- pipeline_abuse_prevention - pipeline_authoring - planning_analytics - portfolio_management diff --git a/config/feature_flags/development/reference_cache_memoization.yml b/config/feature_flags/development/reference_cache_memoization.yml deleted file mode 100644 index 74012208174..00000000000 --- a/config/feature_flags/development/reference_cache_memoization.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: reference_cache_memoization -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71310 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/341849 -milestone: '14.4' -type: development -group: group::source code -default_enabled: true diff --git a/config/feature_flags/ops/lower_relation_max_count_limit.yml b/config/feature_flags/ops/lower_relation_max_count_limit.yml deleted file mode 100644 index 7a532c0948a..00000000000 --- a/config/feature_flags/ops/lower_relation_max_count_limit.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: lower_relation_max_count_limit -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69620 -rollout_issue_url: -milestone: '14.3' -type: ops -group: group::verify -default_enabled: false diff --git a/config/initializers/kaminari_active_record_relation_methods_with_limit.rb b/config/initializers/kaminari_active_record_relation_methods_with_limit.rb index 9a5a95403ad..982cb69e532 100644 --- a/config/initializers/kaminari_active_record_relation_methods_with_limit.rb +++ b/config/initializers/kaminari_active_record_relation_methods_with_limit.rb @@ -4,7 +4,6 @@ module Kaminari # Active Record specific page scope methods implementations module ActiveRecordRelationMethodsWithLimit MAX_COUNT_LIMIT = 10_000 - MAX_COUNT_NEW_LOWER_LIMIT = 1_000 # This is a modified version of # https://github.com/kaminari/kaminari/blob/c5186f5d9b7f23299d115408e62047447fd3189d/kaminari-activerecord/lib/kaminari/activerecord/active_record_relation_methods.rb#L17-L41 @@ -22,8 +21,7 @@ module Kaminari return @total_count = (current_page - 1) * limit_value + @records.length if @records.any? && (@records.length < limit_value) end - max_limit = Feature.enabled?(:lower_relation_max_count_limit, type: :ops) ? MAX_COUNT_NEW_LOWER_LIMIT : MAX_COUNT_LIMIT - limit = options.fetch(:limit, max_limit).to_i + limit = options.fetch(:limit, MAX_COUNT_LIMIT).to_i # #count overrides the #select which could include generated columns referenced in #order, so skip #order here, where it's irrelevant to the result anyway c = except(:offset, :limit, :order) # Remove includes only if they are irrelevant diff --git a/db/migrate/20211011140930_create_ci_namespace_mirrors.rb b/db/migrate/20211011140930_create_ci_namespace_mirrors.rb new file mode 100644 index 00000000000..b9a708c5d7b --- /dev/null +++ b/db/migrate/20211011140930_create_ci_namespace_mirrors.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CreateCiNamespaceMirrors < Gitlab::Database::Migration[1.0] + TABLE_NAME = :ci_namespace_mirrors + INDEX_NAME = "index_gin_#{TABLE_NAME}_on_traversal_ids" + + def change + create_table TABLE_NAME do |t| + t.integer :namespace_id, null: false, index: { unique: true } + t.integer :traversal_ids, array: true, default: [], null: false + + t.index :traversal_ids, name: INDEX_NAME, using: :gin + end + end +end diff --git a/db/migrate/20211011140931_create_ci_project_mirrors.rb b/db/migrate/20211011140931_create_ci_project_mirrors.rb new file mode 100644 index 00000000000..2407b7e0b84 --- /dev/null +++ b/db/migrate/20211011140931_create_ci_project_mirrors.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class CreateCiProjectMirrors < Gitlab::Database::Migration[1.0] + TABLE_NAME = :ci_project_mirrors + + def change + create_table TABLE_NAME do |t| + t.integer :project_id, null: false, index: { unique: true } + t.integer :namespace_id, null: false, index: true + end + end +end diff --git a/db/schema_migrations/20211011140930 b/db/schema_migrations/20211011140930 new file mode 100644 index 00000000000..6347ee5d51d --- /dev/null +++ b/db/schema_migrations/20211011140930 @@ -0,0 +1 @@ +cdae819e8de3b5ad721014376bfd9af97a45e953e2d345daf62784f986a5eb31
\ No newline at end of file diff --git a/db/schema_migrations/20211011140931 b/db/schema_migrations/20211011140931 new file mode 100644 index 00000000000..c959d97074e --- /dev/null +++ b/db/schema_migrations/20211011140931 @@ -0,0 +1 @@ +7e51eb4443fd74da9bef4d9c1c3cc40376c311abbc05ca7871f725fada79b48a
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index a932ea30de5..3cb591e3b52 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11767,6 +11767,21 @@ CREATE SEQUENCE ci_minutes_additional_packs_id_seq ALTER SEQUENCE ci_minutes_additional_packs_id_seq OWNED BY ci_minutes_additional_packs.id; +CREATE TABLE ci_namespace_mirrors ( + id bigint NOT NULL, + namespace_id integer NOT NULL, + traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL +); + +CREATE SEQUENCE ci_namespace_mirrors_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_namespace_mirrors_id_seq OWNED BY ci_namespace_mirrors.id; + CREATE TABLE ci_namespace_monthly_usages ( id bigint NOT NULL, namespace_id bigint NOT NULL, @@ -12015,6 +12030,21 @@ CREATE SEQUENCE ci_platform_metrics_id_seq ALTER SEQUENCE ci_platform_metrics_id_seq OWNED BY ci_platform_metrics.id; +CREATE TABLE ci_project_mirrors ( + id bigint NOT NULL, + project_id integer NOT NULL, + namespace_id integer NOT NULL +); + +CREATE SEQUENCE ci_project_mirrors_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_project_mirrors_id_seq OWNED BY ci_project_mirrors.id; + CREATE TABLE ci_project_monthly_usages ( id bigint NOT NULL, project_id bigint NOT NULL, @@ -21264,6 +21294,8 @@ ALTER TABLE ONLY ci_job_variables ALTER COLUMN id SET DEFAULT nextval('ci_job_va ALTER TABLE ONLY ci_minutes_additional_packs ALTER COLUMN id SET DEFAULT nextval('ci_minutes_additional_packs_id_seq'::regclass); +ALTER TABLE ONLY ci_namespace_mirrors ALTER COLUMN id SET DEFAULT nextval('ci_namespace_mirrors_id_seq'::regclass); + ALTER TABLE ONLY ci_namespace_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_namespace_monthly_usages_id_seq'::regclass); ALTER TABLE ONLY ci_pending_builds ALTER COLUMN id SET DEFAULT nextval('ci_pending_builds_id_seq'::regclass); @@ -21286,6 +21318,8 @@ ALTER TABLE ONLY ci_pipelines_config ALTER COLUMN pipeline_id SET DEFAULT nextva ALTER TABLE ONLY ci_platform_metrics ALTER COLUMN id SET DEFAULT nextval('ci_platform_metrics_id_seq'::regclass); +ALTER TABLE ONLY ci_project_mirrors ALTER COLUMN id SET DEFAULT nextval('ci_project_mirrors_id_seq'::regclass); + ALTER TABLE ONLY ci_project_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_project_monthly_usages_id_seq'::regclass); ALTER TABLE ONLY ci_refs ALTER COLUMN id SET DEFAULT nextval('ci_refs_id_seq'::regclass); @@ -22727,6 +22761,9 @@ ALTER TABLE ONLY ci_job_variables ALTER TABLE ONLY ci_minutes_additional_packs ADD CONSTRAINT ci_minutes_additional_packs_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_namespace_mirrors + ADD CONSTRAINT ci_namespace_mirrors_pkey PRIMARY KEY (id); + ALTER TABLE ONLY ci_namespace_monthly_usages ADD CONSTRAINT ci_namespace_monthly_usages_pkey PRIMARY KEY (id); @@ -22760,6 +22797,9 @@ ALTER TABLE ONLY ci_pipelines ALTER TABLE ONLY ci_platform_metrics ADD CONSTRAINT ci_platform_metrics_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_project_mirrors + ADD CONSTRAINT ci_project_mirrors_pkey PRIMARY KEY (id); + ALTER TABLE ONLY ci_project_monthly_usages ADD CONSTRAINT ci_project_monthly_usages_pkey PRIMARY KEY (id); @@ -25337,6 +25377,8 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables CREATE INDEX index_ci_minutes_additional_packs_on_namespace_id_purchase_xid ON ci_minutes_additional_packs USING btree (namespace_id, purchase_xid); +CREATE UNIQUE INDEX index_ci_namespace_mirrors_on_namespace_id ON ci_namespace_mirrors USING btree (namespace_id); + CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date); CREATE INDEX index_ci_pending_builds_id_on_protected_partial ON ci_pending_builds USING btree (id) WHERE (protected = true); @@ -25425,6 +25467,10 @@ CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_source ON ci_pipel CREATE INDEX index_ci_pipelines_on_user_id_and_id_and_cancelable_status ON ci_pipelines USING btree (user_id, id) WHERE ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('waiting_for_resource'::character varying)::text, ('preparing'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text, ('scheduled'::character varying)::text])); +CREATE INDEX index_ci_project_mirrors_on_namespace_id ON ci_project_mirrors USING btree (namespace_id); + +CREATE UNIQUE INDEX index_ci_project_mirrors_on_project_id ON ci_project_mirrors USING btree (project_id); + CREATE UNIQUE INDEX index_ci_project_monthly_usages_on_project_id_and_date ON ci_project_monthly_usages USING btree (project_id, date); CREATE UNIQUE INDEX index_ci_refs_on_project_id_and_ref_path ON ci_refs USING btree (project_id, ref_path); @@ -26001,6 +26047,8 @@ CREATE INDEX index_geo_repository_updated_events_on_source ON geo_repository_upd CREATE INDEX index_geo_reset_checksum_events_on_project_id ON geo_reset_checksum_events USING btree (project_id); +CREATE INDEX index_gin_ci_namespace_mirrors_on_traversal_ids ON ci_namespace_mirrors USING gin (traversal_ids); + CREATE INDEX index_gin_ci_pending_builds_on_namespace_traversal_ids ON ci_pending_builds USING gin (namespace_traversal_ids); CREATE INDEX index_gitlab_subscription_histories_on_gitlab_subscription_id ON gitlab_subscription_histories USING btree (gitlab_subscription_id); diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index 21e32d145bd..22159b6e9db 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -31,6 +31,7 @@ You can use the following environment variables to override certain values: | `GITLAB_EMAIL_REPLY_TO` | string | The email address used in the **Reply-To** field in emails sent by GitLab. | | `GITLAB_EMAIL_SUBJECT_SUFFIX` | string | The email subject suffix used in emails sent by GitLab. | | `GITLAB_HOST` | string | The full URL of the GitLab server (including `http://` or `https://`). | +| `GITLAB_MARKUP_TIMEOUT` | string | Timeout, in seconds, for `rest2html` and `pod2html` commands executed by the [`gitlab-markup` gem](https://gitlab.com/gitlab-org/gitlab-markup/). Default is `10`. | | `GITLAB_ROOT_PASSWORD` | string | Sets the password for the `root` user on installation. | | `GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN` | string | Sets the initial registration token used for runners. | | `RAILS_ENV` | string | The Rails environment; can be one of `production`, `development`, `staging`, or `test`. | diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index 02cb7ad0bca..1c9b98041dc 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -263,9 +263,9 @@ This sets the concurrency (number of threads) for the Sidekiq process. ## Modify the check interval -To modify the check interval for the additional Sidekiq processes: +To modify `sidekiq-cluster`'s health check interval for the additional Sidekiq processes: -1. Edit `/etc/gitlab/gitlab.rb` and add: +1. Edit `/etc/gitlab/gitlab.rb` and add (the value can be any integer number of seconds): ```ruby sidekiq['interval'] = 5 @@ -273,8 +273,6 @@ To modify the check interval for the additional Sidekiq processes: 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -This tells the additional processes how often to check for enqueued jobs. - ## Troubleshoot using the CLI WARNING: diff --git a/doc/api/personal_access_tokens.md b/doc/api/personal_access_tokens.md index 9c9551a5103..b51866fe9b1 100644 --- a/doc/api/personal_access_tokens.md +++ b/doc/api/personal_access_tokens.md @@ -40,8 +40,9 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a "scopes": [ "api" ], - "active": true, "user_id": 24, + "last_used_at": "2021-10-06T17:58:37.550Z", + "active": true, "expires_at": null } ] @@ -61,8 +62,9 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a "scopes": [ "api" ], - "active": true, "user_id": 3, + "last_used_at": "2021-10-06T17:58:37.550Z", + "active": true, "expires_at": null } ] diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index 1382ec263f2..745e8f456ec 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -1053,7 +1053,9 @@ Guidance for each individual UI element is in [the word list](word_list.md). ### How to write navigation task steps -To be consistent, use this format when you write navigation steps in a task topic. +To be consistent, use these templates when you write navigation steps in a task topic. + +To open project settings: ```markdown 1. On the top bar, select **Menu > Projects** and find your project. @@ -1061,7 +1063,7 @@ To be consistent, use this format when you write navigation steps in a task topi 1. Expand **General pipelines**. ``` -Another example: +To open group settings: ```markdown 1. On the top bar, select **Menu > Groups** and find your group. @@ -1069,7 +1071,7 @@ Another example: 1. Expand **General pipelines**. ``` -An Admin Area example: +To open the Admin Area: ```markdown 1. On the top bar, select **Menu > Admin**. @@ -1081,6 +1083,15 @@ To select your avatar: 1. On the top bar, in the top right corner, select your avatar. ``` +To save the selection in some dropdown lists: + +```markdown +1. Go to your issue. +1. On the right sidebar, in the **Iteration** section, select **Edit**. +1. From the dropdown list, select the iteration to associate this issue with. +1. Select any area outside the dropdown list. +``` + ### Optional steps If a step is optional, start the step with the word `Optional` followed by a period. diff --git a/doc/subscriptions/gitlab_com/index.md b/doc/subscriptions/gitlab_com/index.md index 2f1447358c1..404f1a75204 100644 --- a/doc/subscriptions/gitlab_com/index.md +++ b/doc/subscriptions/gitlab_com/index.md @@ -389,7 +389,9 @@ locked. Projects can only be unlocked by purchasing more storage subscription un ### Purchase more storage -To purchase more storage for either a personal or group namespace: +You can purchase storage for your personal or group namespace. + +#### For your personal namespace 1. Sign in to GitLab SaaS. 1. From either your personal homepage or the group's page, go to **Settings > Usage Quotas**. @@ -409,6 +411,32 @@ To purchase more storage for either a personal or group namespace: The **Purchased storage available** total is incremented by the amount purchased. All locked projects are unlocked and their excess usage is deducted from the additional storage. +#### For your group namespace + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5789) in GitLab 14.6. + +If you're using GitLab SaaS, you can purchase additional storage so your +pipelines aren't blocked after you have used all your storage from your +main quota. You can find pricing for additional storage on the +[GitLab Pricing page](https://about.gitlab.com/pricing/). + +To purchase additional storage for your group on GitLab SaaS: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > Usage Quotas**. +1. Select **Storage** tab. +1. Select **Purchase more storage**. +1. Complete the details. + +After your payment is processed, the extra storage is available for your group +namespace. + +To confirm the available storage, go to your group, and then select +**Settings > Usage Quotas** and select the **Storage** tab. + +The **Purchased storage available** total is incremented by the amount purchased. All locked +projects are unlocked and their excess usage is deducted from the additional storage. + ## Contact Support Learn more about: diff --git a/doc/user/admin_area/analytics/dev_ops_report.md b/doc/user/admin_area/analytics/dev_ops_report.md index b28789cfd5b..62fea3c266a 100644 --- a/doc/user/admin_area/analytics/dev_ops_report.md +++ b/doc/user/admin_area/analytics/dev_ops_report.md @@ -37,7 +37,7 @@ information is **not sent** to any other GitLab instances. If you have just started using GitLab, it might take a few weeks for data to be collected before this feature is available. -## DevOps Adoption **(ULTIMATE)** +## DevOps Adoption **(ULTIMATE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/247112) in GitLab 13.7 as a [Beta feature](https://about.gitlab.com/handbook/product/gitlab-the-product/#beta). > - The Overview tab [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330401) in GitLab 14.1. diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md index a35c1cd8ab1..8a79effba15 100644 --- a/doc/user/group/iterations/index.md +++ b/doc/user/group/iterations/index.md @@ -165,7 +165,7 @@ To group issues by label: 1. Select the **Filter by label** dropdown. 1. Select the labels you want to group by in the labels dropdown. You can also search for labels by typing in the search input. -1. Select or tap outside of the label dropdown. The page is now grouped by the selected labels. +1. Select any area outside the label dropdown list. The page is now grouped by the selected labels. ### Enable or disable iteration cadences **(PREMIUM SELF)** diff --git a/doc/user/markdown.md b/doc/user/markdown.md index b6c8bc19b64..0a51020dbe0 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -11,7 +11,7 @@ GitLab automatically renders Markdown content. For example, when you add a comme you type the text in the Markdown language. When you save the issue, the text is rendered with a set of styles. These styles are described on this page. -For example, in Markdown, an ordered list looks like this: +For example, in Markdown, an unordered list looks like this: ```markdown - Cat diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index b46672c9399..d357f45b1e7 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -393,7 +393,7 @@ To add an issue to an [iteration](../../group/iterations/index.md): 1. Go to your issue. 1. On the right sidebar, in the **Iteration** section, select **Edit**. 1. From the dropdown list, select the iteration to associate this issue with. -1. Click or tap anywhere outside of the dropdown. +1. Select any area outside the dropdown list. You can also use the `/iteration` [quick action](../quick_actions.md#issues-merge-requests-and-epics) @@ -457,7 +457,7 @@ To change the assignee on an issue: 1. Go to your issue. 1. On the right sidebar, in the **Assignee** section, select **Edit**. 1. From the dropdown list, select the user to add as an assignee. -1. Click or tap anywhere outside of the dropdown list. +1. Select any area outside the dropdown list. ## Similar issues diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index e9cbe012110..91f12a36070 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -42,8 +42,8 @@ To assign or unassign a label: You can search repeatedly to add more labels. The selected labels are marked with a checkmark. 1. Click the labels you want to assign or unassign. -1. To apply your changes to labels, click **X** next to **Assign labels** or anywhere outside the - label section. +1. To apply your changes to labels, select **X** next to **Assign labels** or select any area + outside the label section. Alternatively, to unassign a label, click the **X** on the label you want to unassign. diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index d75a180492e..40c9ae8bd59 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -456,7 +456,7 @@ when: - <kbd>Control</kbd> + <kbd>S</kbd> (or <kbd>Command</kbd> + <kbd>S</kbd> on Mac) is pressed while editing a file. -- Anything outside the file editor is clicked after editing a file. +- You select any area outside the file editor after editing a file. - A file or folder is created, deleted, or renamed. ### Limitations diff --git a/lib/api/entities/personal_access_token.rb b/lib/api/entities/personal_access_token.rb index 3846929c903..55764daef9d 100644 --- a/lib/api/entities/personal_access_token.rb +++ b/lib/api/entities/personal_access_token.rb @@ -3,7 +3,7 @@ module API module Entities class PersonalAccessToken < Grape::Entity - expose :id, :name, :revoked, :created_at, :scopes, :user_id + expose :id, :name, :revoked, :created_at, :scopes, :user_id, :last_used_at expose :active?, as: :active expose :expires_at do |personal_access_token| personal_access_token.expires_at ? personal_access_token.expires_at.strftime("%Y-%m-%d") : nil diff --git a/lib/banzai/filter/references/reference_cache.rb b/lib/banzai/filter/references/reference_cache.rb index 259958f1598..c8370d5f9c1 100644 --- a/lib/banzai/filter/references/reference_cache.rb +++ b/lib/banzai/filter/references/reference_cache.rb @@ -29,7 +29,7 @@ module Banzai @references_per_parent[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - prepare_doc_for_scan(filter.doc).to_enum(:scan, regex).each do + prepare_doc_for_scan.to_enum(:scan, regex).each do parent_path = if parent_type == :project full_project_path($~[:namespace], $~[:project]) else @@ -184,14 +184,12 @@ module Banzai Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} end - def prepare_doc_for_scan(doc) - html = if Feature.enabled?(:reference_cache_memoization, project, default_enabled: :yaml) - result[:rendered_html] ||= doc.to_html - else - doc.to_html - end + def prepare_doc_for_scan + filter.requires_unescaping? ? unescape_html_entities(html_content) : html_content + end - filter.requires_unescaping? ? unescape_html_entities(html) : html + def html_content + result[:rendered_html] ||= filter.doc.to_html end def unescape_html_entities(text) diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb index 2aec496880f..04ac0916bbc 100644 --- a/lib/bulk_imports/projects/graphql/get_project_query.rb +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -4,6 +4,7 @@ module BulkImports module Projects module Graphql module GetProjectQuery + extend Queryable extend self def to_s @@ -28,22 +29,6 @@ module BulkImports } GRAPHQL end - - def variables(context) - { full_path: context.entity.source_full_path } - end - - def base_path - %w[data project] - end - - def data_path - base_path - end - - def page_info_path - base_path << 'page_info' - end end end end diff --git a/lib/bulk_imports/projects/graphql/get_repository_query.rb b/lib/bulk_imports/projects/graphql/get_repository_query.rb index d3e377c1175..24efce9e276 100644 --- a/lib/bulk_imports/projects/graphql/get_repository_query.rb +++ b/lib/bulk_imports/projects/graphql/get_repository_query.rb @@ -4,6 +4,7 @@ module BulkImports module Projects module Graphql module GetRepositoryQuery + extend Queryable extend self def to_s @@ -15,22 +16,6 @@ module BulkImports } GRAPHQL end - - def variables(context) - { full_path: context.entity.source_full_path } - end - - def base_path - %w[data project] - end - - def data_path - base_path - end - - def page_info_path - base_path << 'page_info' - end end end end diff --git a/lib/bulk_imports/projects/graphql/queryable.rb b/lib/bulk_imports/projects/graphql/queryable.rb new file mode 100644 index 00000000000..a897632dff3 --- /dev/null +++ b/lib/bulk_imports/projects/graphql/queryable.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module Queryable + def variables(context) + { full_path: context.entity.source_full_path } + end + + def base_path + %w[data project] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/generators/gitlab/usage_metric_generator.rb b/lib/generators/gitlab/usage_metric_generator.rb index c0fdcf21f20..0656dfbc312 100644 --- a/lib/generators/gitlab/usage_metric_generator.rb +++ b/lib/generators/gitlab/usage_metric_generator.rb @@ -5,9 +5,9 @@ require 'rails/generators' module Gitlab class UsageMetricGenerator < Rails::Generators::Base CE_DIR = 'lib/gitlab/usage/metrics/instrumentations' - EE_DIR = 'ee/lib/ee/gitlab/usage/metrics/instrumentations' + EE_DIR = 'ee/lib/gitlab/usage/metrics/instrumentations' SPEC_CE_DIR = 'spec/lib/gitlab/usage/metrics/instrumentations' - SPEC_EE_DIR = 'ee/spec/lib/ee/gitlab/usage/metrics/instrumentations' + SPEC_EE_DIR = 'ee/spec/lib/gitlab/usage/metrics/instrumentations' ALLOWED_SUPERCLASSES = { generic: 'Generic', diff --git a/lib/gitlab/anonymous_session.rb b/lib/gitlab/anonymous_session.rb index fc78b64830a..e58240e16b4 100644 --- a/lib/gitlab/anonymous_session.rb +++ b/lib/gitlab/anonymous_session.rb @@ -10,9 +10,9 @@ module Gitlab def count_session_ip redis_store_class.with do |redis| - redis.pipelined do - redis.incr(session_lookup_name) - redis.expire(session_lookup_name, 24.hours) + redis.pipelined do |pipeline| + pipeline.incr(session_lookup_name) + pipeline.expire(session_lookup_name, 24.hours) end end end diff --git a/lib/gitlab/database/gitlab_loose_foreign_keys.yml b/lib/gitlab/database/gitlab_loose_foreign_keys.yml index 430ac785449..907a10d0fa3 100644 --- a/lib/gitlab/database/gitlab_loose_foreign_keys.yml +++ b/lib/gitlab/database/gitlab_loose_foreign_keys.yml @@ -18,3 +18,14 @@ clusters_applications_runners: - table: ci_runners column: runner_id on_delete: async_nullify +ci_namespace_mirrors: + - table: namespaces + column: namespace_id + on_delete: async_delete +ci_project_mirrors: + - table: projects + column: project_id + on_delete: async_delete + - table: namespaces + column: namespace_id + on_delete: async_delete diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 5695f2f1c14..6733318bfaf 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -86,6 +86,7 @@ ci_job_token_project_scope_links: :gitlab_ci ci_job_variables: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci +ci_namespace_mirrors: :gitlab_ci ci_pending_builds: :gitlab_ci ci_pipeline_artifacts: :gitlab_ci ci_pipeline_chat_data: :gitlab_ci @@ -97,6 +98,7 @@ ci_pipelines: :gitlab_ci ci_pipeline_variables: :gitlab_ci ci_platform_metrics: :gitlab_ci ci_project_monthly_usages: :gitlab_ci +ci_project_mirrors: :gitlab_ci ci_refs: :gitlab_ci ci_resource_groups: :gitlab_ci ci_resources: :gitlab_ci diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 7b5013f137b..4f8a6ffb2cc 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -29,7 +29,7 @@ module Gitlab return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) limited_total_count = pagination_data.total_count_with_limit - if limited_total_count > max_limit + if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` # We need to call `reset` because `without_count` relies on `@arel` being unmemoized pagination_data.reset.without_count @@ -38,14 +38,6 @@ module Gitlab end end - def max_limit - if Feature.enabled?(:lower_relation_max_count_limit, type: :ops) - Kaminari::ActiveRecordRelationMethods::MAX_COUNT_NEW_LOWER_LIMIT - else - Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT - end - end - def needs_pagination?(relation) return true unless relation.respond_to?(:current_page) return true if params[:page].present? && relation.current_page != params[:page].to_i diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 2602cb54adb..6616f0d4cec 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -21,6 +21,8 @@ module Gitlab FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.' FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' + SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze + READ_COMMANDS = %i( get mget @@ -109,6 +111,8 @@ module Gitlab end def log_method_missing(command_name, *_args) + return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) + log_error(MethodMissingError.new, command_name) increment_method_missing_count(command_name) end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 8ccbea7b460..e82b4e64e59 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -21997,6 +21997,18 @@ msgstr "" msgid "MergeRequest|Approved by @%{username}" msgstr "" +msgid "MergeRequest|Can't show this merge request because of an internal error. Contact your administrator." +msgstr "" + +msgid "MergeRequest|Can't show this merge request because the fork project was deleted." +msgstr "" + +msgid "MergeRequest|Can't show this merge request because the source branch %{badge_start}%{source_branch}%{badge_end} is missing from project %{badge_start}%{project_path}%{badge_end}. Close this merge request or update the source branch." +msgstr "" + +msgid "MergeRequest|Can't show this merge request because the target branch %{badge_start}%{target_branch}%{badge_end} is missing from project %{badge_start}%{project_path}%{badge_end}. Close this merge request or update the target branch." +msgstr "" + msgid "MergeRequest|Compare %{target} and %{source}" msgstr "" diff --git a/package.json b/package.json index 61601e10ba1..648847af81e 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "@rails/ujs": "6.1.4-1", "@sentry/browser": "5.30.0", "@sourcegraph/code-host-integration": "0.0.60", - "@tiptap/core": "^2.0.0-beta.142", + "@tiptap/core": "^2.0.0-beta.143", "@tiptap/extension-blockquote": "^2.0.0-beta.25", "@tiptap/extension-bold": "^2.0.0-beta.24", "@tiptap/extension-bullet-list": "^2.0.0-beta.23", @@ -89,7 +89,7 @@ "@tiptap/extension-table-cell": "^2.0.0-beta.20", "@tiptap/extension-table-header": "^2.0.0-beta.22", "@tiptap/extension-table-row": "^2.0.0-beta.19", - "@tiptap/extension-task-item": "^2.0.0-beta.28", + "@tiptap/extension-task-item": "^2.0.0-beta.29", "@tiptap/extension-task-list": "^2.0.0-beta.23", "@tiptap/extension-text": "^2.0.0-beta.15", "@tiptap/vue-2": "^2.0.0-beta.69", @@ -165,7 +165,7 @@ "prosemirror-model": "^1.15.0", "prosemirror-state": "^1.3.4", "prosemirror-tables": "^1.1.1", - "prosemirror-view": "^1.23.2", + "prosemirror-view": "^1.23.3", "raphael": "^2.2.7", "raw-loader": "^4.0.2", "scrollparent": "^2.0.1", diff --git a/spec/features/dashboard/issues_spec.rb b/spec/features/dashboard/issues_spec.rb index 0b2811618b5..a9fb6a2ae7e 100644 --- a/spec/features/dashboard/issues_spec.rb +++ b/spec/features/dashboard/issues_spec.rb @@ -49,7 +49,7 @@ RSpec.describe 'Dashboard Issues' do describe 'new issue dropdown' do it 'shows projects only with issues feature enabled', :js do - find('.new-project-item-select-button').click + click_button 'Toggle project select' page.within('.select2-results') do expect(page).to have_content(project.full_name) @@ -58,7 +58,7 @@ RSpec.describe 'Dashboard Issues' do end it 'shows the new issue page', :js do - find('.new-project-item-select-button').click + click_button 'Toggle project select' wait_for_requests diff --git a/spec/features/dashboard/merge_requests_spec.rb b/spec/features/dashboard/merge_requests_spec.rb index aa2485d4236..6239702edde 100644 --- a/spec/features/dashboard/merge_requests_spec.rb +++ b/spec/features/dashboard/merge_requests_spec.rb @@ -34,7 +34,7 @@ RSpec.describe 'Dashboard Merge Requests' do end it 'shows projects only with merge requests feature enabled', :js do - find('.new-project-item-select-button').click + click_button 'Toggle project select' page.within('.select2-results') do expect(page).to have_content(project.full_name) diff --git a/spec/features/dashboard/milestones_spec.rb b/spec/features/dashboard/milestones_spec.rb index 992ed2f2ce6..1ba16bf879a 100644 --- a/spec/features/dashboard/milestones_spec.rb +++ b/spec/features/dashboard/milestones_spec.rb @@ -35,7 +35,7 @@ RSpec.describe 'Dashboard > Milestones' do describe 'new milestones dropdown', :js do it 'takes user to a new milestone page', :js do - find('.new-project-item-select-button').click + click_button 'Toggle project select' page.within('.select2-results') do first('.select2-result-label').click diff --git a/spec/features/groups/empty_states_spec.rb b/spec/features/groups/empty_states_spec.rb index 4488f53a03f..0317f9162cc 100644 --- a/spec/features/groups/empty_states_spec.rb +++ b/spec/features/groups/empty_states_spec.rb @@ -101,7 +101,7 @@ RSpec.describe 'Group empty states' do it "the new #{issuable_name} button opens a project dropdown" do within '.empty-state' do - find('.new-project-item-select-button').click + click_button 'Toggle project select' end expect(page).to have_selector('.ajax-project-dropdown') diff --git a/spec/features/groups/issues_spec.rb b/spec/features/groups/issues_spec.rb index 4e59ab40d04..1bac1bcdf5a 100644 --- a/spec/features/groups/issues_spec.rb +++ b/spec/features/groups/issues_spec.rb @@ -119,8 +119,9 @@ RSpec.describe 'Group issues page' do end it 'shows projects only with issues feature enabled', :js do - find('.empty-state .js-lazy-loaded') - find('.empty-state .new-project-item-link').click + within '.empty-state' do + click_button 'Toggle project select' + end page.within('.select2-results') do expect(page).to have_content(project.full_name) @@ -158,9 +159,7 @@ RSpec.describe 'Group issues page' do it 'each issue item has a user-can-drag css applied' do visit issues_group_path(group, sort: 'relative_position') - page.within('.manual-ordering') do - expect(page).to have_selector('.issue.user-can-drag', count: 3) - end + expect(page).to have_selector('.issue.user-can-drag', count: 3) end it 'issues should be draggable and persist order' do @@ -224,7 +223,8 @@ RSpec.describe 'Group issues page' do end it 'shows the pagination' do - expect(page).to have_selector('.gl-pagination') + expect(page).to have_link 'Prev' + expect(page).to have_link 'Next' end it 'first pagination item is active' do diff --git a/spec/features/issuables/sorting_list_spec.rb b/spec/features/issuables/sorting_list_spec.rb index 6e07c6ffed2..f646cdbd71b 100644 --- a/spec/features/issuables/sorting_list_spec.rb +++ b/spec/features/issuables/sorting_list_spec.rb @@ -197,17 +197,13 @@ RSpec.describe 'Sort Issuable List' do click_button('Created date') click_on('Last updated') - wait_for_requests - - expect(first_issue).to include(last_updated_issuable.title) - expect(last_issue).to include(first_updated_issuable.title) + expect(page).to have_css('.issue:first-child', text: last_updated_issuable.title) + expect(page).to have_css('.issue:last-child', text: first_updated_issuable.title) click_on 'Sort direction' - wait_for_requests - - expect(first_issue).to include(first_updated_issuable.title) - expect(last_issue).to include(last_updated_issuable.title) + expect(page).to have_css('.issue:first-child', text: first_updated_issuable.title) + expect(page).to have_css('.issue:last-child', text: last_updated_issuable.title) end end end diff --git a/spec/features/issues/user_bulk_edits_issues_labels_spec.rb b/spec/features/issues/user_bulk_edits_issues_labels_spec.rb index 97df2d0208b..71213fb661f 100644 --- a/spec/features/issues/user_bulk_edits_issues_labels_spec.rb +++ b/spec/features/issues/user_bulk_edits_issues_labels_spec.rb @@ -12,6 +12,9 @@ RSpec.describe 'Issues > Labels bulk assignment' do let!(:issue1) { create(:issue, project: project, title: "Issue 1", labels: [frontend]) } let!(:issue2) { create(:issue, project: project, title: "Issue 2") } + let(:issue_1_selector) { "#issue_#{issue1.id}" } + let(:issue_2_selector) { "#issue_#{issue2.id}" } + context 'as an allowed user', :js do before do project.add_maintainer(user) @@ -44,10 +47,10 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'frontend' - expect(find("#issue_#{issue2.id}")).to have_content 'bug' - expect(find("#issue_#{issue2.id}")).not_to have_content 'frontend' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'frontend' + expect(find(issue_2_selector)).to have_content 'bug' + expect(find(issue_2_selector)).not_to have_content 'frontend' end end @@ -60,10 +63,10 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'frontend' - expect(find("#issue_#{issue2.id}")).to have_content 'bug' - expect(find("#issue_#{issue2.id}")).not_to have_content 'frontend' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'frontend' + expect(find(issue_2_selector)).to have_content 'bug' + expect(find(issue_2_selector)).not_to have_content 'frontend' end end @@ -75,10 +78,10 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'frontend' - expect(find("#issue_#{issue2.id}")).not_to have_content 'bug' - expect(find("#issue_#{issue2.id}")).not_to have_content 'frontend' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'frontend' + expect(find(issue_2_selector)).not_to have_content 'bug' + expect(find(issue_2_selector)).not_to have_content 'frontend' end end @@ -90,10 +93,10 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'frontend' - expect(find("#issue_#{issue2.id}")).not_to have_content 'bug' - expect(find("#issue_#{issue2.id}")).not_to have_content 'frontend' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'frontend' + expect(find(issue_2_selector)).not_to have_content 'bug' + expect(find(issue_2_selector)).not_to have_content 'frontend' end end end @@ -107,10 +110,10 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'feature' - expect(find("#issue_#{issue2.id}")).to have_content 'bug' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'feature' + expect(find(issue_2_selector)).to have_content 'bug' + expect(find(issue_2_selector)).to have_content 'feature' end end @@ -122,10 +125,10 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'feature' - expect(find("#issue_#{issue2.id}")).not_to have_content 'bug' - expect(find("#issue_#{issue2.id}")).not_to have_content 'feature' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'feature' + expect(find(issue_2_selector)).not_to have_content 'bug' + expect(find(issue_2_selector)).not_to have_content 'feature' end end end @@ -144,8 +147,8 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue2.id}")).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_2_selector)).to have_content 'bug' end end @@ -164,10 +167,10 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).not_to have_content 'bug' - expect(find("#issue_#{issue1.id}")).not_to have_content 'feature' - expect(find("#issue_#{issue2.id}")).not_to have_content 'bug' - expect(find("#issue_#{issue2.id}")).not_to have_content 'feature' + expect(find(issue_1_selector)).not_to have_content 'bug' + expect(find(issue_1_selector)).not_to have_content 'feature' + expect(find(issue_2_selector)).not_to have_content 'bug' + expect(find(issue_2_selector)).not_to have_content 'feature' end end @@ -183,8 +186,8 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).not_to have_content 'bug' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' + expect(find(issue_1_selector)).not_to have_content 'bug' + expect(find(issue_2_selector)).to have_content 'feature' end end @@ -203,10 +206,10 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).not_to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'feature' - expect(find("#issue_#{issue2.id}")).not_to have_content 'bug' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' + expect(find(issue_1_selector)).not_to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'feature' + expect(find(issue_2_selector)).not_to have_content 'bug' + expect(find(issue_2_selector)).to have_content 'feature' end end end @@ -222,18 +225,18 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it 'keeps labels' do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_2_selector)).to have_content 'feature' check 'Select all' open_milestone_dropdown(['First Release']) update_issues - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'First Release' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' - expect(find("#issue_#{issue2.id}")).to have_content 'First Release' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'First Release' + expect(find(issue_2_selector)).to have_content 'feature' + expect(find(issue_2_selector)).to have_content 'First Release' end end @@ -244,18 +247,18 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it 'keeps existing label and new label is present' do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'bug' check 'Select all' open_milestone_dropdown ['First Release'] open_labels_dropdown ['feature'] update_issues - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'feature' - expect(find("#issue_#{issue1.id}")).to have_content 'First Release' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' - expect(find("#issue_#{issue2.id}")).to have_content 'First Release' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'feature' + expect(find(issue_1_selector)).to have_content 'First Release' + expect(find(issue_2_selector)).to have_content 'feature' + expect(find(issue_2_selector)).to have_content 'First Release' end end @@ -269,9 +272,9 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it 'keeps existing label and new label is present' do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_2_selector)).to have_content 'feature' check 'Select all' @@ -279,11 +282,11 @@ RSpec.describe 'Issues > Labels bulk assignment' do unmark_labels_in_dropdown ['feature'] update_issues - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).not_to have_content 'feature' - expect(find("#issue_#{issue1.id}")).to have_content 'First Release' - expect(find("#issue_#{issue2.id}")).not_to have_content 'feature' - expect(find("#issue_#{issue2.id}")).to have_content 'First Release' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).not_to have_content 'feature' + expect(find(issue_1_selector)).to have_content 'First Release' + expect(find(issue_2_selector)).not_to have_content 'feature' + expect(find(issue_2_selector)).to have_content 'First Release' end end @@ -300,19 +303,19 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it 'keeps labels' do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'First Release' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' - expect(find("#issue_#{issue2.id}")).to have_content 'First Release' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'First Release' + expect(find(issue_2_selector)).to have_content 'feature' + expect(find(issue_2_selector)).to have_content 'First Release' check 'Select all' open_milestone_dropdown(['No milestone']) update_issues - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).not_to have_content 'First Release' - expect(find("#issue_#{issue2.id}")).to have_content 'feature' - expect(find("#issue_#{issue2.id}")).not_to have_content 'First Release' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).not_to have_content 'First Release' + expect(find(issue_2_selector)).to have_content 'feature' + expect(find(issue_2_selector)).not_to have_content 'First Release' end end end @@ -324,7 +327,7 @@ RSpec.describe 'Issues > Labels bulk assignment' do end it do - expect(find("#issue_#{issue1.id}")).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'bug' check_issue issue1 open_labels_dropdown ['feature'] @@ -333,8 +336,8 @@ RSpec.describe 'Issues > Labels bulk assignment' do update_issues sleep 1 # needed - expect(find("#issue_#{issue1.id}")).to have_content 'bug' - expect(find("#issue_#{issue1.id}")).to have_content 'feature' + expect(find(issue_1_selector)).to have_content 'bug' + expect(find(issue_1_selector)).to have_content 'feature' end end @@ -350,7 +353,7 @@ RSpec.describe 'Issues > Labels bulk assignment' do update_issues - expect(find("#issue_#{issue1.id}")).to have_content 'feature' + expect(find(issue_1_selector)).to have_content 'feature' end end @@ -381,12 +384,12 @@ RSpec.describe 'Issues > Labels bulk assignment' do update_issues - first_issue = find("#issue_#{issue1.id}") + first_issue = find(issue_1_selector) expect(first_issue).not_to have_content 'bug' expect(first_issue).to have_content 'feature' expect(first_issue).to have_content 'wontfix' - second_issue = find("#issue_#{issue2.id}") + second_issue = find(issue_2_selector) expect(second_issue).not_to have_content 'bug' expect(second_issue).not_to have_content 'feature' expect(second_issue).to have_content 'wontfix' diff --git a/spec/features/labels_hierarchy_spec.rb b/spec/features/labels_hierarchy_spec.rb index b5f414e39e4..6c8d41fd96f 100644 --- a/spec/features/labels_hierarchy_spec.rb +++ b/spec/features/labels_hierarchy_spec.rb @@ -64,7 +64,7 @@ RSpec.describe 'Labels Hierarchy', :js do end else expect_issues_list_count(1) - expect(page).to have_selector('span.issue-title-text', text: labeled_issue.title) + expect(page).to have_selector('.issue-title', text: labeled_issue.title) end end end @@ -74,7 +74,7 @@ RSpec.describe 'Labels Hierarchy', :js do wait_for_requests - expect(page).not_to have_selector('.btn-link', text: child_group_label.title) + expect(page).not_to have_link child_group_label.title end end @@ -107,9 +107,9 @@ RSpec.describe 'Labels Hierarchy', :js do end else expect_issues_list_count(3) - expect(page).to have_selector('span.issue-title-text', text: labeled_issue.title) - expect(page).to have_selector('span.issue-title-text', text: labeled_issue_2.title) - expect(page).to have_selector('span.issue-title-text', text: labeled_issue_3.title) + expect(page).to have_selector('.issue-title', text: labeled_issue.title) + expect(page).to have_selector('.issue-title', text: labeled_issue_2.title) + expect(page).to have_selector('.issue-title', text: labeled_issue_3.title) end end end @@ -129,7 +129,7 @@ RSpec.describe 'Labels Hierarchy', :js do end else expect_issues_list_count(1) - expect(page).to have_selector('span.issue-title-text', text: labeled_issue_3.title) + expect(page).to have_selector('.issue-title', text: labeled_issue_3.title) end end @@ -231,7 +231,7 @@ RSpec.describe 'Labels Hierarchy', :js do wait_for_requests - expect(page).not_to have_selector('.btn-link', text: child_group_label.title) + expect(page).not_to have_link child_group_label.title end end diff --git a/spec/frontend/pages/milestones/shared/components/delete_milestone_modal_spec.js b/spec/frontend/milestones/components/delete_milestone_modal_spec.js index 1fbec0d996d..8978de0e0e0 100644 --- a/spec/frontend/pages/milestones/shared/components/delete_milestone_modal_spec.js +++ b/spec/frontend/milestones/components/delete_milestone_modal_spec.js @@ -3,8 +3,8 @@ import { TEST_HOST } from 'helpers/test_constants'; import mountComponent from 'helpers/vue_mount_component_helper'; import axios from '~/lib/utils/axios_utils'; import { redirectTo } from '~/lib/utils/url_utility'; -import deleteMilestoneModal from '~/pages/milestones/shared/components/delete_milestone_modal.vue'; -import eventHub from '~/pages/milestones/shared/event_hub'; +import deleteMilestoneModal from '~/milestones/components/delete_milestone_modal.vue'; +import eventHub from '~/milestones/event_hub'; jest.mock('~/lib/utils/url_utility', () => ({ ...jest.requireActual('~/lib/utils/url_utility'), diff --git a/spec/frontend/pages/milestones/shared/components/promote_milestone_modal_spec.js b/spec/frontend/milestones/components/promote_milestone_modal_spec.js index 4280a78c202..11eaa92f2b0 100644 --- a/spec/frontend/pages/milestones/shared/components/promote_milestone_modal_spec.js +++ b/spec/frontend/milestones/components/promote_milestone_modal_spec.js @@ -6,7 +6,7 @@ import waitForPromises from 'helpers/wait_for_promises'; import createFlash from '~/flash'; import axios from '~/lib/utils/axios_utils'; import * as urlUtils from '~/lib/utils/url_utility'; -import PromoteMilestoneModal from '~/pages/milestones/shared/components/promote_milestone_modal.vue'; +import PromoteMilestoneModal from '~/milestones/components/promote_milestone_modal.vue'; jest.mock('~/lib/utils/url_utility'); jest.mock('~/flash'); diff --git a/spec/lib/api/entities/personal_access_token_spec.rb b/spec/lib/api/entities/personal_access_token_spec.rb new file mode 100644 index 00000000000..fd3c53a21b4 --- /dev/null +++ b/spec/lib/api/entities/personal_access_token_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe API::Entities::PersonalAccessToken do + describe '#as_json' do + let_it_be(:user) { create(:user) } + let_it_be(:token) { create(:personal_access_token, user: user, expires_at: nil) } + + let(:entity) { described_class.new(token) } + + it 'returns token data' do + expect(entity.as_json).to eq({ + id: token.id, + name: token.name, + revoked: false, + created_at: token.created_at, + scopes: ['api'], + user_id: user.id, + last_used_at: nil, + active: true, + expires_at: nil + }) + end + end +end diff --git a/spec/lib/banzai/filter/references/reference_cache_spec.rb b/spec/lib/banzai/filter/references/reference_cache_spec.rb index dcd153da16a..dc43c33a08d 100644 --- a/spec/lib/banzai/filter/references/reference_cache_spec.rb +++ b/spec/lib/banzai/filter/references/reference_cache_spec.rb @@ -35,18 +35,6 @@ RSpec.describe Banzai::Filter::References::ReferenceCache do subject end - - context 'when feature flag is disabled' do - before do - stub_feature_flags(reference_cache_memoization: false) - end - - it 'ignores memoized rendered HTML' do - expect(doc).to receive(:to_html).and_call_original - - subject - end - end end context 'when result is not available' do diff --git a/spec/lib/gitlab/pagination/offset_pagination_spec.rb b/spec/lib/gitlab/pagination/offset_pagination_spec.rb index ffecbb06ff8..f8d50fbc517 100644 --- a/spec/lib/gitlab/pagination/offset_pagination_spec.rb +++ b/spec/lib/gitlab/pagination/offset_pagination_spec.rb @@ -82,7 +82,7 @@ RSpec.describe Gitlab::Pagination::OffsetPagination do context 'when the api_kaminari_count_with_limit feature flag is enabled' do before do - stub_feature_flags(api_kaminari_count_with_limit: true, lower_relation_max_count_limit: false) + stub_feature_flags(api_kaminari_count_with_limit: true) end context 'when resources count is less than MAX_COUNT_LIMIT' do @@ -120,41 +120,6 @@ RSpec.describe Gitlab::Pagination::OffsetPagination do end end - context 'when lower_relation_max_count_limit FF is enabled' do - before do - stub_feature_flags(lower_relation_max_count_limit: true) - end - - it_behaves_like 'paginated response' - it_behaves_like 'response with pagination headers' - - context 'when limit is met' do - before do - stub_const("::Kaminari::ActiveRecordRelationMethods::MAX_COUNT_NEW_LOWER_LIMIT", 2) - end - - it_behaves_like 'paginated response' - - it 'does not return the X-Total and X-Total-Pages headers' do - expect_no_header('X-Total') - expect_no_header('X-Total-Pages') - expect_header('X-Per-Page', '2') - expect_header('X-Page', '1') - expect_header('X-Next-Page', '2') - expect_header('X-Prev-Page', '') - - expect_header('Link', anything) do |_key, val| - expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first")) - expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="next")) - expect(val).not_to include('rel="last"') - expect(val).not_to include('rel="prev"') - end - - subject.paginate(resource) - end - end - end - it 'does not return the total headers when excluding them' do expect_no_header('X-Total') expect_no_header('X-Total-Pages') diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb index 0c8396cc90b..76731bb916c 100644 --- a/spec/lib/gitlab/redis/multi_store_spec.rb +++ b/spec/lib/gitlab/redis/multi_store_spec.rb @@ -478,9 +478,7 @@ RSpec.describe Gitlab::Redis::MultiStore do let_it_be(:key) { "redis:counter" } - subject do - multi_store.incr(key) - end + subject { multi_store.incr(key) } it 'executes method missing' do expect(multi_store).to receive(:method_missing) @@ -488,17 +486,35 @@ RSpec.describe Gitlab::Redis::MultiStore do subject end - it 'logs MethodMissingError' do - expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(Gitlab::Redis::MultiStore::MethodMissingError), - hash_including(command_name: :incr, extra: hash_including(instance_name: instance_name))) + context 'when command is not in SKIP_LOG_METHOD_MISSING_FOR_COMMANDS' do + it 'logs MethodMissingError' do + expect(Gitlab::ErrorTracking).to receive(:log_exception).with(an_instance_of(Gitlab::Redis::MultiStore::MethodMissingError), + hash_including(command_name: :incr, extra: hash_including(instance_name: instance_name))) - subject + subject + end + + it 'increments method missing counter' do + expect(counter).to receive(:increment).with(command: :incr, instance_name: instance_name) + + subject + end end - it 'increments method missing counter' do - expect(counter).to receive(:increment).with(command: :incr, instance_name: instance_name) + context 'when command is in SKIP_LOG_METHOD_MISSING_FOR_COMMANDS' do + subject { multi_store.info } - subject + it 'does not log MethodMissingError' do + expect(Gitlab::ErrorTracking).not_to receive(:log_exception) + + subject + end + + it 'does not increment method missing counter' do + expect(counter).not_to receive(:increment) + + subject + end end context 'with feature flag :use_primary_store_as_default_for_test_store is enabled' do diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 0baf75f890f..84b72c54f01 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -2056,4 +2056,8 @@ RSpec.describe Namespace do it { is_expected.to be(true) } end end + + it_behaves_like 'it has loose foreign keys' do + let(:factory_name) { :group } + end end diff --git a/spec/models/postgresql/replication_slot_spec.rb b/spec/models/postgresql/replication_slot_spec.rb index c3b67a2e7b8..63a19541ab5 100644 --- a/spec/models/postgresql/replication_slot_spec.rb +++ b/spec/models/postgresql/replication_slot_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Postgresql::ReplicationSlot do + it { is_expected.to be_a Gitlab::Database::SharedModel } + describe '.in_use?' do it 'returns true when replication slots are present' do expect(described_class).to receive(:exists?).and_return(true) @@ -73,28 +75,22 @@ RSpec.describe Postgresql::ReplicationSlot do before(:all) do skip('max_replication_slots too small') if skip_examples - @current_slot_count = ApplicationRecord + @current_slot_count = described_class .connection - .execute("SELECT COUNT(*) FROM pg_replication_slots;") - .first - .fetch('count') - .to_i + .select_value("SELECT COUNT(*) FROM pg_replication_slots") - @current_unused_count = ApplicationRecord + @current_unused_count = described_class .connection - .execute("SELECT COUNT(*) FROM pg_replication_slots WHERE active = 'f';") - .first - .fetch('count') - .to_i + .select_value("SELECT COUNT(*) FROM pg_replication_slots WHERE active = 'f';") - ApplicationRecord + described_class .connection .execute("SELECT * FROM pg_create_physical_replication_slot('test_slot');") end after(:all) do unless skip_examples - ApplicationRecord + described_class .connection .execute("SELECT pg_drop_replication_slot('test_slot');") end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index a612a30369d..20fec9c21a2 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -7453,6 +7453,12 @@ RSpec.describe Project, factory_default: :keep do end end + it_behaves_like 'it has loose foreign keys' do + let(:factory_name) { :project } + end + + private + def finish_job(export_job) export_job.start export_job.finish diff --git a/spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb b/spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb index 1f2b2937dc5..8c98254e134 100644 --- a/spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb +++ b/spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb @@ -4,6 +4,8 @@ RSpec.shared_examples 'it has loose foreign keys' do let(:factory_name) { nil } let(:table_name) { described_class.table_name } let(:connection) { described_class.connection } + let(:fully_qualified_table_name) { "#{connection.current_schema}.#{table_name}" } + let(:deleted_records) { LooseForeignKeys::DeletedRecord.where(fully_qualified_table_name: fully_qualified_table_name) } it 'has at least one loose foreign key definition' do definitions = Gitlab::Database::LooseForeignKeys.definitions_by_table[table_name] @@ -29,7 +31,7 @@ RSpec.shared_examples 'it has loose foreign keys' do # using delete to avoid cross-database modification errors when associations with dependent option are present model.delete - deleted_record = LooseForeignKeys::DeletedRecord.find_by(fully_qualified_table_name: "#{connection.current_schema}.#{table_name}", primary_key_value: model.id) + deleted_record = deleted_records.find_by(primary_key_value: model.id) expect(deleted_record).not_to be_nil end @@ -37,11 +39,11 @@ RSpec.shared_examples 'it has loose foreign keys' do it 'cleans up record deletions' do model = create(factory_name) # rubocop: disable Rails/SaveBang - expect { model.delete }.to change { LooseForeignKeys::DeletedRecord.count }.by(1) + expect { model.delete }.to change { deleted_records.count }.by(1) LooseForeignKeys::ProcessDeletedRecordsService.new(connection: connection).execute - expect(LooseForeignKeys::DeletedRecord.status_pending.count).to be(0) - expect(LooseForeignKeys::DeletedRecord.status_processed.count).to be(1) + expect(deleted_records.status_pending.count).to be(0) + expect(deleted_records.status_processed.count).to be(1) end end diff --git a/spec/support/shared_examples/workers/background_migration_worker_shared_examples.rb b/spec/support/shared_examples/workers/background_migration_worker_shared_examples.rb new file mode 100644 index 00000000000..ffde2e11c1d --- /dev/null +++ b/spec/support/shared_examples/workers/background_migration_worker_shared_examples.rb @@ -0,0 +1,212 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'it runs background migration jobs' do |tracking_database, metric_name| + describe 'defining the job attributes' do + it 'defines the data_consistency as always' do + expect(described_class.get_data_consistency).to eq(:always) + end + + it 'defines the retry count in sidekiq_options' do + expect(described_class.sidekiq_options['retry']).to eq(3) + end + + it 'defines the feature_category as database' do + expect(described_class.get_feature_category).to eq(:database) + end + + it 'defines the urgency as throttled' do + expect(described_class.get_urgency).to eq(:throttled) + end + + it 'defines the loggable_arguments' do + expect(described_class.loggable_arguments).to match_array([0, 1]) + end + end + + describe '.tracking_database' do + it 'does not raise an error' do + expect { described_class.tracking_database }.not_to raise_error + end + + it 'overrides the method to return the tracking database' do + expect(described_class.tracking_database).to eq(tracking_database) + end + end + + describe '.unhealthy_metric_name' do + it 'does not raise an error' do + expect { described_class.unhealthy_metric_name }.not_to raise_error + end + + it 'overrides the method to return the unhealthy metric name' do + expect(described_class.unhealthy_metric_name).to eq(metric_name) + end + end + + describe '.minimum_interval' do + it 'returns 2 minutes' do + expect(described_class.minimum_interval).to eq(2.minutes.to_i) + end + end + + describe '#perform' do + let(:worker) { described_class.new } + + before do + allow(worker).to receive(:jid).and_return(1) + allow(worker).to receive(:always_perform?).and_return(false) + + allow(Postgresql::ReplicationSlot).to receive(:lag_too_great?).and_return(false) + end + + it 'performs jobs using the coordinator for the correct database' do + expect_next_instance_of(Gitlab::BackgroundMigration::JobCoordinator) do |coordinator| + allow(coordinator).to receive(:with_shared_connection).and_yield + + expect(coordinator.database).to eq(tracking_database) + expect(coordinator).to receive(:perform).with('Foo', [10, 20]) + end + + worker.perform('Foo', [10, 20]) + end + + context 'when lease can be obtained' do + let(:coordinator) { double('job coordinator') } + + before do + allow(Gitlab::BackgroundMigration).to receive(:coordinator_for_database) + .with(tracking_database) + .and_return(coordinator) + + allow(coordinator).to receive(:with_shared_connection).and_yield + end + + it 'sets up the shared connection before checking replication' do + expect(coordinator).to receive(:with_shared_connection).and_yield.ordered + expect(Postgresql::ReplicationSlot).to receive(:lag_too_great?).and_return(false).ordered + + expect(coordinator).to receive(:perform).with('Foo', [10, 20]) + + worker.perform('Foo', [10, 20]) + end + + it 'performs a background migration' do + expect(coordinator).to receive(:perform).with('Foo', [10, 20]) + + worker.perform('Foo', [10, 20]) + end + + context 'when lease_attempts is 1' do + it 'performs a background migration' do + expect(coordinator).to receive(:perform).with('Foo', [10, 20]) + + worker.perform('Foo', [10, 20], 1) + end + end + + it 'can run scheduled job and retried job concurrently' do + expect(coordinator) + .to receive(:perform) + .with('Foo', [10, 20]) + .exactly(2).time + + worker.perform('Foo', [10, 20]) + worker.perform('Foo', [10, 20], described_class::MAX_LEASE_ATTEMPTS - 1) + end + + it 'sets the class that will be executed as the caller_id' do + expect(coordinator).to receive(:perform) do + expect(Gitlab::ApplicationContext.current).to include('meta.caller_id' => 'Foo') + end + + worker.perform('Foo', [10, 20]) + end + end + + context 'when lease not obtained (migration of same class was performed recently)' do + let(:timeout) { described_class.minimum_interval } + let(:lease_key) { "#{described_class.name}:Foo" } + let(:coordinator) { double('job coordinator') } + + before do + allow(Gitlab::BackgroundMigration).to receive(:coordinator_for_database) + .with(tracking_database) + .and_return(coordinator) + + allow(coordinator).to receive(:with_shared_connection).and_yield + + expect(coordinator).not_to receive(:perform) + + Gitlab::ExclusiveLease.new(lease_key, timeout: timeout).try_obtain + end + + it 'reschedules the migration and decrements the lease_attempts' do + expect(described_class) + .to receive(:perform_in) + .with(a_kind_of(Numeric), 'Foo', [10, 20], 4) + + worker.perform('Foo', [10, 20], 5) + end + + context 'when lease_attempts is 1' do + let(:lease_key) { "#{described_class.name}:Foo:retried" } + + it 'reschedules the migration and decrements the lease_attempts' do + expect(described_class) + .to receive(:perform_in) + .with(a_kind_of(Numeric), 'Foo', [10, 20], 0) + + worker.perform('Foo', [10, 20], 1) + end + end + + context 'when lease_attempts is 0' do + let(:lease_key) { "#{described_class.name}:Foo:retried" } + + it 'gives up performing the migration' do + expect(described_class).not_to receive(:perform_in) + expect(Sidekiq.logger).to receive(:warn).with( + class: 'Foo', + message: 'Job could not get an exclusive lease after several tries. Giving up.', + job_id: 1) + + worker.perform('Foo', [10, 20], 0) + end + end + end + + context 'when database is not healthy' do + before do + expect(Postgresql::ReplicationSlot).to receive(:lag_too_great?).and_return(true) + end + + it 'reschedules a migration if the database is not healthy' do + expect(described_class) + .to receive(:perform_in) + .with(a_kind_of(Numeric), 'Foo', [10, 20], 4) + + worker.perform('Foo', [10, 20]) + end + + it 'increments the unhealthy counter' do + counter = Gitlab::Metrics.counter(metric_name, 'msg') + + expect(described_class).to receive(:perform_in) + + expect { worker.perform('Foo', [10, 20]) }.to change { counter.get }.by(1) + end + + context 'when lease_attempts is 0' do + it 'gives up performing the migration' do + expect(described_class).not_to receive(:perform_in) + expect(Sidekiq.logger).to receive(:warn).with( + class: 'Foo', + message: 'Database was unhealthy after several tries. Giving up.', + job_id: 1) + + worker.perform('Foo', [10, 20], 0) + end + end + end + end +end diff --git a/spec/workers/background_migration_worker_spec.rb b/spec/workers/background_migration_worker_spec.rb index 7892eb89e80..06513861c0e 100644 --- a/spec/workers/background_migration_worker_spec.rb +++ b/spec/workers/background_migration_worker_spec.rb @@ -3,148 +3,5 @@ require 'spec_helper' RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_shared_state do - let(:worker) { described_class.new } - - describe '.minimum_interval' do - it 'returns 2 minutes' do - expect(described_class.minimum_interval).to eq(2.minutes.to_i) - end - end - - describe '#perform' do - before do - allow(worker).to receive(:jid).and_return(1) - allow(worker).to receive(:always_perform?).and_return(false) - end - - it 'can run scheduled job and retried job concurrently' do - expect(Gitlab::BackgroundMigration) - .to receive(:perform) - .with('Foo', [10, 20]) - .exactly(2).time - - worker.perform('Foo', [10, 20]) - worker.perform('Foo', [10, 20], described_class::MAX_LEASE_ATTEMPTS - 1) - end - - context 'when lease can be obtained' do - before do - expect(Gitlab::BackgroundMigration) - .to receive(:perform) - .with('Foo', [10, 20]) - end - - it 'performs a background migration' do - worker.perform('Foo', [10, 20]) - end - - context 'when lease_attempts is 1' do - it 'performs a background migration' do - worker.perform('Foo', [10, 20], 1) - end - end - end - - context 'when lease not obtained (migration of same class was performed recently)' do - before do - expect(Gitlab::BackgroundMigration).not_to receive(:perform) - - worker.lease_for('Foo', false).try_obtain - end - - it 'reschedules the migration and decrements the lease_attempts' do - expect(described_class) - .to receive(:perform_in) - .with(a_kind_of(Numeric), 'Foo', [10, 20], 4) - - worker.perform('Foo', [10, 20], 5) - end - - context 'when lease_attempts is 1' do - before do - worker.lease_for('Foo', true).try_obtain - end - - it 'reschedules the migration and decrements the lease_attempts' do - expect(described_class) - .to receive(:perform_in) - .with(a_kind_of(Numeric), 'Foo', [10, 20], 0) - - worker.perform('Foo', [10, 20], 1) - end - end - - context 'when lease_attempts is 0' do - before do - worker.lease_for('Foo', true).try_obtain - end - - it 'gives up performing the migration' do - expect(described_class).not_to receive(:perform_in) - expect(Sidekiq.logger).to receive(:warn).with( - class: 'Foo', - message: 'Job could not get an exclusive lease after several tries. Giving up.', - job_id: 1) - - worker.perform('Foo', [10, 20], 0) - end - end - end - - context 'when database is not healthy' do - before do - allow(worker).to receive(:healthy_database?).and_return(false) - end - - it 'reschedules a migration if the database is not healthy' do - expect(described_class) - .to receive(:perform_in) - .with(a_kind_of(Numeric), 'Foo', [10, 20], 4) - - worker.perform('Foo', [10, 20]) - end - - context 'when lease_attempts is 0' do - it 'gives up performing the migration' do - expect(described_class).not_to receive(:perform_in) - expect(Sidekiq.logger).to receive(:warn).with( - class: 'Foo', - message: 'Database was unhealthy after several tries. Giving up.', - job_id: 1) - - worker.perform('Foo', [10, 20], 0) - end - end - end - - it 'sets the class that will be executed as the caller_id' do - expect(Gitlab::BackgroundMigration).to receive(:perform) do - expect(Gitlab::ApplicationContext.current).to include('meta.caller_id' => 'Foo') - end - - worker.perform('Foo', [10, 20]) - end - end - - describe '#healthy_database?' do - context 'when replication lag is too great' do - it 'returns false' do - allow(Postgresql::ReplicationSlot) - .to receive(:lag_too_great?) - .and_return(true) - - expect(worker.healthy_database?).to eq(false) - end - - context 'when replication lag is small enough' do - it 'returns true' do - allow(Postgresql::ReplicationSlot) - .to receive(:lag_too_great?) - .and_return(false) - - expect(worker.healthy_database?).to eq(true) - end - end - end - end + it_behaves_like 'it runs background migration jobs', :main, :background_migration_database_health_reschedules end diff --git a/yarn.lock b/yarn.lock index 30f67083915..1a8292086cf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1491,10 +1491,10 @@ dom-accessibility-api "^0.5.1" pretty-format "^26.4.2" -"@tiptap/core@^2.0.0-beta.142": - version "2.0.0-beta.142" - resolved "https://registry.yarnpkg.com/@tiptap/core/-/core-2.0.0-beta.142.tgz#44e5dc9f63719669fefad422ddae671e8cfe37c4" - integrity sha512-0oAkc2a4ZKI+1yfqAM9EN5Rk9KwJyosLEQutKK43c8ufdUawfIeEJEIG99rsm2AfORSk4e2eyMwkUwUoBht7gw== +"@tiptap/core@^2.0.0-beta.143": + version "2.0.0-beta.143" + resolved "https://registry.yarnpkg.com/@tiptap/core/-/core-2.0.0-beta.143.tgz#e1eb470351867eb13a8f2169958f2231138a6dc1" + integrity sha512-3AoKMZmuc+Lh/ZsM7dj+mSQvRMFANgLVAT0Fza9DaEMREnbpS6nDZlpcbb0HnoExbd7ZLobuwDGxEIZG5uo3Lw== dependencies: "@types/prosemirror-commands" "^1.0.4" "@types/prosemirror-keymap" "^1.0.4" @@ -1685,10 +1685,10 @@ prosemirror-tables "^1.1.1" prosemirror-view "^1.23.1" -"@tiptap/extension-task-item@^2.0.0-beta.28": - version "2.0.0-beta.28" - resolved "https://registry.yarnpkg.com/@tiptap/extension-task-item/-/extension-task-item-2.0.0-beta.28.tgz#7c514ae94b4ab386f7406297e7fa0bbd5e9b5ddd" - integrity sha512-BucJLBiKUupdUF7k5Gqe73HKPgJddZTISOy+3hERt9XPGwa1iCzjwMB41X9+c8HniBDgSyI2aV0zKTT931xFfg== +"@tiptap/extension-task-item@^2.0.0-beta.29": + version "2.0.0-beta.29" + resolved "https://registry.yarnpkg.com/@tiptap/extension-task-item/-/extension-task-item-2.0.0-beta.29.tgz#789bd2012fd9e89623c29a347da02c9cd6e34906" + integrity sha512-6pkY2NoLMJQxfaJsK8XY5Lv/fx0LVJBJ/J15RE8glUEzkDipNhHTBnHWcfzsa4Qzi6UcSy7NJKtT8ixiyqkRmw== "@tiptap/extension-task-list@^2.0.0-beta.23": version "2.0.0-beta.23" @@ -9884,10 +9884,10 @@ prosemirror-transform@^1.0.0, prosemirror-transform@^1.1.0, prosemirror-transfor dependencies: prosemirror-model "^1.0.0" -prosemirror-view@^1.0.0, prosemirror-view@^1.1.0, prosemirror-view@^1.13.3, prosemirror-view@^1.16.5, prosemirror-view@^1.23.1, prosemirror-view@^1.23.2: - version "1.23.2" - resolved "https://registry.yarnpkg.com/prosemirror-view/-/prosemirror-view-1.23.2.tgz#20606ab3faad8a6a5320182256e92a2b96a87d31" - integrity sha512-iPgRw6tpcN+KH1yKmSnRmDKsJBVkWLFP6laHcz9rh/n0Ndz7YKKCDldtw6FhHBYoWmZeubbhV/rrQW0VCDG9iw== +prosemirror-view@^1.0.0, prosemirror-view@^1.1.0, prosemirror-view@^1.13.3, prosemirror-view@^1.16.5, prosemirror-view@^1.23.1, prosemirror-view@^1.23.3: + version "1.23.3" + resolved "https://registry.yarnpkg.com/prosemirror-view/-/prosemirror-view-1.23.3.tgz#9ba85fefaf45e813c46562b694fc5f6f9a5cba9c" + integrity sha512-89icyMdXXwxmTxYj0TIuG5M/d0iKeu79tr+PVtC/4qtCOoHrPSPrblJcFOuOWcxGlA/Ei8PqJB4g5HkKR8jWvQ== dependencies: prosemirror-model "^1.14.3" prosemirror-state "^1.0.0" |