diff options
Diffstat (limited to 'lib/gitlab/background_migration')
24 files changed, 303 insertions, 96 deletions
diff --git a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb new file mode 100644 index 00000000000..2ee0594d0a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `vulnerability_reads.casted_cluster_agent_id` column + class BackfillClusterAgentsHasVulnerabilities < Gitlab::BackgroundMigration::BatchedMigrationJob + VULNERABILITY_READS_JOIN = <<~SQL + INNER JOIN vulnerability_reads + ON vulnerability_reads.casted_cluster_agent_id = cluster_agents.id AND + vulnerability_reads.project_id = cluster_agents.project_id AND + vulnerability_reads.report_type = 7 + SQL + + RELATION = ->(relation) do + relation + .where(has_vulnerabilities: false) + end + + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: RELATION + ) do |sub_batch| + sub_batch + .joins(VULNERABILITY_READS_JOIN) + .update_all(has_vulnerabilities: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb index b9151343d6a..2d64b7378be 100644 --- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb @@ -9,6 +9,7 @@ module Gitlab # Migration only version of MergeRequest table class MergeRequest < ::ApplicationRecord include EachBatch + validates :suggested_reviewers, json_schema: { filename: 'merge_request_suggested_reviewers' } CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb index 814f5a897a9..ce4c4a28b37 100644 --- a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -22,7 +22,7 @@ module Gitlab ProjectFeature.connection.execute( <<~SQL UPDATE project_features pf - SET package_registry_access_level = (CASE p.packages_enabled + SET package_registry_access_level = (CASE p.packages_enabled WHEN true THEN (CASE p.visibility_level WHEN #{PROJECT_PUBLIC} THEN #{FEATURE_PUBLIC} WHEN #{PROJECT_INTERNAL} THEN #{FEATURE_ENABLED} diff --git a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb new file mode 100644 index 00000000000..815c346bb39 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fills the `issues.namespace_id` by setting it to corresponding project.project_namespace_id + class BackfillProjectNamespaceOnIssues < BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: -> (relation) { + relation.joins("INNER JOIN projects ON projects.id = issues.project_id") + .select("issues.id AS issue_id, projects.project_namespace_id").where(issues: { namespace_id: nil }) + } + ) do |sub_batch| + connection.execute <<~SQL + UPDATE issues + SET namespace_id = projects.project_namespace_id + FROM (#{sub_batch.to_sql}) AS projects(issue_id, project_namespace_id) + WHERE issues.id = issue_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index 05e2ed72fb3..c49ef9d10f5 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -212,8 +212,8 @@ module Gitlab def build_attributes_for_project(project) { project_id: project.id, - shard_id: find_shard_id(project.repository_storage), - disk_path: project.disk_path + shard_id: find_shard_id(project.repository_storage), + disk_path: project.disk_path } end diff --git a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb index 728b60f7a0e..0c41d6af209 100644 --- a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb +++ b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb @@ -10,16 +10,12 @@ module Gitlab vulnerability_reads.project_id = cluster_agents.project_id SQL - RELATION = ->(relation) do - relation - .where(report_type: 7) - end + CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 + + scope_to ->(relation) { relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) } def perform - each_sub_batch( - operation_name: :update_all, - batching_scope: RELATION - ) do |sub_batch| + each_sub_batch(operation_name: :update_all) do |sub_batch| sub_batch .joins(CLUSTER_AGENTS_JOIN) .update_all('casted_cluster_agent_id = CAST(vulnerability_reads.cluster_agent_id AS bigint)') diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index 32962f2bb89..86d53ad798d 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -4,11 +4,9 @@ module Gitlab module BackgroundMigration # Backfills the `issues.work_item_type_id` column, replacing any # instances of `NULL` with the appropriate `work_item_types.id` based on `issues.issue_type` - class BackfillWorkItemTypeIdForIssues + class BackfillWorkItemTypeIdForIssues < BatchedMigrationJob # Basic AR model for issues table class MigrationIssue < ApplicationRecord - include ::EachBatch - self.table_name = 'issues' scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) } @@ -16,29 +14,27 @@ module Gitlab MAX_UPDATE_RETRIES = 3 - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, base_type, base_type_id) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id, base_type) + scope_to ->(relation) { + relation.where(issue_type: base_type) + } + + job_arguments :base_type, :base_type_id - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: -> (relation) { relation.where(work_item_type_id: nil) } + ) do |sub_batch| first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) # The query need to be reconstructed because .each_batch modifies the default scope # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 reconstructed_sub_batch = MigrationIssue.unscoped.base_query(base_type).where(id: first..last) - batch_metrics.time_operation(:update_all) do - update_with_retry(reconstructed_sub_batch, base_type_id) - end - - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) + update_with_retry(reconstructed_sub_batch, base_type_id) end end - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - private # Retry mechanism required as update statements on the issues table will randomly take longer than @@ -64,10 +60,6 @@ module Gitlab def update_batch(sub_batch, base_type_id) sub_batch.update_all(work_item_type_id: base_type_id) end - - def relation_scoped_to_range(source_table, source_key_column, start_id, end_id, base_type) - MigrationIssue.where(source_key_column => start_id..end_id).base_query(base_type) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb deleted file mode 100644 index 7d5fef67c25..00000000000 --- a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module BatchingStrategies - # Batching class to use for back-filling issue's work_item_type_id for a single issue type. - # Batches will be scoped to records where the foreign key is NULL and only of a given issue type - # - # If no more batches exist in the table, returns nil. - class BackfillIssueWorkItemTypeBatchingStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments:, job_class: nil) - issue_type = job_arguments.first - - relation.where(issue_type: issue_type) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb index 9ad119310f7..72da2b5a2b7 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb @@ -3,18 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for back-filling project_statistic's container_registry_size. - # Batches will be scoped to records where the project_ids are migrated - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class BackfillProjectStatisticsWithContainerRegistrySizeBatchingStrategy < PrimaryKeyBatchingStrategy - MIGRATION_PHASE_1_ENDED_AT = Date.new(2022, 01, 23).freeze - - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where(created_at: MIGRATION_PHASE_1_ENDED_AT..).or( - relation.where(migration_state: 'import_done') - ).select(:project_id).distinct - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb index f0d015198dc..c2fa00f66de 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb @@ -3,16 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for back-filling vulnerability_read's casted_cluster_agent_id from cluster_agent_id. - # Batches will be scoped to records where the report_type belongs to cluster_image_scanning. - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class BackfillVulnerabilityReadsClusterAgentBatchingStrategy < PrimaryKeyBatchingStrategy - CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 - - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb index e1855b6cfee..9504d4eec11 100644 --- a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb @@ -3,14 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for setting state in vulnerabilitites table. - # Batches will be scoped to records where the dismissed_at is set. - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class DismissedVulnerabilitiesStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where.not(dismissed_at: nil) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index 1ffa4a052e5..43352b1bf91 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -22,8 +22,8 @@ module Gitlab def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) model_class = define_batchable_model(table_name, connection: connection) - quoted_column_name = model_class.connection.quote_column_name(column_name) - relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + arel_column = model_class.arel_table[column_name] + relation = model_class.where(arel_column.gteq(batch_min_value)) if job_class relation = filter_batch(relation, @@ -32,11 +32,10 @@ module Gitlab ) end - relation = apply_additional_filters(relation, job_arguments: job_arguments, job_class: job_class) next_batch_bounds = nil relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop - next_batch_bounds = batch.pick(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")) + next_batch_bounds = batch.pick(arel_column.minimum, arel_column.maximum) break end @@ -44,15 +43,6 @@ module Gitlab next_batch_bounds end - # Deprecated - # - # Use `scope_to` to define additional filters on the migration job class. - # - # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#adding-additional-filters. - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation - end - private def filter_batch(relation, table_name:, column_name:, job_class:, job_arguments: []) diff --git a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb new file mode 100644 index 00000000000..49525479637 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96478 + class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb new file mode 100644 index 00000000000..739197898d9 --- /dev/null +++ b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete approval rules + # as this feature exists only in EE + class DeleteApprovalRulesWithVulnerability < BatchedMigrationJob + def perform + end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::DeleteApprovalRulesWithVulnerability.prepend_mod_with('Gitlab::BackgroundMigration::DeleteApprovalRulesWithVulnerability') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/destroy_invalid_group_members.rb b/lib/gitlab/background_migration/destroy_invalid_group_members.rb new file mode 100644 index 00000000000..35ac42f76ab --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_group_members.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidGroupMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) do + relation.where(source_type: 'Namespace') + .joins('LEFT OUTER JOIN namespaces ON members.source_id = namespaces.id') + .where(namespaces: { id: nil }) + end + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + invalid_ids = sub_batch.map(&:id) + Gitlab::AppLogger.info({ message: 'Removing invalid group member records', + deleted_count: invalid_ids.size, ids: invalid_ids }) + + sub_batch.delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/destroy_invalid_project_members.rb b/lib/gitlab/background_migration/destroy_invalid_project_members.rb new file mode 100644 index 00000000000..3c60f765c29 --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_project_members.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidProjectMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) { relation.where(source_type: 'Project') } + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + invalid_project_members = sub_batch + .joins('LEFT OUTER JOIN projects ON members.source_id = projects.id') + .where(projects: { id: nil }) + invalid_ids = invalid_project_members.pluck(:id) + + # the actual delete + deleted_count = invalid_project_members.delete_all + + Gitlab::AppLogger.info({ message: 'Removing invalid project member records', + deleted_count: deleted_count, + ids: invalid_ids }) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb new file mode 100644 index 00000000000..824054b31f2 --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for public projects created after 17/02/2022 + class DisableLegacyOpenSourceLicenceForRecentPublicProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob + PUBLIC = 20 + THRESHOLD_DATE = '2022-02-17 09:00:00' + + # Migration only version of `project_settings` table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch( + operation_name: :disable_legacy_open_source_licence_for_recent_public_projects, + batching_scope: ->(relation) { + relation.where(visibility_level: PUBLIC).where('created_at >= ?', THRESHOLD_DATE) + } + ) do |sub_batch| + ProjectSetting.where(project_id: sub_batch) + .where(legacy_open_source_license_available: true) + .update_all(legacy_open_source_license_available: false) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb new file mode 100644 index 00000000000..6e4d5d8ddcb --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for projects less than 1 MB + class DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) { relation.where(legacy_open_source_license_available: true) } + + def perform + each_sub_batch(operation_name: :disable_legacy_open_source_license_for_projects_less_than_one_mb) do |sub_batch| + updates = { legacy_open_source_license_available: false, updated_at: Time.current } + + sub_batch + .joins('INNER JOIN project_statistics ON project_statistics.project_id = project_settings.project_id') + .where('project_statistics.repository_size < ?', 1.megabyte) + .update_all(updates) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb index 3605b157f4f..2bf631c6c7d 100644 --- a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb +++ b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb @@ -11,7 +11,7 @@ module Gitlab @user = user @verification_from_mail = Gitlab.config.gitlab.email_from - mail( + mail_with_locale( template_path: 'unconfirm_mailer', template_name: 'unconfirm_notification_email', to: @user.notification_email_or_default, diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 72380af2c53..9a42d035285 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -58,7 +58,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # r development: "a143e9e2-41b3-47bc-9a19-081d089229f4", test: "a143e9e2-41b3-47bc-9a19-081d089229f4", staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", - production: "58dc0f06-936c-43b3-93bb-71693f1b6570" + production: "58dc0f06-936c-43b3-93bb-71693f1b6570" }.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb new file mode 100644 index 00000000000..d30263976e8 --- /dev/null +++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723. + # These job artifacts will not be deleted and will have their `expire_at` removed. + class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob + # The migration would have backfilled `expire_at` + # to midnight on the 22nd of the month of the local timezone, + # storing it as UTC time in the database. + # + # If the timezone setting has changed since the migration, + # the `expire_at` stored in the database could have changed to a different local time other than midnight. + # For example: + # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00. + # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00 + # on the previous day (21st). + # + # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark + # on the dates 21, 22, 23 of the month will not be deleted. + # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets + EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL + EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23) + AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45) + AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0 + SQL + + scope_to ->(relation) { + relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE) + .or(relation.where(file_type: 3)) + } + + def perform + each_sub_batch( + operation_name: :update_all + ) do |sub_batch| + sub_batch.update_all(expire_at: nil) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb new file mode 100644 index 00000000000..5b1d630bb03 --- /dev/null +++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Removes obsolete wiki notes + class RemoveSelfManagedWikiNotes < BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :delete_all + ) do |sub_batch| + sub_batch.where(noteable_type: 'Wiki').delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb new file mode 100644 index 00000000000..718fb0aaa71 --- /dev/null +++ b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Renames all system notes created when an issuable task is checked/unchecked + # from `task` into `checklist item` + # `marked the task **Task 1** as incomplete` => `marked the checklist item **Task 1** as incomplete` + class RenameTaskSystemNoteToChecklistItem < BatchedMigrationJob + REPLACE_REGEX = '\Amarked\sthe\stask' + TEXT_REPLACEMENT = 'marked the checklist item' + + scope_to ->(relation) { + relation.where(system_note_metadata: { action: :task }) + } + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + ApplicationRecord.connection.execute <<~SQL + UPDATE notes + SET note = REGEXP_REPLACE(notes.note,'#{REPLACE_REGEX}', '#{TEXT_REPLACEMENT}') + FROM (#{sub_batch.select(:note_id).to_sql}) AS metadata_fields(note_id) + WHERE notes.id = note_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb index fd6cbcb8d05..a0cfeed618a 100644 --- a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -6,11 +6,10 @@ module Gitlab class SetCorrectVulnerabilityState < BatchedMigrationJob DISMISSED_STATE = 2 + scope_to ->(relation) { relation.where.not(dismissed_at: nil) } + def perform - each_sub_batch( - operation_name: :update_vulnerabilities_state, - batching_scope: -> (relation) { relation.where.not(dismissed_at: nil) } - ) do |sub_batch| + each_sub_batch(operation_name: :update_vulnerabilities_state) do |sub_batch| sub_batch.update_all(state: DISMISSED_STATE) end end |