diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-20 12:40:42 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-20 12:40:42 +0300 |
commit | ee664acb356f8123f4f6b00b73c1e1cf0866c7fb (patch) | |
tree | f8479f94a28f66654c6a4f6fb99bad6b4e86a40e /lib/gitlab/background_migration | |
parent | 62f7d5c5b69180e82ae8196b7b429eeffc8e7b4f (diff) |
Add latest changes from gitlab-org/gitlab@15-5-stable-eev15.5.0-rc42
Diffstat (limited to 'lib/gitlab/background_migration')
10 files changed, 228 insertions, 0 deletions
diff --git a/lib/gitlab/background_migration/backfill_epic_cache_counts.rb b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb new file mode 100644 index 00000000000..bd61d1a0f07 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +Gitlab::BackgroundMigration::BackfillEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/backfill_internal_on_notes.rb b/lib/gitlab/background_migration/backfill_internal_on_notes.rb new file mode 100644 index 00000000000..300f2cff6ca --- /dev/null +++ b/lib/gitlab/background_migration/backfill_internal_on_notes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This syncs the data to `internal` from `confidential` as we rename the column. + class BackfillInternalOnNotes < BatchedMigrationJob + scope_to -> (relation) { relation.where(confidential: true) } + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + sub_batch.update_all(internal: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb new file mode 100644 index 00000000000..b8a51b576b6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill namespace_details for a range of namespaces + class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + each_sub_batch(operation_name: :backfill_namespace_details) do |sub_batch| + upsert_namespace_details(sub_batch) + end + end + + def upsert_namespace_details(relation) + connection.execute( + <<~SQL + INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id) + SELECT namespaces.description, namespaces.description_html, namespaces.cached_markdown_version, now(), now(), namespaces.id + FROM namespaces + WHERE namespaces.id IN(#{relation.select(:id).to_sql}) + AND namespaces.type <> 'Project' + ON CONFLICT (namespace_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb new file mode 100644 index 00000000000..c3e1019b72f --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting orphaned operational vulnerabilities (without findings) + class DeleteOrphanedOperationalVulnerabilities < ::Gitlab::BackgroundMigration::BatchedMigrationJob + REPORT_TYPES = { + cluster_image_scanning: 7, + custom: 99 + }.freeze + + NOT_EXISTS_SQL = <<-SQL + NOT EXISTS ( + SELECT FROM vulnerability_occurrences + WHERE "vulnerability_occurrences"."vulnerability_id" = "vulnerabilities"."id" + ) + SQL + + scope_to ->(relation) do + relation + .where(report_type: [REPORT_TYPES[:cluster_image_scanning], REPORT_TYPES[:custom]]) + end + + def perform + each_sub_batch(operation_name: :delete_orphaned_operational_vulnerabilities) do |sub_batch| + sub_batch + .where(NOT_EXISTS_SQL) + .delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/destroy_invalid_members.rb b/lib/gitlab/background_migration/destroy_invalid_members.rb new file mode 100644 index 00000000000..7d78795bea9 --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_members.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) { relation.where(member_namespace_id: nil) } + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + deleted_members_data = sub_batch.map do |m| + { id: m.id, source_id: m.source_id, source_type: m.source_type } + end + + deleted_count = sub_batch.delete_all + + Gitlab::AppLogger.info({ message: 'Removing invalid member records', + deleted_count: deleted_count, + deleted_member_data: deleted_members_data }) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb new file mode 100644 index 00000000000..2257dc016be --- /dev/null +++ b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete merge request level rules + # as this feature exists only in EE + class PopulateApprovalMergeRequestRulesWithSecurityOrchestration < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::PopulateApprovalMergeRequestRulesWithSecurityOrchestration.prepend_mod_with('Gitlab::BackgroundMigration::PopulateApprovalMergeRequestRulesWithSecurityOrchestration') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb new file mode 100644 index 00000000000..1d0c0010551 --- /dev/null +++ b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete merge request level rules + # as this feature exists only in EE + class PopulateApprovalProjectRulesWithSecurityOrchestration < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::PopulateApprovalProjectRulesWithSecurityOrchestration.prepend_mod_with('Gitlab::BackgroundMigration::PopulateApprovalProjectRulesWithSecurityOrchestration') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb new file mode 100644 index 00000000000..952f3b0e3c3 --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate token_encrypted values in ci_runners table in batches + class ResetDuplicateCiRunnersTokenEncryptedValues < BatchedMigrationJob + def perform + each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_encrypted_values) do |sub_batch| + # Reset duplicate runner encrypted tokens that would prevent creating an unique index. + nullify_duplicate_ci_runner_token_encrypted_values(sub_batch) + end + end + + private + + def nullify_duplicate_ci_runner_token_encrypted_values(sub_batch) + batchable_model = define_batchable_model(batch_table, connection: connection) + + duplicate_tokens = batchable_model + .where(token_encrypted: sub_batch.select(:token_encrypted).distinct) + .group(:token_encrypted) + .having('COUNT(*) > 1') + .pluck(:token_encrypted) + + return if duplicate_tokens.empty? + + batchable_model.where(token_encrypted: duplicate_tokens).update_all(token_encrypted: nil) + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb new file mode 100644 index 00000000000..cfd6a4e4091 --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate token values in ci_runners table in batches + class ResetDuplicateCiRunnersTokenValues < BatchedMigrationJob + def perform + each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_values) do |sub_batch| + # Reset duplicate runner tokens that would prevent creating an unique index. + nullify_duplicate_ci_runner_token_values(sub_batch) + end + end + + private + + def nullify_duplicate_ci_runner_token_values(sub_batch) + batchable_model = define_batchable_model(batch_table, connection: connection) + + duplicate_tokens = batchable_model + .where(token: sub_batch.select(:token).distinct) + .group(:token) + .having('COUNT(*) > 1') + .pluck(:token) + + batchable_model.where(token: duplicate_tokens).update_all(token: nil) if duplicate_tokens.any? + end + end + end +end diff --git a/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb new file mode 100644 index 00000000000..84183753158 --- /dev/null +++ b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The `ci_pipeline_artifacts.locked` column was added in + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97194 to + # speed up the finding of expired, pipeline artifacts. By default, + # the value is "unknown" (2), but the correct value should be the + # value of the associated `ci_pipelines.locked` value. This class + # does an UPDATE join to make the values match. + class UpdateCiPipelineArtifactsUnknownLockedStatus < BatchedMigrationJob + def perform + connection.exec_query(<<~SQL) + UPDATE ci_pipeline_artifacts + SET locked = ci_pipelines.locked + FROM ci_pipelines + WHERE ci_pipeline_artifacts.id BETWEEN #{start_id} AND #{end_id} + AND ci_pipeline_artifacts.locked = 2 + AND ci_pipelines.id = ci_pipeline_artifacts.pipeline_id; + SQL + end + end + end +end |