diff options
Diffstat (limited to 'db/post_migrate')
77 files changed, 1080 insertions, 58 deletions
diff --git a/db/post_migrate/20210302150310_schedule_migrate_pages_to_zip_storage.rb b/db/post_migrate/20210302150310_schedule_migrate_pages_to_zip_storage.rb index 7f6d7ffe9b7..87c7d373329 100644 --- a/db/post_migrate/20210302150310_schedule_migrate_pages_to_zip_storage.rb +++ b/db/post_migrate/20210302150310_schedule_migrate_pages_to_zip_storage.rb @@ -11,26 +11,7 @@ class ScheduleMigratePagesToZipStorage < ActiveRecord::Migration[6.0] disable_ddl_transaction! - class ProjectPagesMetadatum < ActiveRecord::Base - extend SuppressCompositePrimaryKeyWarning - - include EachBatch - - self.primary_key = :project_id - self.table_name = 'project_pages_metadata' - self.inheritance_column = :_type_disabled - - scope :deployed, -> { where(deployed: true) } - scope :only_on_legacy_storage, -> { deployed.where(pages_deployment_id: nil) } - end - def up - queue_background_migration_jobs_by_range_at_intervals( - ProjectPagesMetadatum.only_on_legacy_storage, - MIGRATION, - BATCH_TIME, - batch_size: BATCH_SIZE, - primary_column_name: :project_id - ) + # no-op end end diff --git a/db/post_migrate/20210311120155_backfill_events_id_for_bigint_conversion.rb b/db/post_migrate/20210311120155_backfill_events_id_for_bigint_conversion.rb index 37546a793f7..b9427f7cc93 100644 --- a/db/post_migrate/20210311120155_backfill_events_id_for_bigint_conversion.rb +++ b/db/post_migrate/20210311120155_backfill_events_id_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class BackfillEventsIdForBigintConversion < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + DOWNTIME = false def up diff --git a/db/post_migrate/20210311120156_backfill_push_event_payload_event_id_for_bigint_conversion.rb b/db/post_migrate/20210311120156_backfill_push_event_payload_event_id_for_bigint_conversion.rb index 87250c82244..bb444f5b407 100644 --- a/db/post_migrate/20210311120156_backfill_push_event_payload_event_id_for_bigint_conversion.rb +++ b/db/post_migrate/20210311120156_backfill_push_event_payload_event_id_for_bigint_conversion.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class BackfillPushEventPayloadEventIdForBigintConversion < ActiveRecord::Migration[6.0] + disable_ddl_transaction! + include Gitlab::Database::MigrationHelpers DOWNTIME = false diff --git a/db/post_migrate/20210415101228_backfill_ci_build_needs_for_bigint_conversion.rb b/db/post_migrate/20210415101228_backfill_ci_build_needs_for_bigint_conversion.rb index 3e4f1450632..1ee67cd9dda 100644 --- a/db/post_migrate/20210415101228_backfill_ci_build_needs_for_bigint_conversion.rb +++ b/db/post_migrate/20210415101228_backfill_ci_build_needs_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class BackfillCiBuildNeedsForBigintConversion < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + DOWNTIME = false def up diff --git a/db/post_migrate/20210420121149_backfill_conversion_of_ci_job_artifacts.rb b/db/post_migrate/20210420121149_backfill_conversion_of_ci_job_artifacts.rb index b6b37acab3b..67076cc647a 100644 --- a/db/post_migrate/20210420121149_backfill_conversion_of_ci_job_artifacts.rb +++ b/db/post_migrate/20210420121149_backfill_conversion_of_ci_job_artifacts.rb @@ -5,6 +5,8 @@ class BackfillConversionOfCiJobArtifacts < ActiveRecord::Migration[6.0] DOWNTIME = false + disable_ddl_transaction! + def up return unless should_run? diff --git a/db/post_migrate/20210422023046_backfill_ci_sources_pipelines_source_job_id_for_bigint_conversion.rb b/db/post_migrate/20210422023046_backfill_ci_sources_pipelines_source_job_id_for_bigint_conversion.rb index 0aea924c982..bde91473ee3 100644 --- a/db/post_migrate/20210422023046_backfill_ci_sources_pipelines_source_job_id_for_bigint_conversion.rb +++ b/db/post_migrate/20210422023046_backfill_ci_sources_pipelines_source_job_id_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class BackfillCiSourcesPipelinesSourceJobIdForBigintConversion < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + def up return unless should_run? diff --git a/db/post_migrate/20210426094549_backfill_ci_builds_for_bigint_conversion.rb b/db/post_migrate/20210426094549_backfill_ci_builds_for_bigint_conversion.rb index 972df41f678..fd071ec9a1a 100644 --- a/db/post_migrate/20210426094549_backfill_ci_builds_for_bigint_conversion.rb +++ b/db/post_migrate/20210426094549_backfill_ci_builds_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class BackfillCiBuildsForBigintConversion < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + DOWNTIME = false TABLE = :ci_builds diff --git a/db/post_migrate/20210427045711_backfill_ci_build_trace_chunks_for_bigint_conversion.rb b/db/post_migrate/20210427045711_backfill_ci_build_trace_chunks_for_bigint_conversion.rb index 4c656f56a32..7e11d38868b 100644 --- a/db/post_migrate/20210427045711_backfill_ci_build_trace_chunks_for_bigint_conversion.rb +++ b/db/post_migrate/20210427045711_backfill_ci_build_trace_chunks_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class BackfillCiBuildTraceChunksForBigintConversion < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + TABLE = :ci_build_trace_chunks COLUMNS = %i(build_id) diff --git a/db/post_migrate/20210427085020_backfill_ci_builds_runner_session_for_bigint_conversion.rb b/db/post_migrate/20210427085020_backfill_ci_builds_runner_session_for_bigint_conversion.rb index b548309cdb7..fa0420fe426 100644 --- a/db/post_migrate/20210427085020_backfill_ci_builds_runner_session_for_bigint_conversion.rb +++ b/db/post_migrate/20210427085020_backfill_ci_builds_runner_session_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class BackfillCiBuildsRunnerSessionForBigintConversion < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + TABLE = :ci_builds_runner_session COLUMNS = %i(build_id) diff --git a/db/post_migrate/20210430121542_backfill_ci_build_trace_sections_for_bigint_conversion.rb b/db/post_migrate/20210430121542_backfill_ci_build_trace_sections_for_bigint_conversion.rb index 979ce5edfeb..340dceda254 100644 --- a/db/post_migrate/20210430121542_backfill_ci_build_trace_sections_for_bigint_conversion.rb +++ b/db/post_migrate/20210430121542_backfill_ci_build_trace_sections_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class BackfillCiBuildTraceSectionsForBigintConversion < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + TABLE = :ci_build_trace_sections COLUMN = :build_id diff --git a/db/post_migrate/20210519132129_backfill_ci_builds_metadata_for_bigint_conversion.rb b/db/post_migrate/20210519132129_backfill_ci_builds_metadata_for_bigint_conversion.rb index 3aeabbcc0ad..39dd78975da 100644 --- a/db/post_migrate/20210519132129_backfill_ci_builds_metadata_for_bigint_conversion.rb +++ b/db/post_migrate/20210519132129_backfill_ci_builds_metadata_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class BackfillCiBuildsMetadataForBigintConversion < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + TABLE = :ci_builds_metadata COLUMN = :build_id diff --git a/db/post_migrate/20210520012430_backfill_pk_conversion_for_self_managed.rb b/db/post_migrate/20210520012430_backfill_pk_conversion_for_self_managed.rb index d554b412420..68bc82059bc 100644 --- a/db/post_migrate/20210520012430_backfill_pk_conversion_for_self_managed.rb +++ b/db/post_migrate/20210520012430_backfill_pk_conversion_for_self_managed.rb @@ -3,6 +3,8 @@ class BackfillPkConversionForSelfManaged < ActiveRecord::Migration[6.0] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + CONVERSIONS = [ { table: :events, columns: %i(id), sub_batch_size: 500 }, { table: :push_event_payloads, columns: %i(event_id), sub_batch_size: 2500, primary_key: :event_id }, diff --git a/db/post_migrate/20210705143150_backfill_ci_builds_metadata_id_for_bigint_conversion.rb b/db/post_migrate/20210705143150_backfill_ci_builds_metadata_id_for_bigint_conversion.rb index ecf9040eb46..a9f6d4ea2d9 100644 --- a/db/post_migrate/20210705143150_backfill_ci_builds_metadata_id_for_bigint_conversion.rb +++ b/db/post_migrate/20210705143150_backfill_ci_builds_metadata_id_for_bigint_conversion.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class BackfillCiBuildsMetadataIdForBigintConversion < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + include Gitlab::Database::MigrationHelpers TABLE = :ci_builds_metadata diff --git a/db/post_migrate/20210722110515_revert_backfill_ci_build_trace_sections_for_bigint_conversion.rb b/db/post_migrate/20210722110515_revert_backfill_ci_build_trace_sections_for_bigint_conversion.rb index 840540bb0d4..78ae4f23286 100644 --- a/db/post_migrate/20210722110515_revert_backfill_ci_build_trace_sections_for_bigint_conversion.rb +++ b/db/post_migrate/20210722110515_revert_backfill_ci_build_trace_sections_for_bigint_conversion.rb @@ -3,6 +3,8 @@ class RevertBackfillCiBuildTraceSectionsForBigintConversion < ActiveRecord::Migration[6.1] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + TABLE = :ci_build_trace_sections COLUMN = :build_id diff --git a/db/post_migrate/20210727113447_backfill_integrations_type_new.rb b/db/post_migrate/20210727113447_backfill_integrations_type_new.rb index 8544c236fd7..47238bae4d5 100644 --- a/db/post_migrate/20210727113447_backfill_integrations_type_new.rb +++ b/db/post_migrate/20210727113447_backfill_integrations_type_new.rb @@ -3,6 +3,8 @@ class BackfillIntegrationsTypeNew < ActiveRecord::Migration[6.1] include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + MIGRATION = 'BackfillIntegrationsTypeNew' INTERVAL = 2.minutes @@ -16,8 +18,6 @@ class BackfillIntegrationsTypeNew < ActiveRecord::Migration[6.1] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :integrations, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :integrations, :id, []) end end diff --git a/db/post_migrate/20210930211936_backfill_user_namespace.rb b/db/post_migrate/20210930211936_backfill_user_namespace.rb index c5dd170f18e..6d71b09ad63 100644 --- a/db/post_migrate/20210930211936_backfill_user_namespace.rb +++ b/db/post_migrate/20210930211936_backfill_user_namespace.rb @@ -7,6 +7,8 @@ class BackfillUserNamespace < Gitlab::Database::Migration[1.0] SUB_BATCH_SIZE = 200 DOWNTIME = false + disable_ddl_transaction! + def up queue_batched_background_migration( MIGRATION, @@ -19,8 +21,6 @@ class BackfillUserNamespace < Gitlab::Database::Migration[1.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :namespaces, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :namespaces, :id, []) end end diff --git a/db/post_migrate/20211026070408_backfill_issue_search_data.rb b/db/post_migrate/20211026070408_backfill_issue_search_data.rb index a840adcb991..a2cf7b36a59 100644 --- a/db/post_migrate/20211026070408_backfill_issue_search_data.rb +++ b/db/post_migrate/20211026070408_backfill_issue_search_data.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class BackfillIssueSearchData < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + MIGRATION = 'BackfillIssueSearchData' def up @@ -15,8 +17,6 @@ class BackfillIssueSearchData < Gitlab::Database::Migration[1.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :issues, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :issues, :id, []) end end diff --git a/db/post_migrate/20211209103048_backfill_project_namespaces_for_group.rb b/db/post_migrate/20211209103048_backfill_project_namespaces_for_group.rb index 0b65db7aab4..44dffc798d3 100644 --- a/db/post_migrate/20211209103048_backfill_project_namespaces_for_group.rb +++ b/db/post_migrate/20211209103048_backfill_project_namespaces_for_group.rb @@ -36,7 +36,6 @@ class BackfillProjectNamespacesForGroup < Gitlab::Database::Migration[1.0] def down return unless Gitlab.com? || Gitlab.staging? - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :projects, :id, [GROUP_ID, 'up']).delete_all + delete_batched_background_migration(MIGRATION, :projects, :id, [GROUP_ID, 'up']) end end diff --git a/db/post_migrate/20220120123800_backfill_namespace_id_for_namespace_routes.rb b/db/post_migrate/20220120123800_backfill_namespace_id_for_namespace_routes.rb index 4aac0903502..68279456ddd 100644 --- a/db/post_migrate/20220120123800_backfill_namespace_id_for_namespace_routes.rb +++ b/db/post_migrate/20220120123800_backfill_namespace_id_for_namespace_routes.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class BackfillNamespaceIdForNamespaceRoutes < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + MIGRATION = 'BackfillNamespaceIdForNamespaceRoute' INTERVAL = 2.minutes BATCH_SIZE = 1_000 @@ -20,8 +22,6 @@ class BackfillNamespaceIdForNamespaceRoutes < Gitlab::Database::Migration[1.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :routes, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :routes, :id, []) end end diff --git a/db/post_migrate/20220120211832_backfill_member_namespace_id_for_group_members.rb b/db/post_migrate/20220120211832_backfill_member_namespace_id_for_group_members.rb index 947c0a1edd0..279084e0fc5 100644 --- a/db/post_migrate/20220120211832_backfill_member_namespace_id_for_group_members.rb +++ b/db/post_migrate/20220120211832_backfill_member_namespace_id_for_group_members.rb @@ -7,6 +7,8 @@ class BackfillMemberNamespaceIdForGroupMembers < Gitlab::Database::Migration[1.0 MAX_BATCH_SIZE = 2_000 SUB_BATCH_SIZE = 100 + disable_ddl_transaction! + def up queue_batched_background_migration( MIGRATION, @@ -20,8 +22,6 @@ class BackfillMemberNamespaceIdForGroupMembers < Gitlab::Database::Migration[1.0 end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :members, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :members, :id, []) end end diff --git a/db/post_migrate/20220128103042_schedule_delete_invalid_epic_issues_revised.rb b/db/post_migrate/20220128103042_schedule_delete_invalid_epic_issues_revised.rb new file mode 100644 index 00000000000..642bf012ce0 --- /dev/null +++ b/db/post_migrate/20220128103042_schedule_delete_invalid_epic_issues_revised.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ScheduleDeleteInvalidEpicIssuesRevised < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + MIGRATION = 'DeleteInvalidEpicIssues' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 2_000 + SUB_BATCH_SIZE = 50 + + def up + queue_batched_background_migration( + MIGRATION, + :epics, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :epics, :id, []) + end +end diff --git a/db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb b/db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb index e4005885c3b..10b6ab8204d 100644 --- a/db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb +++ b/db/post_migrate/20220223112304_schedule_nullify_orphan_runner_id_on_ci_builds.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ScheduleNullifyOrphanRunnerIdOnCiBuilds < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + MIGRATION = 'NullifyOrphanRunnerIdOnCiBuilds' INTERVAL = 2.minutes BATCH_SIZE = 50_000 @@ -20,8 +22,6 @@ class ScheduleNullifyOrphanRunnerIdOnCiBuilds < Gitlab::Database::Migration[1.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :ci_builds, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :ci_builds, :id, []) end end diff --git a/db/post_migrate/20220301093434_backfill_all_project_namespaces.rb b/db/post_migrate/20220301093434_backfill_all_project_namespaces.rb index 7071e6241ce..607161004cf 100644 --- a/db/post_migrate/20220301093434_backfill_all_project_namespaces.rb +++ b/db/post_migrate/20220301093434_backfill_all_project_namespaces.rb @@ -24,7 +24,6 @@ class BackfillAllProjectNamespaces < Gitlab::Database::Migration[1.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :projects, :id, [nil, 'up']).delete_all + delete_batched_background_migration(MIGRATION, :projects, :id, [nil, 'up']) end end diff --git a/db/post_migrate/20220302114046_backfill_group_features.rb b/db/post_migrate/20220302114046_backfill_group_features.rb index dd8961b8bee..5e0ff9095a3 100644 --- a/db/post_migrate/20220302114046_backfill_group_features.rb +++ b/db/post_migrate/20220302114046_backfill_group_features.rb @@ -22,8 +22,6 @@ class BackfillGroupFeatures < Gitlab::Database::Migration[1.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :namespaces, :id, [BATCH_SIZE]) - .delete_all + delete_batched_background_migration(MIGRATION, :namespaces, :id, [BATCH_SIZE]) end end diff --git a/db/post_migrate/20220322071127_finalize_project_namespaces_backfill.rb b/db/post_migrate/20220322071127_finalize_project_namespaces_backfill.rb index 5a4d14dcfb7..21b8dc78f7d 100644 --- a/db/post_migrate/20220322071127_finalize_project_namespaces_backfill.rb +++ b/db/post_migrate/20220322071127_finalize_project_namespaces_backfill.rb @@ -4,6 +4,8 @@ # so that other migrations that depend on project namespace back-filling cannot be run unless project namespace # back-filling has finalized successfully. class FinalizeProjectNamespacesBackfill < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + MIGRATION = 'ProjectNamespaces::BackfillProjectNamespaces' def up diff --git a/db/post_migrate/20220323023800_backfill_namespace_id_for_project_routes.rb b/db/post_migrate/20220323023800_backfill_namespace_id_for_project_routes.rb index b938688be2c..0b2c7318887 100644 --- a/db/post_migrate/20220323023800_backfill_namespace_id_for_project_routes.rb +++ b/db/post_migrate/20220323023800_backfill_namespace_id_for_project_routes.rb @@ -22,8 +22,6 @@ class BackfillNamespaceIdForProjectRoutes < Gitlab::Database::Migration[1.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :routes, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :routes, :id, []) end end diff --git a/db/post_migrate/20220324165436_schedule_backfill_project_settings.rb b/db/post_migrate/20220324165436_schedule_backfill_project_settings.rb index c1aaea44bfd..07a7b8b8989 100644 --- a/db/post_migrate/20220324165436_schedule_backfill_project_settings.rb +++ b/db/post_migrate/20220324165436_schedule_backfill_project_settings.rb @@ -6,6 +6,8 @@ class ScheduleBackfillProjectSettings < Gitlab::Database::Migration[1.0] BATCH_SIZE = 5_000 SUB_BATCH_SIZE = 200 + disable_ddl_transaction! + def up queue_batched_background_migration( MIGRATION, @@ -18,8 +20,6 @@ class ScheduleBackfillProjectSettings < Gitlab::Database::Migration[1.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :projects, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :projects, :id, []) end end diff --git a/db/post_migrate/20220407163559_schedule_purging_stale_security_scans.rb b/db/post_migrate/20220407163559_schedule_purging_stale_security_scans.rb new file mode 100644 index 00000000000..fdceb2f2594 --- /dev/null +++ b/db/post_migrate/20220407163559_schedule_purging_stale_security_scans.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class SchedulePurgingStaleSecurityScans < Gitlab::Database::Migration[2.0] + MIGRATION = 'PurgeStaleSecurityScans' + BATCH_SIZE = 10_000 + DELAY_INTERVAL = 2.minutes + + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + def up + return unless should_run? + + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::PurgeStaleSecurityScans::SecurityScan.to_purge, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # no-op + end + + private + + def should_run? + Gitlab.dev_or_test_env? || Gitlab.com? + end +end diff --git a/db/post_migrate/20220418180958_remove_integrations_properties.rb b/db/post_migrate/20220418180958_remove_integrations_properties.rb new file mode 100644 index 00000000000..781b8c8ac72 --- /dev/null +++ b/db/post_migrate/20220418180958_remove_integrations_properties.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveIntegrationsProperties < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def change + remove_column :integrations, :properties, :text + end +end diff --git a/db/post_migrate/20220502015011_clean_up_fix_merge_request_diff_commit_users.rb b/db/post_migrate/20220502015011_clean_up_fix_merge_request_diff_commit_users.rb index 582a0f5cf37..a3e59b38975 100644 --- a/db/post_migrate/20220502015011_clean_up_fix_merge_request_diff_commit_users.rb +++ b/db/post_migrate/20220502015011_clean_up_fix_merge_request_diff_commit_users.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class CleanUpFixMergeRequestDiffCommitUsers < Gitlab::Database::Migration[1.0] +class CleanUpFixMergeRequestDiffCommitUsers < Gitlab::Database::Migration[2.0] disable_ddl_transaction! MIGRATION_CLASS = 'FixMergeRequestDiffCommitUsers' diff --git a/db/post_migrate/20220506123922_add_not_null_constraint_without_validation_to_requirements_issue_id.rb b/db/post_migrate/20220506123922_add_not_null_constraint_without_validation_to_requirements_issue_id.rb new file mode 100644 index 00000000000..1484a96d27b --- /dev/null +++ b/db/post_migrate/20220506123922_add_not_null_constraint_without_validation_to_requirements_issue_id.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddNotNullConstraintWithoutValidationToRequirementsIssueId < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + CONSTRAINT_NAME = 'check_requirement_issue_not_null' + + def up + add_not_null_constraint( + :requirements, + :issue_id, + constraint_name: CONSTRAINT_NAME, + validate: false + ) + end + + def down + remove_not_null_constraint :requirements, :issue_id, constraint_name: CONSTRAINT_NAME + end +end diff --git a/db/post_migrate/20220506124021_schedule_populate_requirements_issue_id.rb b/db/post_migrate/20220506124021_schedule_populate_requirements_issue_id.rb new file mode 100644 index 00000000000..8f22324bf8d --- /dev/null +++ b/db/post_migrate/20220506124021_schedule_populate_requirements_issue_id.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class SchedulePopulateRequirementsIssueId < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_main + + # 2022-05-06 There are no requirements with nil issue_id on .com + # this migration is supposed to fix records that could have nil issue_id + # on self managed instances. + BATCH_SIZE = 100 + + MIGRATION = 'MigrateRequirementsToWorkItems' + + disable_ddl_transaction! + + class Requirement < MigrationRecord + include EachBatch + + self.table_name = 'requirements' + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + Requirement.where(issue_id: nil), + MIGRATION, + 2.minutes, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # NO OP + end +end diff --git a/db/post_migrate/20220510121338_remove_threat_monitoring_alerts.rb b/db/post_migrate/20220510121338_remove_threat_monitoring_alerts.rb new file mode 100644 index 00000000000..ff171ee516c --- /dev/null +++ b/db/post_migrate/20220510121338_remove_threat_monitoring_alerts.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveThreatMonitoringAlerts < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + execute("DELETE FROM alert_management_alerts WHERE domain = 1") + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220510192117_index_expirable_unknown_artifacts_for_removal.rb b/db/post_migrate/20220510192117_index_expirable_unknown_artifacts_for_removal.rb new file mode 100644 index 00000000000..3e25ca6c7b9 --- /dev/null +++ b/db/post_migrate/20220510192117_index_expirable_unknown_artifacts_for_removal.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class IndexExpirableUnknownArtifactsForRemoval < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + TABLE_NAME = 'ci_job_artifacts' + INDEX_NAME = 'tmp_index_ci_job_artifacts_on_expire_at_where_locked_unknown' + CONDITIONS = 'locked = 2 AND expire_at IS NOT NULL' + + def up + prepare_async_index TABLE_NAME, [:expire_at, :job_id], name: INDEX_NAME, where: CONDITIONS + end + + def down + unprepare_async_index_by_name TABLE_NAME, INDEX_NAME + end +end diff --git a/db/post_migrate/20220511212620_add_index_on_expirable_unknown_artifacts_for_removal.rb b/db/post_migrate/20220511212620_add_index_on_expirable_unknown_artifacts_for_removal.rb new file mode 100644 index 00000000000..fc9912d714e --- /dev/null +++ b/db/post_migrate/20220511212620_add_index_on_expirable_unknown_artifacts_for_removal.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexOnExpirableUnknownArtifactsForRemoval < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + TABLE_NAME = 'ci_job_artifacts' + INDEX_NAME = 'tmp_index_ci_job_artifacts_on_expire_at_where_locked_unknown' + CONDITIONS = 'locked = 2 AND expire_at IS NOT NULL' + + def up + add_concurrent_index TABLE_NAME, [:expire_at, :job_id], name: INDEX_NAME, where: CONDITIONS + end + + def down + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end +end diff --git a/db/post_migrate/20220512190659_remove_web_hooks_web_hook_logs_web_hook_id_fk.rb b/db/post_migrate/20220512190659_remove_web_hooks_web_hook_logs_web_hook_id_fk.rb new file mode 100644 index 00000000000..b73d3a7f102 --- /dev/null +++ b/db/post_migrate/20220512190659_remove_web_hooks_web_hook_logs_web_hook_id_fk.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +class RemoveWebHooksWebHookLogsWebHookIdFk < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + PARENT_TABLE_NAME = :web_hook_logs + FK_NAME = "fk_rails_bb3355782d" + + def up + with_lock_retries do + execute('LOCK web_hooks, web_hook_logs IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:web_hook_logs, :web_hooks, name: FK_NAME) + end + end + + def down + fk_attrs = { + name: FK_NAME, # Note we need the same name for every partition + column: :web_hook_id, + target_column: :id, + on_delete: :cascade + } + + # Must add child FK's first, then to the partitioned table. + child_tables.each do |tbl| + add_concurrent_foreign_key( + tbl, :web_hooks, + # This embeds the lock table statement in the with_lock_retries inside add_concurrent_foreign_key + reverse_lock_order: true, + **fk_attrs) + end + + with_lock_retries do + execute("LOCK web_hooks, #{PARENT_TABLE_NAME} IN ACCESS EXCLUSIVE MODE") if transaction_open? + add_foreign_key(:web_hook_logs, :web_hooks, **fk_attrs) + end + end + + # This table is partitioned: we need to apply the index changes to each + # partition separately. + def child_tables + @child_tables ||= execute(<<~SQL.squish).pluck("child") + SELECT inhrelid::regclass AS child + FROM pg_catalog.pg_inherits + WHERE inhparent = '#{PARENT_TABLE_NAME}'::regclass + ORDER BY inhrelid ASC + SQL + end +end diff --git a/db/post_migrate/20220513043344_reschedule_expire_o_auth_tokens.rb b/db/post_migrate/20220513043344_reschedule_expire_o_auth_tokens.rb index 0c8a2386583..4b24b1cf65a 100644 --- a/db/post_migrate/20220513043344_reschedule_expire_o_auth_tokens.rb +++ b/db/post_migrate/20220513043344_reschedule_expire_o_auth_tokens.rb @@ -10,9 +10,7 @@ class RescheduleExpireOAuthTokens < Gitlab::Database::Migration[2.0] def up # remove the original migration from db/post_migrate/20220428133724_schedule_expire_o_auth_tokens.rb - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :oauth_access_tokens, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :oauth_access_tokens, :id, []) # reschedule queue_batched_background_migration( @@ -24,8 +22,6 @@ class RescheduleExpireOAuthTokens < Gitlab::Database::Migration[2.0] end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :oauth_access_tokens, :id, []) - .delete_all + delete_batched_background_migration(MIGRATION, :oauth_access_tokens, :id, []) end end diff --git a/db/post_migrate/20220516054011_schedule_backfill_project_member_namespace_id.rb b/db/post_migrate/20220516054011_schedule_backfill_project_member_namespace_id.rb new file mode 100644 index 00000000000..e721fdf8c44 --- /dev/null +++ b/db/post_migrate/20220516054011_schedule_backfill_project_member_namespace_id.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class ScheduleBackfillProjectMemberNamespaceId < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = 'BackfillProjectMemberNamespaceId' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 2_000 + SUB_BATCH_SIZE = 200 + + disable_ddl_transaction! + + def up + queue_batched_background_migration( + MIGRATION, + :members, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :members, :id, []) + end +end diff --git a/db/post_migrate/20220516100652_drop_compliance_management_frameworks_regulated.rb b/db/post_migrate/20220516100652_drop_compliance_management_frameworks_regulated.rb new file mode 100644 index 00000000000..f9d3405d486 --- /dev/null +++ b/db/post_migrate/20220516100652_drop_compliance_management_frameworks_regulated.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class DropComplianceManagementFrameworksRegulated < Gitlab::Database::Migration[2.0] + def up + remove_column :compliance_management_frameworks, :regulated + end + + def down + add_column :compliance_management_frameworks, :regulated, :boolean, default: true + end +end diff --git a/db/post_migrate/20220516103353_finalize_routes_backfilling_for_projects.rb b/db/post_migrate/20220516103353_finalize_routes_backfilling_for_projects.rb new file mode 100644 index 00000000000..f65003b5da9 --- /dev/null +++ b/db/post_migrate/20220516103353_finalize_routes_backfilling_for_projects.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class FinalizeRoutesBackfillingForProjects < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = 'BackfillNamespaceIdForProjectRoute' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: :routes, + column_name: :id, + job_arguments: [] + ) + end + + def down + # noop + end +end diff --git a/db/post_migrate/20220517101119_create_confidential_notes_index_synchronously.rb b/db/post_migrate/20220517101119_create_confidential_notes_index_synchronously.rb new file mode 100644 index 00000000000..25b7fe7f7bb --- /dev/null +++ b/db/post_migrate/20220517101119_create_confidential_notes_index_synchronously.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CreateConfidentialNotesIndexSynchronously < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_notes_on_confidential' + + disable_ddl_transaction! + + def up + add_concurrent_index :notes, :confidential, where: 'confidential = true', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :notes, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220517133753_migrate_free_user_cap_remediation.rb b/db/post_migrate/20220517133753_migrate_free_user_cap_remediation.rb new file mode 100644 index 00000000000..44f30134c94 --- /dev/null +++ b/db/post_migrate/20220517133753_migrate_free_user_cap_remediation.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class MigrateFreeUserCapRemediation < Gitlab::Database::Migration[2.0] + def up + sidekiq_queue_migrate 'cronjob:namespaces_free_user_cap', to: 'cronjob:namespaces_free_user_cap_remediation' + end + + def down + sidekiq_queue_migrate 'cronjob:namespaces_free_user_cap_remediation', to: 'cronjob:namespaces_free_user_cap' + end +end diff --git a/db/post_migrate/20220518102414_add_index_to_error_tracking_client_keys.rb b/db/post_migrate/20220518102414_add_index_to_error_tracking_client_keys.rb new file mode 100644 index 00000000000..25923bf00f6 --- /dev/null +++ b/db/post_migrate/20220518102414_add_index_to_error_tracking_client_keys.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddIndexToErrorTrackingClientKeys < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'index_error_tracking_client_for_enabled_check' + + def up + add_concurrent_index( + :error_tracking_client_keys, + [:project_id, :public_key], + where: 'active = true', + name: INDEX_NAME + ) + end + + def down + remove_concurrent_index_by_name(:error_tracking_client_keys, INDEX_NAME) + end +end diff --git a/db/post_migrate/20220519045133_bulk_insert_cluster_enabled_grants.rb b/db/post_migrate/20220519045133_bulk_insert_cluster_enabled_grants.rb new file mode 100644 index 00000000000..6c1d9058673 --- /dev/null +++ b/db/post_migrate/20220519045133_bulk_insert_cluster_enabled_grants.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class BulkInsertClusterEnabledGrants < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_main + + disable_ddl_transaction! + + def up + return unless Gitlab.dev_or_test_env? || Gitlab.com? + + define_batchable_model('cluster_groups').each_batch do |batch| + min, max = batch.pick('MIN(id), MAX(id)') + + bulk_insert = <<-SQL + INSERT INTO cluster_enabled_grants (namespace_id, created_at) + SELECT DISTINCT(traversal_ids[1]), NOW() + FROM cluster_groups + INNER JOIN namespaces ON cluster_groups.group_id = namespaces.id + WHERE cluster_groups.id BETWEEN #{min} AND #{max} + ON CONFLICT (namespace_id) DO NOTHING + SQL + + connection.execute(bulk_insert) + end + + define_batchable_model('cluster_projects').each_batch do |batch| + min, max = batch.pick('MIN(id), MAX(id)') + + bulk_insert = <<-SQL + INSERT INTO cluster_enabled_grants (namespace_id, created_at) + SELECT DISTINCT(traversal_ids[1]), NOW() + FROM cluster_projects + INNER JOIN projects ON cluster_projects.project_id = projects.id + INNER JOIN namespaces on projects.namespace_id = namespaces.id + WHERE cluster_projects.id BETWEEN #{min} AND #{max} + ON CONFLICT (namespace_id) DO NOTHING + SQL + + connection.execute(bulk_insert) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220520040416_schedule_set_legacy_open_source_license_available_for_non_public_projects.rb b/db/post_migrate/20220520040416_schedule_set_legacy_open_source_license_available_for_non_public_projects.rb new file mode 100644 index 00000000000..47b1c169d74 --- /dev/null +++ b/db/post_migrate/20220520040416_schedule_set_legacy_open_source_license_available_for_non_public_projects.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class ScheduleSetLegacyOpenSourceLicenseAvailableForNonPublicProjects < Gitlab::Database::Migration[2.0] + MIGRATION = 'SetLegacyOpenSourceLicenseAvailableForNonPublicProjects' + INTERVAL = 2.minutes + BATCH_SIZE = 5_000 + SUB_BATCH_SIZE = 200 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + return unless Gitlab.com? + + queue_batched_background_migration( + MIGRATION, + :projects, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + return unless Gitlab.com? + + delete_batched_background_migration(MIGRATION, :projects, :id, []) + end +end diff --git a/db/post_migrate/20220523171107_drop_deploy_tokens_token_column.rb b/db/post_migrate/20220523171107_drop_deploy_tokens_token_column.rb new file mode 100644 index 00000000000..79a145bc466 --- /dev/null +++ b/db/post_migrate/20220523171107_drop_deploy_tokens_token_column.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class DropDeployTokensTokenColumn < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + COMPOSITE_INDEX_NAME = 'index_deploy_tokens_on_token_and_expires_at_and_id' + + def up + remove_column :deploy_tokens, :token + end + + def down + unless column_exists?(:deploy_tokens, :token) + add_column :deploy_tokens, :token, :string + end + + add_concurrent_index(:deploy_tokens, :token, unique: true) + add_concurrent_index(:deploy_tokens, %i[token expires_at id], where: 'revoked IS FALSE', name: COMPOSITE_INDEX_NAME) + end +end diff --git a/db/post_migrate/20220524074947_finalize_backfill_null_note_discussion_ids.rb b/db/post_migrate/20220524074947_finalize_backfill_null_note_discussion_ids.rb new file mode 100644 index 00000000000..f11846ebe1d --- /dev/null +++ b/db/post_migrate/20220524074947_finalize_backfill_null_note_discussion_ids.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class FinalizeBackfillNullNoteDiscussionIds < Gitlab::Database::Migration[2.0] + MIGRATION = 'BackfillNoteDiscussionId' + BATCH_SIZE = 10_000 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + disable_ddl_transaction! + + def up + Gitlab::BackgroundMigration.steal(MIGRATION) + + define_batchable_model('notes').where(discussion_id: nil).each_batch(of: BATCH_SIZE) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + Gitlab::BackgroundMigration::BackfillNoteDiscussionId.new.perform(*range) + end + end + + def down + end +end diff --git a/db/post_migrate/20220524080944_cleanup_orphaned_routes.rb b/db/post_migrate/20220524080944_cleanup_orphaned_routes.rb new file mode 100644 index 00000000000..a5ce0ba4646 --- /dev/null +++ b/db/post_migrate/20220524080944_cleanup_orphaned_routes.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class CleanupOrphanedRoutes < Gitlab::Database::Migration[2.0] + MIGRATION = 'CleanupOrphanedRoutes' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 100_000 + MAX_BATCH_SIZE = 100_000 + SUB_BATCH_SIZE = 100 + + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :routes, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main + ) + end + + def down + delete_batched_background_migration(MIGRATION, :routes, :id, []) + end +end diff --git a/db/post_migrate/20220524081955_remove_notes_null_discussion_id_temp_index.rb b/db/post_migrate/20220524081955_remove_notes_null_discussion_id_temp_index.rb new file mode 100644 index 00000000000..caca7b0b092 --- /dev/null +++ b/db/post_migrate/20220524081955_remove_notes_null_discussion_id_temp_index.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveNotesNullDiscussionIdTempIndex < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'tmp_index_notes_on_id_where_discussion_id_is_null' + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :notes, INDEX_NAME + end + + def down + add_concurrent_index :notes, :id, where: 'discussion_id IS NULL', name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220524132416_prepare_job_artifact_project_id_index.rb b/db/post_migrate/20220524132416_prepare_job_artifact_project_id_index.rb new file mode 100644 index 00000000000..4e27e7f0624 --- /dev/null +++ b/db/post_migrate/20220524132416_prepare_job_artifact_project_id_index.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class PrepareJobArtifactProjectIdIndex < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_ci_job_artifacts_on_project_id_and_id' + + def up + prepare_async_index :ci_job_artifacts, [:project_id, :id], name: INDEX_NAME + end + + def down + unprepare_async_index :notes, [:project_id, :id], name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220524202158_drop_index_on_deployments_on_created_at_cluster_id_and_project_id.rb b/db/post_migrate/20220524202158_drop_index_on_deployments_on_created_at_cluster_id_and_project_id.rb new file mode 100644 index 00000000000..91b1e5f8ce3 --- /dev/null +++ b/db/post_migrate/20220524202158_drop_index_on_deployments_on_created_at_cluster_id_and_project_id.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class DropIndexOnDeploymentsOnCreatedAtClusterIdAndProjectId < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'tp_index_created_at_cluster_id_project_id_on_deployments' + + def up + remove_concurrent_index_by_name :deployments, INDEX_NAME + end + + def down + # no-op + # + # There's no need to re-add this index as it's purpose was temporary, served only + # for a specific CR query which is now closed, and should not be re-opened. + end +end diff --git a/db/post_migrate/20220525131557_cleanup_backfill_integrations_enable_ssl_verification.rb b/db/post_migrate/20220525131557_cleanup_backfill_integrations_enable_ssl_verification.rb new file mode 100644 index 00000000000..3bd4b21c6fd --- /dev/null +++ b/db/post_migrate/20220525131557_cleanup_backfill_integrations_enable_ssl_verification.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CleanupBackfillIntegrationsEnableSslVerification < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + MIGRATION = 'BackfillIntegrationsEnableSslVerification' + + def up + finalize_background_migration(MIGRATION) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220525131624_drop_temporary_index_for_backfill_integrations_enable_ssl_verification.rb b/db/post_migrate/20220525131624_drop_temporary_index_for_backfill_integrations_enable_ssl_verification.rb new file mode 100644 index 00000000000..95abac4b7ac --- /dev/null +++ b/db/post_migrate/20220525131624_drop_temporary_index_for_backfill_integrations_enable_ssl_verification.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class DropTemporaryIndexForBackfillIntegrationsEnableSslVerification < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'tmp_index_integrations_on_id_where_type_droneci_or_teamcity' + INDEX_CONDITION = "type_new IN ('Integrations::DroneCi', 'Integrations::Teamcity') " \ + "AND encrypted_properties IS NOT NULL" + + def up + remove_concurrent_index_by_name :integrations, INDEX_NAME + end + + def down + # this index is used in 20220209121435_backfill_integrations_enable_ssl_verification + add_concurrent_index :integrations, :id, where: INDEX_CONDITION, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220525141540_migrate_project_service_worker_queue.rb b/db/post_migrate/20220525141540_migrate_project_service_worker_queue.rb new file mode 100644 index 00000000000..acfb8b227fe --- /dev/null +++ b/db/post_migrate/20220525141540_migrate_project_service_worker_queue.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class MigrateProjectServiceWorkerQueue < Gitlab::Database::Migration[2.0] + def up + sidekiq_queue_migrate 'project_service', to: 'integrations_execute' + end + + def down + sidekiq_queue_migrate 'integrations_execute', to: 'project_service' + end +end diff --git a/db/post_migrate/20220525165334_migrate_irker_worker_queue.rb b/db/post_migrate/20220525165334_migrate_irker_worker_queue.rb new file mode 100644 index 00000000000..19454c007c8 --- /dev/null +++ b/db/post_migrate/20220525165334_migrate_irker_worker_queue.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class MigrateIrkerWorkerQueue < Gitlab::Database::Migration[2.0] + def up + sidekiq_queue_migrate 'irker', to: 'integrations_irker' + end + + def down + sidekiq_queue_migrate 'integrations_irker', to: 'irker' + end +end diff --git a/db/post_migrate/20220525172001_migrate_cluster_integration_worker_queues.rb b/db/post_migrate/20220525172001_migrate_cluster_integration_worker_queues.rb new file mode 100644 index 00000000000..4372ca9f965 --- /dev/null +++ b/db/post_migrate/20220525172001_migrate_cluster_integration_worker_queues.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class MigrateClusterIntegrationWorkerQueues < Gitlab::Database::Migration[2.0] + def up + sidekiq_queue_migrate 'gcp_cluster:clusters_applications_activate_service', + to: 'gcp_cluster:clusters_applications_activate_integration' + sidekiq_queue_migrate 'gcp_cluster:clusters_applications_deactivate_service', + to: 'gcp_cluster:clusters_applications_deactivate_integration' + end + + def down + sidekiq_queue_migrate 'gcp_cluster:clusters_applications_activate_integration', + to: 'gcp_cluster:clusters_applications_activate_service' + sidekiq_queue_migrate 'gcp_cluster:clusters_applications_deactivate_integration', + to: 'gcp_cluster:clusters_applications_deactivate_service' + end +end diff --git a/db/post_migrate/20220526042017_remove_geo_lfs_object_deleted_event_references.rb b/db/post_migrate/20220526042017_remove_geo_lfs_object_deleted_event_references.rb new file mode 100644 index 00000000000..4569424dca0 --- /dev/null +++ b/db/post_migrate/20220526042017_remove_geo_lfs_object_deleted_event_references.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class RemoveGeoLfsObjectDeletedEventReferences < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + with_lock_retries do + remove_column :geo_event_log, :lfs_object_deleted_event_id, :bigint + end + end + + def down + with_lock_retries do + unless column_exists?(:geo_event_log, :lfs_object_deleted_event_id) + add_column(:geo_event_log, :lfs_object_deleted_event_id, :bigint) + end + end + + add_concurrent_foreign_key :geo_event_log, :geo_lfs_object_deleted_events, + column: :lfs_object_deleted_event_id, + name: 'fk_d5af95fcd9', + on_delete: :cascade + + add_concurrent_index :geo_event_log, + :lfs_object_deleted_event_id, + name: 'index_geo_event_log_on_lfs_object_deleted_event_id', + where: "(lfs_object_deleted_event_id IS NOT NULL)" + end +end diff --git a/db/post_migrate/20220526044516_remove_geo_lfs_object_deleted_events_table.rb b/db/post_migrate/20220526044516_remove_geo_lfs_object_deleted_events_table.rb new file mode 100644 index 00000000000..29a2e34aefa --- /dev/null +++ b/db/post_migrate/20220526044516_remove_geo_lfs_object_deleted_events_table.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class RemoveGeoLfsObjectDeletedEventsTable < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + drop_table :geo_lfs_object_deleted_events + end + + def down + create_table :geo_lfs_object_deleted_events, id: :bigserial do |t| + t.integer :lfs_object_id, null: false, index: true + t.string :oid, null: false + t.string :file_path, null: false + end + end +end diff --git a/db/post_migrate/20220530044712_add_index_for_vulnerability_reads_location_image.rb b/db/post_migrate/20220530044712_add_index_for_vulnerability_reads_location_image.rb new file mode 100644 index 00000000000..384924941c1 --- /dev/null +++ b/db/post_migrate/20220530044712_add_index_for_vulnerability_reads_location_image.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AddIndexForVulnerabilityReadsLocationImage < Gitlab::Database::Migration[2.0] + INDEX = :index_vulnerability_reads_on_location_image_partial + + disable_ddl_transaction! + + REPORT_TYPES = { + container_scanning: 2, + cluster_image_scanning: 7 + } + + QUERY = <<~SQL + report_type in (#{REPORT_TYPES[:container_scanning]}, #{REPORT_TYPES[:cluster_image_scanning]}) AND + location_image IS NOT NULL + SQL + + def up + add_concurrent_index :vulnerability_reads, [:project_id, :location_image], + where: QUERY, + name: INDEX + end + + def down + remove_concurrent_index_by_name :vulnerability_reads, INDEX + end +end diff --git a/db/post_migrate/20220530074027_add_project_id_index_to_job_artifacts.rb b/db/post_migrate/20220530074027_add_project_id_index_to_job_artifacts.rb new file mode 100644 index 00000000000..fb75f368835 --- /dev/null +++ b/db/post_migrate/20220530074027_add_project_id_index_to_job_artifacts.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddProjectIdIndexToJobArtifacts < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_ci_job_artifacts_on_project_id_and_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_job_artifacts, [:project_id, :id], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + end +end diff --git a/db/post_migrate/20220530082653_add_traversal_id_type_group_index.rb b/db/post_migrate/20220530082653_add_traversal_id_type_group_index.rb new file mode 100644 index 00000000000..00d997e1f52 --- /dev/null +++ b/db/post_migrate/20220530082653_add_traversal_id_type_group_index.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddTraversalIdTypeGroupIndex < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_namespaces_on_traversal_ids_for_groups_btree' + + disable_ddl_transaction! + + def up + add_concurrent_index :namespaces, :traversal_ids, using: :btree, where: "type='Group'", name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :namespaces, INDEX_NAME + end +end diff --git a/db/post_migrate/20220530170915_add_index_for_vulnerability_state_transition.rb b/db/post_migrate/20220530170915_add_index_for_vulnerability_state_transition.rb new file mode 100644 index 00000000000..46ff79c2441 --- /dev/null +++ b/db/post_migrate/20220530170915_add_index_for_vulnerability_state_transition.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddIndexForVulnerabilityStateTransition < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + NEW_INDEX_NAME = 'index_vulnerability_state_transitions_id_and_vulnerability_id' + OLD_INDEX_NAME = 'index_vulnerability_state_transitions_on_vulnerability_id' + + def up + add_concurrent_index(:vulnerability_state_transitions, [:vulnerability_id, :id], name: NEW_INDEX_NAME) + + remove_concurrent_index_by_name(:vulnerability_state_transitions, OLD_INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(:vulnerability_state_transitions, NEW_INDEX_NAME) + + add_concurrent_index(:vulnerability_state_transitions, [:vulnerability_id], name: OLD_INDEX_NAME) + end +end diff --git a/db/post_migrate/20220531024142_track_clusters_deletions.rb b/db/post_migrate/20220531024142_track_clusters_deletions.rb new file mode 100644 index 00000000000..a9c4e50f7dc --- /dev/null +++ b/db/post_migrate/20220531024142_track_clusters_deletions.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class TrackClustersDeletions < Gitlab::Database::Migration[2.0] + include Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers + + enable_lock_retries! + + def up + track_record_deletions(:clusters) + end + + def down + untrack_record_deletions(:clusters) + end +end diff --git a/db/post_migrate/20220531024143_remove_clusters_deployments_cluster_id_fk.rb b/db/post_migrate/20220531024143_remove_clusters_deployments_cluster_id_fk.rb new file mode 100644 index 00000000000..261a06a9b9c --- /dev/null +++ b/db/post_migrate/20220531024143_remove_clusters_deployments_cluster_id_fk.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class RemoveClustersDeploymentsClusterIdFk < Gitlab::Database::Migration[2.0] + FK_NAME = 'fk_289bba3222' + + disable_ddl_transaction! + + def up + with_lock_retries do + remove_foreign_key_if_exists( + :deployments, + :clusters, + name: FK_NAME, + reverse_lock_order: true + ) + end + end + + def down + add_concurrent_foreign_key( + :deployments, + :clusters, + name: FK_NAME, + column: :cluster_id, + target_column: :id, + on_delete: :nullify + ) + end +end diff --git a/db/post_migrate/20220531233600_remove_sse_usage_data_from_redis.rb b/db/post_migrate/20220531233600_remove_sse_usage_data_from_redis.rb new file mode 100644 index 00000000000..b7b02e483df --- /dev/null +++ b/db/post_migrate/20220531233600_remove_sse_usage_data_from_redis.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveSseUsageDataFromRedis < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + Gitlab::Redis::SharedState.with { |r| r.del("USAGE_STATIC_SITE_EDITOR_VIEWS") } + Gitlab::Redis::SharedState.with { |r| r.del("USAGE_STATIC_SITE_EDITOR_COMMITS") } + Gitlab::Redis::SharedState.with { |r| r.del("USAGE_STATIC_SITE_EDITOR_MERGE_REQUESTS") } + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220601040233_finalize_backfill_issue_search_data.rb b/db/post_migrate/20220601040233_finalize_backfill_issue_search_data.rb new file mode 100644 index 00000000000..7b5cd405fa8 --- /dev/null +++ b/db/post_migrate/20220601040233_finalize_backfill_issue_search_data.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class FinalizeBackfillIssueSearchData < Gitlab::Database::Migration[2.0] + MIGRATION = 'BackfillIssueSearchData' + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: :issues, + column_name: :id, + job_arguments: [], + finalize: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220602070502_queue_backfill_project_feature_package_registry_access_level.rb b/db/post_migrate/20220602070502_queue_backfill_project_feature_package_registry_access_level.rb new file mode 100644 index 00000000000..2c772a79327 --- /dev/null +++ b/db/post_migrate/20220602070502_queue_backfill_project_feature_package_registry_access_level.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class QueueBackfillProjectFeaturePackageRegistryAccessLevel < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + MIGRATION = 'BackfillProjectFeaturePackageRegistryAccessLevel' + DELAY_INTERVAL = 2.minutes + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :projects, + :id, + job_interval: DELAY_INTERVAL + ) + end + + def down + delete_batched_background_migration(MIGRATION, :projects, :id, []) + end +end diff --git a/db/post_migrate/20220602111923_prepare_index_for_open_issues_count.rb b/db/post_migrate/20220602111923_prepare_index_for_open_issues_count.rb new file mode 100644 index 00000000000..e90524df1ba --- /dev/null +++ b/db/post_migrate/20220602111923_prepare_index_for_open_issues_count.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class PrepareIndexForOpenIssuesCount < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'idx_open_issues_on_project_id_and_confidential' + + def up + prepare_async_index :issues, [:project_id, :confidential], where: 'state_id = 1', name: INDEX_NAME + end + + def down + unprepare_async_index_by_name :issues, INDEX_NAME + end +end diff --git a/db/post_migrate/20220606060825_set_on_delete_cascade_on_namespace_id_on_routes_table.rb b/db/post_migrate/20220606060825_set_on_delete_cascade_on_namespace_id_on_routes_table.rb new file mode 100644 index 00000000000..74660d1474c --- /dev/null +++ b/db/post_migrate/20220606060825_set_on_delete_cascade_on_namespace_id_on_routes_table.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class SetOnDeleteCascadeOnNamespaceIdOnRoutesTable < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + TARGET_COLUMN = :namespace_id + + def up + # add the new FK before removing the old one + add_concurrent_foreign_key( + :routes, + :namespaces, + column: TARGET_COLUMN, + name: fk_name("#{TARGET_COLUMN}_new"), + on_delete: :cascade + ) + + with_lock_retries do + remove_foreign_key_if_exists(:routes, column: TARGET_COLUMN, name: fk_name(TARGET_COLUMN)) + end + end + + def down + add_concurrent_foreign_key( + :routes, + :namespaces, + column: TARGET_COLUMN, + name: fk_name(TARGET_COLUMN), + on_delete: :nullify + ) + + with_lock_retries do + remove_foreign_key_if_exists(:routes, column: TARGET_COLUMN, name: fk_name("#{TARGET_COLUMN}_new")) + end + end + + def fk_name(column_name) + # generate a FK name + concurrent_foreign_key_name(:routes, column_name) + end +end diff --git a/db/post_migrate/20220606060850_add_not_null_constraint_on_routes_namespace_id.rb b/db/post_migrate/20220606060850_add_not_null_constraint_on_routes_namespace_id.rb new file mode 100644 index 00000000000..4c9f087f62d --- /dev/null +++ b/db/post_migrate/20220606060850_add_not_null_constraint_on_routes_namespace_id.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddNotNullConstraintOnRoutesNamespaceId < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + add_not_null_constraint :routes, :namespace_id, validate: false + end + + def down + remove_not_null_constraint :routes, :namespace_id + end +end diff --git a/db/post_migrate/20220607140222_remove_invalid_integrations.rb b/db/post_migrate/20220607140222_remove_invalid_integrations.rb new file mode 100644 index 00000000000..49834d7b120 --- /dev/null +++ b/db/post_migrate/20220607140222_remove_invalid_integrations.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class RemoveInvalidIntegrations < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + BATCH_SIZE = 100 + + def up + loop do + deleted = Integration.where(type_new: nil).limit(BATCH_SIZE).delete_all + + break if deleted < BATCH_SIZE + end + end + + # Isolated version of the Integration model + class Integration < MigrationRecord + self.table_name = 'integrations' + self.inheritance_column = :_type_disabled + end +end diff --git a/db/post_migrate/20220609195803_remove_deployment_clusters_cluster_id_fk.rb b/db/post_migrate/20220609195803_remove_deployment_clusters_cluster_id_fk.rb new file mode 100644 index 00000000000..2d81816fcfd --- /dev/null +++ b/db/post_migrate/20220609195803_remove_deployment_clusters_cluster_id_fk.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class RemoveDeploymentClustersClusterIdFk < Gitlab::Database::Migration[2.0] + FK_NAME = 'fk_rails_4e6243e120' + + disable_ddl_transaction! + + def up + with_lock_retries do + remove_foreign_key_if_exists( + :deployment_clusters, + :clusters, + name: FK_NAME, + reverse_lock_order: true + ) + end + end + + def down + add_concurrent_foreign_key( + :deployment_clusters, + :clusters, + name: FK_NAME, + column: :cluster_id, + target_column: :id, + on_delete: :cascade + ) + end +end diff --git a/db/post_migrate/20220610223040_add_index_on_available_pypi_packages.rb b/db/post_migrate/20220610223040_add_index_on_available_pypi_packages.rb new file mode 100644 index 00000000000..891272c8a51 --- /dev/null +++ b/db/post_migrate/20220610223040_add_index_on_available_pypi_packages.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddIndexOnAvailablePypiPackages < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'index_packages_on_available_pypi_packages' + + def up + add_concurrent_index :packages_packages, + [:project_id, :id], + where: "status IN (0,1) AND package_type = 5 AND version IS NOT NULL", + name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :packages_packages, INDEX_NAME + end +end diff --git a/db/post_migrate/20220613095911_create_confidential_notes_index_on_id.rb b/db/post_migrate/20220613095911_create_confidential_notes_index_on_id.rb new file mode 100644 index 00000000000..80c26c3ea8a --- /dev/null +++ b/db/post_migrate/20220613095911_create_confidential_notes_index_on_id.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class CreateConfidentialNotesIndexOnId < Gitlab::Database::Migration[2.0] + OLD_INDEX_NAME = 'index_notes_on_confidential' + INDEX_NAME = 'index_notes_on_id_where_confidential' + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :notes, name: OLD_INDEX_NAME + add_concurrent_index :notes, :id, where: 'confidential = true', name: INDEX_NAME + end + + def down + # we don't have to re-create OLD_INDEX_NAME index + # because it wasn't used yet, also its creation might be expensive + remove_concurrent_index_by_name :notes, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220616092541_prepare_index_issues_on_project_id_and_closed_at.rb b/db/post_migrate/20220616092541_prepare_index_issues_on_project_id_and_closed_at.rb new file mode 100644 index 00000000000..084cdb6166d --- /dev/null +++ b/db/post_migrate/20220616092541_prepare_index_issues_on_project_id_and_closed_at.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class PrepareIndexIssuesOnProjectIdAndClosedAt < Gitlab::Database::Migration[2.0] + NEW_INDEX_NAME_1 = 'index_issues_on_project_id_closed_at_desc_state_id_and_id' + NEW_INDEX_NAME_2 = 'index_issues_on_project_id_closed_at_state_id_and_id' + + def up + # Index to improve performance when sorting issues by closed_at desc + prepare_async_index :issues, 'project_id, closed_at DESC NULLS LAST, state_id, id', name: NEW_INDEX_NAME_1 + + # Index to improve performance when sorting issues by closed_at asc + # This replaces the old index which didn't account for state_id and id + prepare_async_index :issues, [:project_id, :closed_at, :state_id, :id], name: NEW_INDEX_NAME_2 + end + + def down + unprepare_async_index_by_name :issues, NEW_INDEX_NAME_1 + unprepare_async_index_by_name :issues, NEW_INDEX_NAME_2 + end +end diff --git a/db/post_migrate/20220616222253_remove_ci_secure_files_permissions_column.rb b/db/post_migrate/20220616222253_remove_ci_secure_files_permissions_column.rb new file mode 100644 index 00000000000..31c559a9d44 --- /dev/null +++ b/db/post_migrate/20220616222253_remove_ci_secure_files_permissions_column.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class RemoveCiSecureFilesPermissionsColumn < Gitlab::Database::Migration[2.0] + def up + remove_column :ci_secure_files, :permissions + end + + def down + add_column :ci_secure_files, :permissions, :integer, null: false, default: 0, limit: 2 + end +end diff --git a/db/post_migrate/20220617073407_add_index_for_open_issues_count.rb b/db/post_migrate/20220617073407_add_index_for_open_issues_count.rb new file mode 100644 index 00000000000..5c11e7b1b9a --- /dev/null +++ b/db/post_migrate/20220617073407_add_index_for_open_issues_count.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexForOpenIssuesCount < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'idx_open_issues_on_project_id_and_confidential' + + def up + add_concurrent_index :issues, [:project_id, :confidential], where: 'state_id = 1', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :issues, INDEX_NAME + end +end |