diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 12:55:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 12:55:51 +0300 |
commit | e8d2c2579383897a1dd7f9debd359abe8ae8373d (patch) | |
tree | c42be41678c2586d49a75cabce89322082698334 /db/post_migrate | |
parent | fc845b37ec3a90aaa719975f607740c22ba6a113 (diff) |
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to 'db/post_migrate')
23 files changed, 540 insertions, 0 deletions
diff --git a/db/post_migrate/20210602164044_schedule_latest_pipeline_id_population.rb b/db/post_migrate/20210602164044_schedule_latest_pipeline_id_population.rb new file mode 100644 index 00000000000..dfd2806fece --- /dev/null +++ b/db/post_migrate/20210602164044_schedule_latest_pipeline_id_population.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ScheduleLatestPipelineIdPopulation < ActiveRecord::Migration[6.1] + def up + # no-op: This migration has been marked as no-op and replaced by + # `ReScheduleLatestPipelineIdPopulation` as we've found some problems. + # For more information: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65280 + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210604133651_schedule_merge_request_diff_users_background_migration.rb b/db/post_migrate/20210604133651_schedule_merge_request_diff_users_background_migration.rb new file mode 100644 index 00000000000..b9b694012f2 --- /dev/null +++ b/db/post_migrate/20210604133651_schedule_merge_request_diff_users_background_migration.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class ScheduleMergeRequestDiffUsersBackgroundMigration < ActiveRecord::Migration[6.1] + def up + # no-op + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210609202501_schedule_backfill_draft_status_on_merge_requests.rb b/db/post_migrate/20210609202501_schedule_backfill_draft_status_on_merge_requests.rb new file mode 100644 index 00000000000..72c4168af50 --- /dev/null +++ b/db/post_migrate/20210609202501_schedule_backfill_draft_status_on_merge_requests.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class ScheduleBackfillDraftStatusOnMergeRequests < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + INDEX_NAME = "tmp_index_merge_requests_draft_and_status" + MIGRATION = 'BackfillDraftStatusOnMergeRequests' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 100 + + disable_ddl_transaction! + + def up + add_concurrent_index :merge_requests, :id, + where: "draft = false AND state_id = 1 AND ((title)::text ~* '^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP'::text)", + name: INDEX_NAME + + eligible_mrs = Gitlab::BackgroundMigration::BackfillDraftStatusOnMergeRequests::MergeRequest.eligible + + queue_background_migration_jobs_by_range_at_intervals( + eligible_mrs, + MIGRATION, + DELAY_INTERVAL, + track_jobs: true, + batch_size: BATCH_SIZE + ) + end + + def down + remove_concurrent_index_by_name :merge_requests, INDEX_NAME + end +end diff --git a/db/post_migrate/20210610042700_remove_clusters_applications_fluentd_table.rb b/db/post_migrate/20210610042700_remove_clusters_applications_fluentd_table.rb new file mode 100644 index 00000000000..9d37180326f --- /dev/null +++ b/db/post_migrate/20210610042700_remove_clusters_applications_fluentd_table.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class RemoveClustersApplicationsFluentdTable < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + drop_table :clusters_applications_fluentd + end + + def down + create_table :clusters_applications_fluentd do |t| + t.integer :protocol, null: false, limit: 2 + t.integer :status, null: false + t.integer :port, null: false + t.references :cluster, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade } + t.timestamps_with_timezone null: false + t.string :version, null: false, limit: 255 + t.string :host, null: false, limit: 255 + t.boolean :cilium_log_enabled, default: true, null: false + t.boolean :waf_log_enabled, default: true, null: false + t.text :status_reason # rubocop:disable Migration/AddLimitToTextColumns + end + end +end diff --git a/db/post_migrate/20210614124111_add_devops_adoption_sast_dast_indexes.rb b/db/post_migrate/20210614124111_add_devops_adoption_sast_dast_indexes.rb new file mode 100644 index 00000000000..9d40fe30ed6 --- /dev/null +++ b/db/post_migrate/20210614124111_add_devops_adoption_sast_dast_indexes.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddDevopsAdoptionSastDastIndexes < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + INDEX_SAST = 'index_ci_job_artifacts_sast_for_devops_adoption' + INDEX_DAST = 'index_ci_job_artifacts_dast_for_devops_adoption' + + def up + add_concurrent_index :ci_job_artifacts, [:project_id, :created_at], where: "file_type = 5", name: INDEX_SAST + add_concurrent_index :ci_job_artifacts, [:project_id, :created_at], where: "file_type = 8", name: INDEX_DAST + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_SAST + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_DAST + end +end diff --git a/db/post_migrate/20210615234935_fix_batched_migrations_old_format_job_arguments.rb b/db/post_migrate/20210615234935_fix_batched_migrations_old_format_job_arguments.rb new file mode 100644 index 00000000000..535f7426938 --- /dev/null +++ b/db/post_migrate/20210615234935_fix_batched_migrations_old_format_job_arguments.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class FixBatchedMigrationsOldFormatJobArguments < ActiveRecord::Migration[6.1] + class BatchedMigration < ActiveRecord::Base + self.table_name = 'batched_background_migrations' + end + + def up + # rubocop:disable Style/WordArray + [ + ['events', 'id', ['id', 'id_convert_to_bigint'], [['id'], ['id_convert_to_bigint']]], + ['push_event_payloads', 'event_id', ['event_id', 'event_id_convert_to_bigint'], [['event_id'], ['event_id_convert_to_bigint']]] + ].each do |table_name, column_name, legacy_job_arguments, current_job_arguments| + base_scope = BatchedMigration + .where(job_class_name: 'CopyColumnUsingBackgroundMigrationJob', table_name: table_name, column_name: column_name) + # rubocop:enable Style/WordArray + + # rubocop:disable Rails/WhereEquals + base_scope + .where('job_arguments = ?', legacy_job_arguments.to_json) + .where('NOT EXISTS (?)', base_scope.select('1').where('job_arguments = ?', current_job_arguments.to_json)) + .update_all(job_arguments: current_job_arguments) + # rubocop:enable Rails/WhereEquals + end + end + + def down + # No-op, there is no way to know were the existing record migrated from + # legacy job arguments, or were using the current format from the start. + # There is no reason to go back anyway. + end +end diff --git a/db/post_migrate/20210616145254_add_partial_index_for_ci_builds_token.rb b/db/post_migrate/20210616145254_add_partial_index_for_ci_builds_token.rb new file mode 100644 index 00000000000..6e169a6f0e1 --- /dev/null +++ b/db/post_migrate/20210616145254_add_partial_index_for_ci_builds_token.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddPartialIndexForCiBuildsToken < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + NAME = 'index_ci_builds_on_token_partial' + + def up + add_concurrent_index :ci_builds, :token, unique: true, where: 'token IS NOT NULL', name: NAME + end + + def down + remove_concurrent_index_by_name :ci_builds, NAME + end +end diff --git a/db/post_migrate/20210621111747_add_ci_artifacts_devops_adoption_index.rb b/db/post_migrate/20210621111747_add_ci_artifacts_devops_adoption_index.rb new file mode 100644 index 00000000000..71fd61072ac --- /dev/null +++ b/db/post_migrate/20210621111747_add_ci_artifacts_devops_adoption_index.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +# +class AddCiArtifactsDevopsAdoptionIndex < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + NEW_INDEX = 'index_ci_job_artifacts_on_file_type_for_devops_adoption' + + def up + add_concurrent_index :ci_job_artifacts, [:file_type, :project_id, :created_at], name: NEW_INDEX, where: 'file_type IN (5,6,8,23)' + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, NEW_INDEX + end +end diff --git a/db/post_migrate/20210621155328_replace_project_authorizations_project_id_index.rb b/db/post_migrate/20210621155328_replace_project_authorizations_project_id_index.rb new file mode 100644 index 00000000000..498090c3ab9 --- /dev/null +++ b/db/post_migrate/20210621155328_replace_project_authorizations_project_id_index.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class ReplaceProjectAuthorizationsProjectIdIndex < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + OLD_INDEX_NAME = 'index_project_authorizations_on_project_id' + NEW_INDEX_NAME = 'index_project_authorizations_on_project_id_user_id' + + def up + add_concurrent_index(:project_authorizations, [:project_id, :user_id], name: NEW_INDEX_NAME) + remove_concurrent_index_by_name(:project_authorizations, OLD_INDEX_NAME) + end + + def down + add_concurrent_index(:project_authorizations, :project_id, name: OLD_INDEX_NAME) + remove_concurrent_index_by_name(:project_authorizations, NEW_INDEX_NAME) + end +end diff --git a/db/post_migrate/20210621164210_drop_remove_on_close_from_labels.rb b/db/post_migrate/20210621164210_drop_remove_on_close_from_labels.rb new file mode 100644 index 00000000000..0430c8447d9 --- /dev/null +++ b/db/post_migrate/20210621164210_drop_remove_on_close_from_labels.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class DropRemoveOnCloseFromLabels < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + def up + # Migration that adds column was reverted, but run in Gitlab SaaS stg and prod + return unless column_exists?(:labels, :remove_on_close) + + with_lock_retries do + remove_column :labels, :remove_on_close + end + end + + def down + # No rollback as the original migration was reverted in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62056 + # up simply removes the column from envs where the original migration was run + end +end diff --git a/db/post_migrate/20210621223000_steal_background_jobs_that_reference_services.rb b/db/post_migrate/20210621223000_steal_background_jobs_that_reference_services.rb new file mode 100644 index 00000000000..8d326036a68 --- /dev/null +++ b/db/post_migrate/20210621223000_steal_background_jobs_that_reference_services.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class StealBackgroundJobsThatReferenceServices < ActiveRecord::Migration[6.1] + def up + Gitlab::BackgroundMigration.steal('BackfillJiraTrackerDeploymentType2') + Gitlab::BackgroundMigration.steal('FixProjectsWithoutPrometheusService') + Gitlab::BackgroundMigration.steal('MigrateIssueTrackersSensitiveData') + Gitlab::BackgroundMigration.steal('RemoveDuplicateServices') + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210621223242_finalize_rename_services_to_integrations.rb b/db/post_migrate/20210621223242_finalize_rename_services_to_integrations.rb new file mode 100644 index 00000000000..803a6fa0aca --- /dev/null +++ b/db/post_migrate/20210621223242_finalize_rename_services_to_integrations.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class FinalizeRenameServicesToIntegrations < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + finalize_table_rename(:services, :integrations) + end + + def down + undo_finalize_table_rename(:services, :integrations) + end +end diff --git a/db/post_migrate/20210622041846_finalize_push_event_payloads_bigint_conversion.rb b/db/post_migrate/20210622041846_finalize_push_event_payloads_bigint_conversion.rb new file mode 100644 index 00000000000..38b081e3e5e --- /dev/null +++ b/db/post_migrate/20210622041846_finalize_push_event_payloads_bigint_conversion.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +class FinalizePushEventPayloadsBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'push_event_payloads' + INDEX_NAME = 'index_push_event_payloads_on_event_id_convert_to_bigint' + + def up + return unless should_run? + + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'event_id', + job_arguments: [["event_id"], ["event_id_convert_to_bigint"]] + ) + + swap_columns + end + + def down + return unless should_run? + + swap_columns + end + + private + + def should_run? + Gitlab.dev_or_test_env? || Gitlab.com? + end + + def swap_columns + add_concurrent_index TABLE_NAME, :event_id_convert_to_bigint, unique: true, name: INDEX_NAME + + # Add a foreign key on `event_id_convert_to_bigint` before we swap the columns and drop the old FK (fk_36c74129da) + add_concurrent_foreign_key TABLE_NAME, :events, column: :event_id_convert_to_bigint, on_delete: :cascade + + with_lock_retries(raise_on_exhaustion: true) do + # Swap column names + temp_name = 'event_id_tmp' + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:event_id)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:event_id_convert_to_bigint)} TO #{quote_column_name(:event_id)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:event_id_convert_to_bigint)}" + + # We need to update the trigger function in order to make PostgreSQL to + # regenerate the execution plan for it. This is to avoid type mismatch errors like + # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)" + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:event_id, :event_id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # Swap defaults + change_column_default TABLE_NAME, :event_id, nil + change_column_default TABLE_NAME, :event_id_convert_to_bigint, 0 + + # Swap PK constraint + execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT push_event_payloads_pkey" + rename_index TABLE_NAME, INDEX_NAME, 'push_event_payloads_pkey' + execute "ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT push_event_payloads_pkey PRIMARY KEY USING INDEX push_event_payloads_pkey" + + # Drop original FK on the old int4 `event_id` (fk_36c74129da) + remove_foreign_key TABLE_NAME, name: concurrent_foreign_key_name(TABLE_NAME, :event_id) + # We swapped the columns but the FK for event_id is still using the old name for the event_id_convert_to_bigint column + # So we have to also swap the FK name now that we dropped the other one with the same + rename_constraint( + TABLE_NAME, + concurrent_foreign_key_name(TABLE_NAME, :event_id_convert_to_bigint), + concurrent_foreign_key_name(TABLE_NAME, :event_id) + ) + end + end +end diff --git a/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb b/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb new file mode 100644 index 00000000000..cbd0d0ea3a2 --- /dev/null +++ b/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class ScheduleDeleteOrphanedDeployments < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'DeleteOrphanedDeployments' + BATCH_SIZE = 100_000 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + define_batchable_model('deployments'), + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210628124505_reset_job_token_scope_enabled.rb b/db/post_migrate/20210628124505_reset_job_token_scope_enabled.rb new file mode 100644 index 00000000000..1176e704d0a --- /dev/null +++ b/db/post_migrate/20210628124505_reset_job_token_scope_enabled.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class ResetJobTokenScopeEnabled < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + with_lock_retries do + remove_column :project_ci_cd_settings, :job_token_scope_enabled + add_column :project_ci_cd_settings, :job_token_scope_enabled, :boolean, default: false, null: false + end + end + + def down + # Irreversible + end +end diff --git a/db/post_migrate/20210629101712_remove_deprecated_modsecurity_columns.rb b/db/post_migrate/20210629101712_remove_deprecated_modsecurity_columns.rb new file mode 100644 index 00000000000..371298aef62 --- /dev/null +++ b/db/post_migrate/20210629101712_remove_deprecated_modsecurity_columns.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveDeprecatedModsecurityColumns < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + INDEX_NAME = 'index_clusters_applications_ingress_on_modsecurity' + + def up + remove_column :clusters_applications_ingress, :modsecurity_enabled if column_exists?(:clusters_applications_ingress, :modsecurity_enabled) + remove_column :clusters_applications_ingress, :modsecurity_mode if column_exists?(:clusters_applications_ingress, :modsecurity_mode) + end + + def down + add_column :clusters_applications_ingress, :modsecurity_enabled, :boolean unless column_exists?(:clusters_applications_ingress, :modsecurity_enabled) + add_column :clusters_applications_ingress, :modsecurity_mode, :smallint, null: false, default: 0 unless column_exists?(:clusters_applications_ingress, :modsecurity_mode) + + add_concurrent_index :clusters_applications_ingress, [:modsecurity_enabled, :modsecurity_mode, :cluster_id], name: INDEX_NAME + end +end diff --git a/db/post_migrate/20210629104933_drop_index_on_ci_builds_for_token.rb b/db/post_migrate/20210629104933_drop_index_on_ci_builds_for_token.rb new file mode 100644 index 00000000000..cc26db924fe --- /dev/null +++ b/db/post_migrate/20210629104933_drop_index_on_ci_builds_for_token.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class DropIndexOnCiBuildsForToken < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + INDEXNAME = :index_ci_builds_on_token + + def up + remove_concurrent_index_by_name :ci_builds, INDEXNAME + end + + def down + add_concurrent_index :ci_builds, :token, unique: true, name: INDEXNAME + end +end diff --git a/db/post_migrate/20210630025020_migrate_push_event_payloads_event_id_back_to_integer_for_gitlab_com.rb b/db/post_migrate/20210630025020_migrate_push_event_payloads_event_id_back_to_integer_for_gitlab_com.rb new file mode 100644 index 00000000000..834a23a5c7a --- /dev/null +++ b/db/post_migrate/20210630025020_migrate_push_event_payloads_event_id_back_to_integer_for_gitlab_com.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require Rails.root.join('db', 'post_migrate', '20210622041846_finalize_push_event_payloads_bigint_conversion') + +class MigratePushEventPayloadsEventIdBackToIntegerForGitlabCom < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + def up + FinalizePushEventPayloadsBigintConversion.new.down + end + + def down + FinalizePushEventPayloadsBigintConversion.new.up + end +end diff --git a/db/post_migrate/20210701111909_backfill_issues_upvotes_count.rb b/db/post_migrate/20210701111909_backfill_issues_upvotes_count.rb new file mode 100644 index 00000000000..0afc0bc1d08 --- /dev/null +++ b/db/post_migrate/20210701111909_backfill_issues_upvotes_count.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class BackfillIssuesUpvotesCount < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + MIGRATION = 'BackfillUpvotesCountOnIssues' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 5_000 + + def up + scope = Issue.joins("INNER JOIN award_emoji e ON e.awardable_id = issues.id AND e.awardable_type = 'Issue' AND e.name = 'thumbsup'") + + queue_background_migration_jobs_by_range_at_intervals( + scope, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210706115312_add_upvotes_count_index_to_issues.rb b/db/post_migrate/20210706115312_add_upvotes_count_index_to_issues.rb new file mode 100644 index 00000000000..65ec43930ea --- /dev/null +++ b/db/post_migrate/20210706115312_add_upvotes_count_index_to_issues.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddUpvotesCountIndexToIssues < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + INDEX_NAME = 'index_issues_on_project_id_and_upvotes_count' + + def up + add_concurrent_index :issues, [:project_id, :upvotes_count], name: INDEX_NAME + end + + def down + remove_concurrent_index :issues, [:project_id, :upvotes_count], name: INDEX_NAME + end +end diff --git a/db/post_migrate/20210706120847_remove_framework_column_from_compliance_management_frameworks.rb b/db/post_migrate/20210706120847_remove_framework_column_from_compliance_management_frameworks.rb new file mode 100644 index 00000000000..73344ee061f --- /dev/null +++ b/db/post_migrate/20210706120847_remove_framework_column_from_compliance_management_frameworks.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class RemoveFrameworkColumnFromComplianceManagementFrameworks < ActiveRecord::Migration[6.1] + def change + remove_column :project_compliance_framework_settings, :framework, :smallint + end +end diff --git a/db/post_migrate/20210706142819_re_schedule_latest_pipeline_id_population.rb b/db/post_migrate/20210706142819_re_schedule_latest_pipeline_id_population.rb new file mode 100644 index 00000000000..709e0be8b79 --- /dev/null +++ b/db/post_migrate/20210706142819_re_schedule_latest_pipeline_id_population.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class ReScheduleLatestPipelineIdPopulation < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 100 + MIGRATION = 'PopulateLatestPipelineIds' + + disable_ddl_transaction! + + def up + return unless Gitlab.ee? + + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::PopulateLatestPipelineIds::ProjectSetting.has_vulnerabilities_without_latest_pipeline_set, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + primary_column_name: 'project_id' + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210708130419_reschedule_merge_request_diff_users_background_migration.rb b/db/post_migrate/20210708130419_reschedule_merge_request_diff_users_background_migration.rb new file mode 100644 index 00000000000..53f13ca96d2 --- /dev/null +++ b/db/post_migrate/20210708130419_reschedule_merge_request_diff_users_background_migration.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +class RescheduleMergeRequestDiffUsersBackgroundMigration < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + # The number of rows to process in a single migration job. + # + # The minimum interval for background migrations is two minutes. On staging we + # observed we can process roughly 20 000 rows in a minute. Based on the total + # number of rows on staging, this translates to a total processing time of + # roughly 14 days. + # + # By using a batch size of 40 000, we maintain a rate of roughly 20 000 rows + # per minute, hopefully keeping the total migration time under two weeks; + # instead of four weeks. + BATCH_SIZE = 40_000 + + MIGRATION_NAME = 'MigrateMergeRequestDiffCommitUsers' + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + end + + def up + start = MergeRequestDiff.minimum(:id).to_i + max = MergeRequestDiff.maximum(:id).to_i + delay = BackgroundMigrationWorker.minimum_interval + + Gitlab::Database::BackgroundMigrationJob + .where(class_name: MIGRATION_NAME) + .delete_all + + # The table merge_request_diff_commits contains _a lot_ of rows (roughly 400 + # 000 000 on staging). Iterating a table that large to determine job ranges + # would take a while. + # + # To avoid that overhead, we simply schedule fixed ranges according to the + # minimum and maximum IDs. The background migration in turn only processes + # rows that actually exist. + while start < max + stop = start + BATCH_SIZE + + migrate_in(delay, MIGRATION_NAME, [start, stop]) + + Gitlab::Database::BackgroundMigrationJob + .create!(class_name: MIGRATION_NAME, arguments: [start, stop]) + + delay += BackgroundMigrationWorker.minimum_interval + start += BATCH_SIZE + end + end + + def down + # no-op + end +end |