diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 12:08:42 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 12:08:42 +0300 |
commit | b76ae638462ab0f673e5915986070518dd3f9ad3 (patch) | |
tree | bdab0533383b52873be0ec0eb4d3c66598ff8b91 /db/post_migrate | |
parent | 434373eabe7b4be9593d18a585fb763f1e5f1a6f (diff) |
Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42
Diffstat (limited to 'db/post_migrate')
41 files changed, 1337 insertions, 55 deletions
diff --git a/db/post_migrate/20190517153211_migrate_k8s_service_integration.rb b/db/post_migrate/20190517153211_migrate_k8s_service_integration.rb index 4bd04edb239..0b409cd2866 100644 --- a/db/post_migrate/20190517153211_migrate_k8s_service_integration.rb +++ b/db/post_migrate/20190517153211_migrate_k8s_service_integration.rb @@ -70,7 +70,7 @@ class MigrateK8sServiceIntegration < ActiveRecord::Migration[5.1] private def parsed_properties - @parsed_properties ||= JSON.parse(self.properties) + @parsed_properties ||= JSON.parse(self.properties) # rubocop:disable Gitlab/Json end end diff --git a/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb b/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb index b41c55ce622..570eec53be3 100644 --- a/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb +++ b/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb @@ -20,7 +20,7 @@ class MigrateSamlIdentitiesToScimIdentities < ActiveRecord::Migration[6.0] record.attributes.extract!("extern_uid", "user_id", "group_id", "active", "created_at", "updated_at") end - Gitlab::Database.bulk_insert(:scim_identities, data_to_insert, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(:scim_identities, data_to_insert, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert end end diff --git a/db/post_migrate/20210609202501_schedule_backfill_draft_status_on_merge_requests.rb b/db/post_migrate/20210609202501_schedule_backfill_draft_status_on_merge_requests.rb index 72c4168af50..ed9a64c84ab 100644 --- a/db/post_migrate/20210609202501_schedule_backfill_draft_status_on_merge_requests.rb +++ b/db/post_migrate/20210609202501_schedule_backfill_draft_status_on_merge_requests.rb @@ -1,32 +1,13 @@ # frozen_string_literal: true class ScheduleBackfillDraftStatusOnMergeRequests < ActiveRecord::Migration[6.1] - include Gitlab::Database::MigrationHelpers - - INDEX_NAME = "tmp_index_merge_requests_draft_and_status" - MIGRATION = 'BackfillDraftStatusOnMergeRequests' - DELAY_INTERVAL = 2.minutes - BATCH_SIZE = 100 - - disable_ddl_transaction! - def up - add_concurrent_index :merge_requests, :id, - where: "draft = false AND state_id = 1 AND ((title)::text ~* '^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP'::text)", - name: INDEX_NAME - - eligible_mrs = Gitlab::BackgroundMigration::BackfillDraftStatusOnMergeRequests::MergeRequest.eligible - - queue_background_migration_jobs_by_range_at_intervals( - eligible_mrs, - MIGRATION, - DELAY_INTERVAL, - track_jobs: true, - batch_size: BATCH_SIZE - ) + # noop + # end def down - remove_concurrent_index_by_name :merge_requests, INDEX_NAME + # noop + # end end diff --git a/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb b/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb index cbd0d0ea3a2..a38cb68ff08 100644 --- a/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb +++ b/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb @@ -11,13 +11,12 @@ class ScheduleDeleteOrphanedDeployments < ActiveRecord::Migration[6.1] disable_ddl_transaction! def up - queue_background_migration_jobs_by_range_at_intervals( - define_batchable_model('deployments'), - MIGRATION, - DELAY_INTERVAL, - batch_size: BATCH_SIZE, - track_jobs: true - ) + # no-op. + # This background migration is rescheduled in 20210722010101_cleanup_delete_orphaned_deployments_background_migration.rb + # with a smaller batch size, because the initial attempt caused + # 80 failures out of 1639 batches (faiulre rate is 4.88%) due to statement timeouts, + # that takes approx. 1 hour to perform a cleanup/sync migration. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/335071#note_618380503 for more information. end def down diff --git a/db/post_migrate/20210701033622_finalize_ci_builds_needs_bigint_conversion.rb b/db/post_migrate/20210701033622_finalize_ci_builds_needs_bigint_conversion.rb new file mode 100644 index 00000000000..baee9fb3848 --- /dev/null +++ b/db/post_migrate/20210701033622_finalize_ci_builds_needs_bigint_conversion.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class FinalizeCiBuildsNeedsBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'ci_build_needs' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [['build_id'], ['build_id_convert_to_bigint']] + ) + + swap + end + + def down + swap + end + + private + + def swap + # This is to replace the existing "index_ci_build_needs_on_build_id_and_name" UNIQUE, btree (build_id, name) + add_concurrent_index TABLE_NAME, [:build_id_convert_to_bigint, :name], unique: true, name: 'index_ci_build_needs_on_build_id_convert_to_bigint_and_name' + + # Add a foreign key on `build_id_convert_to_bigint` before we swap the columns and drop the old FK (fk_rails_3cf221d4ed) + add_concurrent_foreign_key TABLE_NAME, :ci_builds, + column: :build_id_convert_to_bigint, on_delete: :cascade, + name: 'fk_rails_3cf221d4ed_tmp', reverse_lock_order: true + + with_lock_retries(raise_on_exhaustion: true) do + # We'll need ACCESS EXCLUSIVE lock on the related tables, + # lets make sure it can be acquired from the start + execute "LOCK TABLE ci_builds, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + # Swap column names + temp_name = 'build_id_tmp' + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:build_id)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:build_id_convert_to_bigint)} TO #{quote_column_name(:build_id)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:build_id_convert_to_bigint)}" + + # We need to update the trigger function in order to make PostgreSQL to + # regenerate the execution plan for it. This is to avoid type mismatch errors like + # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)" + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:build_id, :build_id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # Swap defaults + change_column_default TABLE_NAME, :build_id, nil + change_column_default TABLE_NAME, :build_id_convert_to_bigint, 0 + + # Rename the index on the `bigint` column to match the new column name + # (we already hold an exclusive lock, so no need to use DROP INDEX CONCURRENTLY here) + execute 'DROP INDEX index_ci_build_needs_on_build_id_and_name' + rename_index TABLE_NAME, 'index_ci_build_needs_on_build_id_convert_to_bigint_and_name', 'index_ci_build_needs_on_build_id_and_name' + + # Drop original FK on the old int4 `build_id` (fk_rails_3cf221d4ed) + remove_foreign_key TABLE_NAME, name: 'fk_rails_3cf221d4ed' + # We swapped the columns but the FK for buil_id is still using the temporary name for the buil_id_convert_to_bigint column + # So we have to also swap the FK name now that we dropped the other one with the same + rename_constraint(TABLE_NAME, 'fk_rails_3cf221d4ed_tmp', 'fk_rails_3cf221d4ed') + end + end +end diff --git a/db/post_migrate/20210706142819_re_schedule_latest_pipeline_id_population.rb b/db/post_migrate/20210706142819_re_schedule_latest_pipeline_id_population.rb index 709e0be8b79..255b64f9bc7 100644 --- a/db/post_migrate/20210706142819_re_schedule_latest_pipeline_id_population.rb +++ b/db/post_migrate/20210706142819_re_schedule_latest_pipeline_id_population.rb @@ -1,28 +1,9 @@ # frozen_string_literal: true class ReScheduleLatestPipelineIdPopulation < ActiveRecord::Migration[6.1] - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - DELAY_INTERVAL = 2.minutes.to_i - BATCH_SIZE = 100 - MIGRATION = 'PopulateLatestPipelineIds' - - disable_ddl_transaction! - - def up - return unless Gitlab.ee? - - queue_background_migration_jobs_by_range_at_intervals( - Gitlab::BackgroundMigration::PopulateLatestPipelineIds::ProjectSetting.has_vulnerabilities_without_latest_pipeline_set, - MIGRATION, - DELAY_INTERVAL, - batch_size: BATCH_SIZE, - primary_column_name: 'project_id' - ) - end - - def down - # no-op + def change + # no-op: This migration has been marked as no-op and replaced by + # `ReScheduleLatestPipelineIdPopulationWithLogging` as we've found some problems. + # For more information: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66050 end end diff --git a/db/post_migrate/20210706212710_finalize_ci_job_artifacts_bigint_conversion.rb b/db/post_migrate/20210706212710_finalize_ci_job_artifacts_bigint_conversion.rb new file mode 100644 index 00000000000..40977277bd1 --- /dev/null +++ b/db/post_migrate/20210706212710_finalize_ci_job_artifacts_bigint_conversion.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class FinalizeCiJobArtifactsBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'ci_job_artifacts' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [%w[id job_id], %w[id_convert_to_bigint job_id_convert_to_bigint]] + ) + + swap + end + + def down + swap + end + + private + + def swap + add_concurrent_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: 'index_ci_job_artifact_on_id_convert_to_bigint' + # This is to replace the existing "index_ci_job_artifacts_for_terraform_reports" btree (project_id, id) where (file_type = 18) + add_concurrent_index TABLE_NAME, [:project_id, :id_convert_to_bigint], name: 'index_ci_job_artifacts_for_terraform_reports_bigint', where: "file_type = 18" + # This is to replace the existing "index_ci_job_artifacts_id_for_terraform_reports" btree (id) where (file_type = 18) + add_concurrent_index TABLE_NAME, [:id_convert_to_bigint], name: 'index_ci_job_artifacts_id_for_terraform_reports_bigint', where: "file_type = 18" + + # Add a FK on `project_pages_metadata(artifacts_archive_id)` to `id_convert_to_bigint`, the old FK (fk_69366a119e) + # will be removed when ci_job_artifacts_pkey constraint is droppped. + fk_artifacts_archive_id = concurrent_foreign_key_name(:project_pages_metadata, :artifacts_archive_id) + fk_artifacts_archive_id_tmp = "#{fk_artifacts_archive_id}_tmp" + add_concurrent_foreign_key :project_pages_metadata, TABLE_NAME, + column: :artifacts_archive_id, target_column: :id_convert_to_bigint, + name: fk_artifacts_archive_id_tmp, + on_delete: :nullify, + reverse_lock_order: true + + with_lock_retries(raise_on_exhaustion: true) do + # We'll need ACCESS EXCLUSIVE lock on the related tables, + # lets make sure it can be acquired from the start + execute "LOCK TABLE #{TABLE_NAME}, project_pages_metadata IN ACCESS EXCLUSIVE MODE" + + # Swap column names + temp_name = 'id_tmp' + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:id)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:id_convert_to_bigint)} TO #{quote_column_name(:id)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:id_convert_to_bigint)}" + + # We need to update the trigger function in order to make PostgreSQL to + # regenerate the execution plan for it. This is to avoid type mismatch errors like + # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)" + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name([:id, :job_id], [:id_convert_to_bigint, :job_id_convert_to_bigint]) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # Swap defaults + execute "ALTER SEQUENCE ci_job_artifacts_id_seq OWNED BY #{TABLE_NAME}.id" + change_column_default TABLE_NAME, :id, -> { "nextval('ci_job_artifacts_id_seq'::regclass)" } + change_column_default TABLE_NAME, :id_convert_to_bigint, 0 + + # Swap PK constraint + execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT ci_job_artifacts_pkey CASCADE" # this will drop ci_job_artifacts_pkey primary key + rename_index TABLE_NAME, 'index_ci_job_artifact_on_id_convert_to_bigint', 'ci_job_artifacts_pkey' + execute "ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT ci_job_artifacts_pkey PRIMARY KEY USING INDEX ci_job_artifacts_pkey" + + # Rename the rest of the indexes (we already hold an exclusive lock, so no need to use DROP INDEX CONCURRENTLY here + execute 'DROP INDEX index_ci_job_artifacts_for_terraform_reports' + rename_index TABLE_NAME, 'index_ci_job_artifacts_for_terraform_reports_bigint', 'index_ci_job_artifacts_for_terraform_reports' + execute 'DROP INDEX index_ci_job_artifacts_id_for_terraform_reports' + rename_index TABLE_NAME, 'index_ci_job_artifacts_id_for_terraform_reports_bigint', 'index_ci_job_artifacts_id_for_terraform_reports' + + # Change the name of the temporary FK for project_pages_metadata(artifacts_archive_id) -> id + rename_constraint(:project_pages_metadata, fk_artifacts_archive_id_tmp, fk_artifacts_archive_id) + end + end +end diff --git a/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb b/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb new file mode 100644 index 00000000000..2e294a0b2e3 --- /dev/null +++ b/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class FinalizeCiStagesBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + + TABLE_NAME = 'ci_stages' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [['id'], ['id_convert_to_bigint']] + ) + + swap + end + + def down + swap + end + + def swap + # This will replace the existing ci_stages_pkey index for the primary key + add_concurrent_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: 'index_ci_stages_on_id_convert_to_bigint' + + # This will replace the existing ci_stages_on_pipeline_id_and_id index + add_concurrent_index TABLE_NAME, [:pipeline_id, :id_convert_to_bigint], + name: 'index_ci_stages_on_pipeline_id_and_id_convert_to_bigint', + where: 'status in (0, 1, 2, 8, 9, 10)' + + # Add a foreign key on ci_builds(stage_id_convert_to_bigint), which we'll rename later. Give it the correct final name + fk_stage_id = concurrent_foreign_key_name(:ci_builds, :stage_id) + fk_stage_id_tmp = "#{fk_stage_id}_tmp" + add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, + target_column: :id_convert_to_bigint, + name: fk_stage_id_tmp, + on_delete: :cascade, + reverse_lock_order: true + + # Now it's time to do things in a transaction + with_lock_retries(raise_on_exhaustion: true) do + execute "LOCK TABLE #{TABLE_NAME}, ci_builds IN ACCESS EXCLUSIVE MODE" + + temp_name = quote_column_name('id_tmp') + id_name = quote_column_name(:id) + id_convert_to_bigint_name = quote_column_name(:id_convert_to_bigint) + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_name} TO #{temp_name}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_convert_to_bigint_name} TO #{id_name}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{temp_name} TO #{id_convert_to_bigint_name}" + + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:id, :id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # Swap defaults + execute "ALTER SEQUENCE ci_stages_id_seq OWNED BY #{TABLE_NAME}.id" + change_column_default TABLE_NAME, :id, -> { "nextval('ci_stages_id_seq'::regclass)"} + change_column_default TABLE_NAME, :id_convert_to_bigint, 0 + + # Swap pkey constraint + # This will drop fk_3a9eaa254d (ci_builds(stage_id) references ci_stages(id)) + execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT ci_stages_pkey CASCADE" + rename_index TABLE_NAME, 'index_ci_stages_on_id_convert_to_bigint', 'ci_stages_pkey' + execute "ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT ci_stages_pkey PRIMARY KEY USING INDEX ci_stages_pkey" + + # Rename the other indexes + execute "DROP INDEX index_ci_stages_on_pipeline_id_and_id" + rename_index TABLE_NAME, 'index_ci_stages_on_pipeline_id_and_id_convert_to_bigint', 'index_ci_stages_on_pipeline_id_and_id' + + rename_constraint(:ci_builds, fk_stage_id_tmp, fk_stage_id) + end + end +end diff --git a/db/post_migrate/20210709024048_finalize_push_event_payloads_bigint_conversion_2.rb b/db/post_migrate/20210709024048_finalize_push_event_payloads_bigint_conversion_2.rb new file mode 100644 index 00000000000..aa812151164 --- /dev/null +++ b/db/post_migrate/20210709024048_finalize_push_event_payloads_bigint_conversion_2.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class FinalizePushEventPayloadsBigintConversion2 < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + # no-op due to https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5256 + end + + def down + # no-op due to https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5256 + end +end diff --git a/db/post_migrate/20210713042153_finalize_ci_sources_pipelines_bigint_conversion.rb b/db/post_migrate/20210713042153_finalize_ci_sources_pipelines_bigint_conversion.rb new file mode 100644 index 00000000000..938c8c172a3 --- /dev/null +++ b/db/post_migrate/20210713042153_finalize_ci_sources_pipelines_bigint_conversion.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +class FinalizeCiSourcesPipelinesBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'ci_sources_pipelines' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [['source_job_id'], ['source_job_id_convert_to_bigint']] + ) + + swap + end + + def down + swap + end + + private + + def swap + # This is to replace the existing "index_ci_sources_pipelines_on_source_job_id" btree (source_job_id) + add_concurrent_index TABLE_NAME, :source_job_id_convert_to_bigint, name: 'index_ci_sources_pipelines_on_source_job_id_convert_to_bigint' + + # Add a foreign key on `source_job_id_convert_to_bigint` before we swap the columns and drop the old FK (fk_be5624bf37) + add_concurrent_foreign_key TABLE_NAME, :ci_builds, + column: :source_job_id_convert_to_bigint, on_delete: :cascade, + name: 'fk_be5624bf37_tmp', reverse_lock_order: true + + with_lock_retries(raise_on_exhaustion: true) do + # We'll need ACCESS EXCLUSIVE lock on the related tables, + # lets make sure it can be acquired from the start + execute "LOCK TABLE ci_builds, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + # Swap column names + temp_name = 'source_job_id_tmp' + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:source_job_id)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:source_job_id_convert_to_bigint)} TO #{quote_column_name(:source_job_id)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:source_job_id_convert_to_bigint)}" + + # We need to update the trigger function in order to make PostgreSQL to + # regenerate the execution plan for it. This is to avoid type mismatch errors like + # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)" + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:source_job_id, :source_job_id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # No need to swap defaults, both columns have no default value + + # Rename the index on the `bigint` column to match the new column name + # (we already hold an exclusive lock, so no need to use DROP INDEX CONCURRENTLY here) + execute 'DROP INDEX index_ci_sources_pipelines_on_source_job_id' + rename_index TABLE_NAME, 'index_ci_sources_pipelines_on_source_job_id_convert_to_bigint', 'index_ci_sources_pipelines_on_source_job_id' + + # Drop original FK on the old int4 `source_job_id` (fk_be5624bf37) + remove_foreign_key TABLE_NAME, name: 'fk_be5624bf37' + # We swapped the columns but the FK is still using the temporary name + # So we have to also swap the FK name now that we dropped the other one + rename_constraint(TABLE_NAME, 'fk_be5624bf37_tmp', 'fk_be5624bf37') + end + end +end diff --git a/db/post_migrate/20210713075117_re_schedule_latest_pipeline_id_population_with_logging.rb b/db/post_migrate/20210713075117_re_schedule_latest_pipeline_id_population_with_logging.rb new file mode 100644 index 00000000000..457e7826f38 --- /dev/null +++ b/db/post_migrate/20210713075117_re_schedule_latest_pipeline_id_population_with_logging.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class ReScheduleLatestPipelineIdPopulationWithLogging < ActiveRecord::Migration[6.1] + def up + # no-op: The content of the migration has been moved to + # `ReScheduleLatestPipelineIdPopulationWithAllSecurityRelatedArtifactTypes`. + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210714015537_finalize_ci_build_trace_chunks_bigint_conversion.rb b/db/post_migrate/20210714015537_finalize_ci_build_trace_chunks_bigint_conversion.rb new file mode 100644 index 00000000000..9195b662776 --- /dev/null +++ b/db/post_migrate/20210714015537_finalize_ci_build_trace_chunks_bigint_conversion.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +class FinalizeCiBuildTraceChunksBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'ci_build_trace_chunks' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [['build_id'], ['build_id_convert_to_bigint']] + ) + + swap + end + + def down + swap + end + + private + + def swap + # This is to replace the existing "index_ci_build_trace_chunks_on_build_id_and_chunk_index" UNIQUE, btree (build_id, chunk_index) + add_concurrent_index TABLE_NAME, [:build_id_convert_to_bigint, :chunk_index], unique: true, name: 'i_ci_build_trace_chunks_build_id_convert_to_bigint_chunk_index' + + # Add a foreign key on `build_id_convert_to_bigint` before we swap the columns and drop the old FK () + add_concurrent_foreign_key TABLE_NAME, :ci_builds, column: :build_id_convert_to_bigint, on_delete: :cascade, name: 'fk_rails_1013b761f2_tmp' + + with_lock_retries(raise_on_exhaustion: true) do + # We'll need ACCESS EXCLUSIVE lock on the related tables, + # lets make sure it can be acquired from the start + execute "LOCK TABLE #{TABLE_NAME}, ci_builds IN ACCESS EXCLUSIVE MODE" + + # Swap column names + temp_name = 'build_id_tmp' + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:build_id)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:build_id_convert_to_bigint)} TO #{quote_column_name(:build_id)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:build_id_convert_to_bigint)}" + + # We need to update the trigger function in order to make PostgreSQL to + # regenerate the execution plan for it. This is to avoid type mismatch errors like + # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)" + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:build_id, :build_id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # Swap defaults + change_column_default TABLE_NAME, :build_id, nil + change_column_default TABLE_NAME, :build_id_convert_to_bigint, 0 + + # Rename the index on the `bigint` column to match the new column name + # (we already hold an exclusive lock, so no need to use DROP INDEX CONCURRENTLY here) + execute 'DROP INDEX index_ci_build_trace_chunks_on_build_id_and_chunk_index' + rename_index TABLE_NAME, 'i_ci_build_trace_chunks_build_id_convert_to_bigint_chunk_index', 'index_ci_build_trace_chunks_on_build_id_and_chunk_index' + + # Drop original FK on the old int4 `build_id` (fk_rails_1013b761f2) + remove_foreign_key TABLE_NAME, name: 'fk_rails_1013b761f2' + # We swapped the columns but the FK for buil_id is still using the temporary name for the build_id_convert_to_bigint column + # So we have to also swap the FK name now that we dropped the other one with the same + rename_constraint(TABLE_NAME, 'fk_rails_1013b761f2_tmp', 'fk_rails_1013b761f2') + end + end +end diff --git a/db/post_migrate/20210721125804_remove_signing_keys_from_packages_debian_project_distributions.rb b/db/post_migrate/20210721125804_remove_signing_keys_from_packages_debian_project_distributions.rb new file mode 100644 index 00000000000..076a238381e --- /dev/null +++ b/db/post_migrate/20210721125804_remove_signing_keys_from_packages_debian_project_distributions.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveSigningKeysFromPackagesDebianProjectDistributions < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def change + remove_column :packages_debian_project_distributions, :encrypted_signing_keys, :text + remove_column :packages_debian_project_distributions, :encrypted_signing_keys_iv, :text + end +end diff --git a/db/post_migrate/20210721125820_remove_signing_keys_from_packages_debian_group_distributions.rb b/db/post_migrate/20210721125820_remove_signing_keys_from_packages_debian_group_distributions.rb new file mode 100644 index 00000000000..4b751c06972 --- /dev/null +++ b/db/post_migrate/20210721125820_remove_signing_keys_from_packages_debian_group_distributions.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveSigningKeysFromPackagesDebianGroupDistributions < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def change + remove_column :packages_debian_group_distributions, :encrypted_signing_keys, :text + remove_column :packages_debian_group_distributions, :encrypted_signing_keys_iv, :text + end +end diff --git a/db/post_migrate/20210721174521_add_non_null_constraint_for_escalation_rule_on_pending_alert_escalations.rb b/db/post_migrate/20210721174521_add_non_null_constraint_for_escalation_rule_on_pending_alert_escalations.rb new file mode 100644 index 00000000000..5d42fd4896d --- /dev/null +++ b/db/post_migrate/20210721174521_add_non_null_constraint_for_escalation_rule_on_pending_alert_escalations.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +class AddNonNullConstraintForEscalationRuleOnPendingAlertEscalations < ActiveRecord::Migration[6.1] + ELAPSED_WHOLE_MINUTES_IN_SECONDS = <<~SQL + ABS(ROUND( + EXTRACT(EPOCH FROM (escalations.process_at - escalations.created_at))/60*60 + )) + SQL + + INSERT_RULES_FOR_ESCALATIONS_WITHOUT_RULES = <<~SQL + INSERT INTO incident_management_escalation_rules (policy_id, oncall_schedule_id, status, elapsed_time_seconds, is_removed) + SELECT + policies.id, + schedule_id, + status, + #{ELAPSED_WHOLE_MINUTES_IN_SECONDS} AS elapsed_time_seconds, + TRUE + FROM incident_management_pending_alert_escalations AS escalations + INNER JOIN incident_management_oncall_schedules AS schedules ON schedules.id = schedule_id + INNER JOIN incident_management_escalation_policies AS policies ON policies.project_id = schedules.project_id + WHERE rule_id IS NULL + GROUP BY policies.id, schedule_id, status, elapsed_time_seconds + ON CONFLICT DO NOTHING; + SQL + + UPDATE_EMPTY_RULE_IDS = <<~SQL + UPDATE incident_management_pending_alert_escalations AS escalations + SET rule_id = rules.id + FROM incident_management_pending_alert_escalations AS through_escalations + INNER JOIN incident_management_oncall_schedules AS schedules ON schedules.id = through_escalations.schedule_id + INNER JOIN incident_management_escalation_policies AS policies ON policies.project_id = schedules.project_id + INNER JOIN incident_management_escalation_rules AS rules ON rules.policy_id = policies.id + WHERE escalations.rule_id IS NULL + AND rules.status = escalations.status + AND rules.oncall_schedule_id = escalations.schedule_id + AND rules.elapsed_time_seconds = #{ELAPSED_WHOLE_MINUTES_IN_SECONDS}; + SQL + + DELETE_LEFTOVER_ESCALATIONS_WITHOUT_RULES = 'DELETE FROM incident_management_pending_alert_escalations WHERE rule_id IS NULL;' + + # For each alert which has a pending escalation without a corresponding rule, + # create a rule with the expected attributes for the project's policy. + # + # Deletes all escalations without rules/policy & adds non-null constraint for rule_id. + def up + exec_query INSERT_RULES_FOR_ESCALATIONS_WITHOUT_RULES + exec_query UPDATE_EMPTY_RULE_IDS + exec_query DELETE_LEFTOVER_ESCALATIONS_WITHOUT_RULES + + change_column_null :incident_management_pending_alert_escalations, :rule_id, false + end + + def down + change_column_null :incident_management_pending_alert_escalations, :rule_id, true + end +end diff --git a/db/post_migrate/20210722010101_reschedule_delete_orphaned_deployments.rb b/db/post_migrate/20210722010101_reschedule_delete_orphaned_deployments.rb new file mode 100644 index 00000000000..b0608f15ce1 --- /dev/null +++ b/db/post_migrate/20210722010101_reschedule_delete_orphaned_deployments.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class RescheduleDeleteOrphanedDeployments < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + MIGRATION = 'DeleteOrphanedDeployments' + BATCH_SIZE = 10_000 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + Gitlab::BackgroundMigration.steal(MIGRATION) + Gitlab::Database::BackgroundMigrationJob.for_migration_class(MIGRATION).delete_all + + queue_background_migration_jobs_by_range_at_intervals( + define_batchable_model('deployments'), + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210722042939_update_issuable_slas_where_issue_closed.rb b/db/post_migrate/20210722042939_update_issuable_slas_where_issue_closed.rb new file mode 100644 index 00000000000..b611b51e3ff --- /dev/null +++ b/db/post_migrate/20210722042939_update_issuable_slas_where_issue_closed.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class UpdateIssuableSlasWhereIssueClosed < ActiveRecord::Migration[6.1] + ISSUE_CLOSED_STATUS = 2 + + class IssuableSla < ActiveRecord::Base + include EachBatch + + self.table_name = 'issuable_slas' + + belongs_to :issue, class_name: 'Issue' + end + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + + has_one :issuable_sla, class_name: 'IssuableSla' + end + + def up + IssuableSla.each_batch(of: 50) do |relation| + relation.joins(:issue) + .where(issues: { state_id: ISSUE_CLOSED_STATUS } ) + .update_all(issuable_closed: true) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210722055217_add_updated_at_index_on_merge_requests.rb b/db/post_migrate/20210722055217_add_updated_at_index_on_merge_requests.rb new file mode 100644 index 00000000000..c66c14d1900 --- /dev/null +++ b/db/post_migrate/20210722055217_add_updated_at_index_on_merge_requests.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddUpdatedAtIndexOnMergeRequests < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + INDEX_NAME = 'index_merge_requests_on_target_project_id_and_updated_at_and_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :merge_requests, [:target_project_id, :updated_at, :id], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :merge_requests, INDEX_NAME + end +end diff --git a/db/post_migrate/20210722155635_finalize_convert_geo_job_artifact_deleted_events_bigint.rb b/db/post_migrate/20210722155635_finalize_convert_geo_job_artifact_deleted_events_bigint.rb new file mode 100644 index 00000000000..7d3809a9dbe --- /dev/null +++ b/db/post_migrate/20210722155635_finalize_convert_geo_job_artifact_deleted_events_bigint.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +class FinalizeConvertGeoJobArtifactDeletedEventsBigint < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'geo_job_artifact_deleted_events' + COLUMN_NAME = 'job_artifact_id' + COLUMN_NAME_CONVERTED = "#{COLUMN_NAME}_convert_to_bigint" + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [[COLUMN_NAME], [COLUMN_NAME_CONVERTED]] + ) + + swap + end + + def down + swap + end + + def swap + old_index_name = 'index_geo_job_artifact_deleted_events_on_job_artifact_id' + + bigint_index_name = 'index_geo_job_artifact_deleted_events_on_job_artifact_id_bigint' + add_concurrent_index TABLE_NAME, COLUMN_NAME_CONVERTED, name: bigint_index_name + + with_lock_retries(raise_on_exhaustion: true) do + execute("LOCK TABLE #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE") + + temp_name = quote_column_name("#{COLUMN_NAME}_tmp") + old_column_name = quote_column_name(COLUMN_NAME) + new_column_name = quote_column_name(COLUMN_NAME_CONVERTED) + + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{old_column_name} TO #{temp_name}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{new_column_name} TO #{old_column_name}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{temp_name} TO #{new_column_name}" + + change_column_default TABLE_NAME, COLUMN_NAME, nil + change_column_default TABLE_NAME, COLUMN_NAME_CONVERTED, 0 + + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(COLUMN_NAME, COLUMN_NAME_CONVERTED) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + execute "DROP INDEX #{old_index_name}" + + rename_index TABLE_NAME, bigint_index_name, old_index_name + end + end +end diff --git a/db/post_migrate/20210727113447_backfill_integrations_type_new.rb b/db/post_migrate/20210727113447_backfill_integrations_type_new.rb new file mode 100644 index 00000000000..8544c236fd7 --- /dev/null +++ b/db/post_migrate/20210727113447_backfill_integrations_type_new.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class BackfillIntegrationsTypeNew < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + MIGRATION = 'BackfillIntegrationsTypeNew' + INTERVAL = 2.minutes + + def up + queue_batched_background_migration( + MIGRATION, + :integrations, + :id, + job_interval: INTERVAL + ) + end + + def down + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(MIGRATION, :integrations, :id, []) + .delete_all + end +end diff --git a/db/post_migrate/20210727175201_remove_unused_columns_from_elastic_reindexing_tasks.rb b/db/post_migrate/20210727175201_remove_unused_columns_from_elastic_reindexing_tasks.rb new file mode 100644 index 00000000000..d4ce1e7024f --- /dev/null +++ b/db/post_migrate/20210727175201_remove_unused_columns_from_elastic_reindexing_tasks.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class RemoveUnusedColumnsFromElasticReindexingTasks < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + remove_column :elastic_reindexing_tasks, :documents_count, :integer + remove_column :elastic_reindexing_tasks, :index_name_from, :text + remove_column :elastic_reindexing_tasks, :index_name_to, :text + remove_column :elastic_reindexing_tasks, :elastic_task, :text + remove_column :elastic_reindexing_tasks, :documents_count_target, :integer + end + + def down + add_column :elastic_reindexing_tasks, :documents_count, :integer + add_column :elastic_reindexing_tasks, :index_name_from, :text + add_column :elastic_reindexing_tasks, :index_name_to, :text + add_column :elastic_reindexing_tasks, :elastic_task, :text + add_column :elastic_reindexing_tasks, :documents_count_target, :integer + + add_text_limit :elastic_reindexing_tasks, :index_name_from, 255 + add_text_limit :elastic_reindexing_tasks, :index_name_to, 255 + add_text_limit :elastic_reindexing_tasks, :elastic_task, 255 + end +end diff --git a/db/post_migrate/20210729163312_re_schedule_latest_pipeline_id_population_with_all_security_related_artifact_types.rb b/db/post_migrate/20210729163312_re_schedule_latest_pipeline_id_population_with_all_security_related_artifact_types.rb new file mode 100644 index 00000000000..8ec608453a0 --- /dev/null +++ b/db/post_migrate/20210729163312_re_schedule_latest_pipeline_id_population_with_all_security_related_artifact_types.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ReScheduleLatestPipelineIdPopulationWithAllSecurityRelatedArtifactTypes < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 100 + MIGRATION = 'PopulateLatestPipelineIds' + + disable_ddl_transaction! + + def up + return unless Gitlab.ee? + + Gitlab::BackgroundMigration.steal(MIGRATION) + + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::PopulateLatestPipelineIds::ProjectSetting.has_vulnerabilities_without_latest_pipeline_set, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + primary_column_name: 'project_id' + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210729192959_drop_ci_test_case_failures_table.rb b/db/post_migrate/20210729192959_drop_ci_test_case_failures_table.rb new file mode 100644 index 00000000000..ad6676a1704 --- /dev/null +++ b/db/post_migrate/20210729192959_drop_ci_test_case_failures_table.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiTestCaseFailuresTable < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + drop_table :ci_test_case_failures + end + + def down + create_table :ci_test_case_failures do |t| + t.datetime_with_timezone :failed_at + t.bigint :test_case_id, null: false + t.bigint :build_id, null: false + + t.index [:test_case_id, :failed_at, :build_id], name: 'index_test_case_failures_unique_columns', unique: true, order: { failed_at: :desc } + t.index :build_id + end + end +end diff --git a/db/post_migrate/20210729193056_drop_ci_test_cases_table.rb b/db/post_migrate/20210729193056_drop_ci_test_cases_table.rb new file mode 100644 index 00000000000..2de1749721d --- /dev/null +++ b/db/post_migrate/20210729193056_drop_ci_test_cases_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class DropCiTestCasesTable < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + drop_table :ci_test_cases + end + + def down + create_table_with_constraints :ci_test_cases do |t| + t.bigint :project_id, null: false + t.text :key_hash, null: false + t.text_limit :key_hash, 64 + + t.index [:project_id, :key_hash], unique: true + end + end +end diff --git a/db/post_migrate/20210730170823_schedule_security_setting_creation.rb b/db/post_migrate/20210730170823_schedule_security_setting_creation.rb new file mode 100644 index 00000000000..cea7b976bf9 --- /dev/null +++ b/db/post_migrate/20210730170823_schedule_security_setting_creation.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class ScheduleSecuritySettingCreation < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + MIGRATION = 'CreateSecuritySetting' + BATCH_SIZE = 1000 + INTERVAL = 5.minutes.to_i + + disable_ddl_transaction! + + def up + return unless Gitlab.ee? # Security Settings available only in EE version + + queue_background_migration_jobs_by_range_at_intervals( + define_batchable_model('projects'), + MIGRATION, + INTERVAL, + batch_size: BATCH_SIZE + ) + end + + # We're adding data so no need for rollback + def down + end +end diff --git a/db/post_migrate/20210802043253_finalize_push_event_payloads_bigint_conversion_3.rb b/db/post_migrate/20210802043253_finalize_push_event_payloads_bigint_conversion_3.rb new file mode 100644 index 00000000000..4b825ae72ba --- /dev/null +++ b/db/post_migrate/20210802043253_finalize_push_event_payloads_bigint_conversion_3.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +class FinalizePushEventPayloadsBigintConversion3 < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'push_event_payloads' + INDEX_NAME = 'index_push_event_payloads_on_event_id_convert_to_bigint' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'event_id', + job_arguments: [["event_id"], ["event_id_convert_to_bigint"]] + ) + + return if already_swapped? + + swap_columns + end + + def down + swap_columns + end + + private + + def already_swapped? + push_event_payloads_columns = columns(TABLE_NAME) + event_id = push_event_payloads_columns.find {|c| c.name == 'event_id'} + event_id_convert_to_bigint = push_event_payloads_columns.find {|c| c.name == 'event_id_convert_to_bigint'} + + event_id.sql_type == 'bigint' && event_id_convert_to_bigint.sql_type == 'integer' + end + + def swap_columns + add_concurrent_index TABLE_NAME, :event_id_convert_to_bigint, unique: true, name: INDEX_NAME + + # Add a foreign key on `event_id_convert_to_bigint` before we swap the columns and drop the old FK (fk_36c74129da) + add_concurrent_foreign_key TABLE_NAME, :events, column: :event_id_convert_to_bigint, + on_delete: :cascade, reverse_lock_order: true + + with_lock_retries(raise_on_exhaustion: true) do + # We'll need ACCESS EXCLUSIVE lock on the related tables, + # lets make sure it can be acquired from the start. + # Lock order should be + # 1. events + # 2. push_event_payloads + # in order to match the order in EventCreateService#create_push_event, + # and avoid deadlocks. + execute "LOCK TABLE events, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + # Swap column names + temp_name = 'event_id_tmp' + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:event_id)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:event_id_convert_to_bigint)} TO #{quote_column_name(:event_id)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:event_id_convert_to_bigint)}" + + # We need to update the trigger function in order to make PostgreSQL to + # regenerate the execution plan for it. This is to avoid type mismatch errors like + # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)" + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:event_id, :event_id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # Swap defaults + change_column_default TABLE_NAME, :event_id, nil + change_column_default TABLE_NAME, :event_id_convert_to_bigint, 0 + + # Swap PK constraint + execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT push_event_payloads_pkey" + rename_index TABLE_NAME, INDEX_NAME, 'push_event_payloads_pkey' + execute "ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT push_event_payloads_pkey PRIMARY KEY USING INDEX push_event_payloads_pkey" + + # Drop original FK on the old int4 `event_id` (fk_36c74129da) + remove_foreign_key TABLE_NAME, name: concurrent_foreign_key_name(TABLE_NAME, :event_id) + # We swapped the columns but the FK for event_id is still using the old name for the event_id_convert_to_bigint column + # So we have to also swap the FK name now that we dropped the other one with the same + rename_constraint( + TABLE_NAME, + concurrent_foreign_key_name(TABLE_NAME, :event_id_convert_to_bigint), + concurrent_foreign_key_name(TABLE_NAME, :event_id) + ) + end + end +end diff --git a/db/post_migrate/20210802112233_add_index_issues_on_project_id_and_state_id_and_created_at_and_id.rb b/db/post_migrate/20210802112233_add_index_issues_on_project_id_and_state_id_and_created_at_and_id.rb new file mode 100644 index 00000000000..318d9e8a218 --- /dev/null +++ b/db/post_migrate/20210802112233_add_index_issues_on_project_id_and_state_id_and_created_at_and_id.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddIndexIssuesOnProjectIdAndStateIdAndCreatedAtAndId < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_issues_on_project_id_and_state_id_and_created_at_and_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :issues, [:project_id, :state_id, :created_at, :id], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :issues, INDEX_NAME + end +end diff --git a/db/post_migrate/20210802131812_finalize_convert_deployments_bigint.rb b/db/post_migrate/20210802131812_finalize_convert_deployments_bigint.rb new file mode 100644 index 00000000000..31afb7e0a29 --- /dev/null +++ b/db/post_migrate/20210802131812_finalize_convert_deployments_bigint.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class FinalizeConvertDeploymentsBigint < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'deployments' + COLUMN_NAME = 'deployable_id' + COLUMN_NAME_BIGINT = "#{COLUMN_NAME}_convert_to_bigint" + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [[COLUMN_NAME], [COLUMN_NAME_BIGINT]] + ) + + swap + end + + def down + swap + end + + def swap + old_index_name = 'index_deployments_on_deployable_type_and_deployable_id' + bigint_index_name = 'index_deployments_on_deployable_type_and_deployable_id_bigint' + add_concurrent_index TABLE_NAME, ['deployable_type', COLUMN_NAME_BIGINT], name: bigint_index_name + + with_lock_retries(raise_on_exhaustion: true) do + # Swap columns + temp_name = "#{COLUMN_NAME}_tmp" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{quote_column_name(COLUMN_NAME)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{quote_column_name(COLUMN_NAME_BIGINT)} TO #{quote_column_name(COLUMN_NAME)}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(COLUMN_NAME_BIGINT)}" + + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(COLUMN_NAME, COLUMN_NAME_BIGINT) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + execute "DROP INDEX #{old_index_name}" + rename_index TABLE_NAME, bigint_index_name, old_index_name + end + end +end diff --git a/db/post_migrate/20210804150624_remove_cluster_image_scanning_from_approval_project_rules.rb b/db/post_migrate/20210804150624_remove_cluster_image_scanning_from_approval_project_rules.rb new file mode 100644 index 00000000000..a2736a563c7 --- /dev/null +++ b/db/post_migrate/20210804150624_remove_cluster_image_scanning_from_approval_project_rules.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class RemoveClusterImageScanningFromApprovalProjectRules < ActiveRecord::Migration[6.1] + def up + execute("update approval_project_rules set scanners = array_remove(scanners, 'cluster_image_scanning') where scanners @> '{cluster_image_scanning}'") + end + + def down + # nothing to do here + end +end diff --git a/db/post_migrate/20210804151444_prepare_indexes_for_ci_job_artifact_bigint_conversion.rb b/db/post_migrate/20210804151444_prepare_indexes_for_ci_job_artifact_bigint_conversion.rb new file mode 100644 index 00000000000..8115465e311 --- /dev/null +++ b/db/post_migrate/20210804151444_prepare_indexes_for_ci_job_artifact_bigint_conversion.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class PrepareIndexesForCiJobArtifactBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + prepare_async_index :ci_job_artifacts, :id_convert_to_bigint, unique: true, + name: :index_ci_job_artifact_on_id_convert_to_bigint + + prepare_async_index :ci_job_artifacts, [:project_id, :id_convert_to_bigint], where: 'file_type = 18', + name: :index_ci_job_artifacts_for_terraform_reports_bigint + + prepare_async_index :ci_job_artifacts, :id_convert_to_bigint, where: 'file_type = 18', + name: :index_ci_job_artifacts_id_for_terraform_reports_bigint + + prepare_async_index :ci_job_artifacts, [:expire_at, :job_id_convert_to_bigint], + name: :index_ci_job_artifacts_on_expire_at_and_job_id_bigint + + prepare_async_index :ci_job_artifacts, [:job_id_convert_to_bigint, :file_type], unique: true, + name: :index_ci_job_artifacts_on_job_id_and_file_type_bigint + end + + def down + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifacts_on_job_id_and_file_type_bigint + + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifacts_on_expire_at_and_job_id_bigint + + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifacts_id_for_terraform_reports_bigint + + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifacts_for_terraform_reports_bigint + + unprepare_async_index_by_name :ci_job_artifacts, :index_ci_job_artifact_on_id_convert_to_bigint + end +end diff --git a/db/post_migrate/20210804153307_prepare_indexes_for_tagging_bigint_conversion.rb b/db/post_migrate/20210804153307_prepare_indexes_for_tagging_bigint_conversion.rb new file mode 100644 index 00000000000..98f90bafce3 --- /dev/null +++ b/db/post_migrate/20210804153307_prepare_indexes_for_tagging_bigint_conversion.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class PrepareIndexesForTaggingBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + prepare_async_index :taggings, :id_convert_to_bigint, unique: true, + name: :index_taggings_on_id_convert_to_bigint + + prepare_async_index :taggings, [:taggable_id_convert_to_bigint, :taggable_type], + name: :i_taggings_on_taggable_id_convert_to_bigint_and_taggable_type + + prepare_async_index :taggings, [:taggable_id_convert_to_bigint, :taggable_type, :context], + name: :i_taggings_on_taggable_bigint_and_taggable_type_and_context + + prepare_async_index :taggings, [:tag_id, :taggable_id_convert_to_bigint, :taggable_type, :context, :tagger_id, :tagger_type], + unique: true, name: :taggings_idx_tmp + end + + def down + unprepare_async_index_by_name :taggings, :taggings_idx_tmp + + unprepare_async_index_by_name :taggings, :i_taggings_on_taggable_bigint_and_taggable_type_and_context + + unprepare_async_index_by_name :taggings, :i_taggings_on_taggable_id_convert_to_bigint_and_taggable_type + + unprepare_async_index_by_name :taggings, :index_taggings_on_id_convert_to_bigint + end +end diff --git a/db/post_migrate/20210804154407_prepare_indexes_for_ci_stage_bigint_conversion.rb b/db/post_migrate/20210804154407_prepare_indexes_for_ci_stage_bigint_conversion.rb new file mode 100644 index 00000000000..82af595b2d3 --- /dev/null +++ b/db/post_migrate/20210804154407_prepare_indexes_for_ci_stage_bigint_conversion.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class PrepareIndexesForCiStageBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + prepare_async_index :ci_stages, :id_convert_to_bigint, unique: true, + name: :index_ci_stages_on_id_convert_to_bigint + + prepare_async_index :ci_stages, [:pipeline_id, :id_convert_to_bigint], where: 'status in (0, 1, 2, 8, 9, 10)', + name: :index_ci_stages_on_pipeline_id_and_id_convert_to_bigint + end + + def down + unprepare_async_index_by_name :ci_stages, :index_ci_stages_on_pipeline_id_and_id_convert_to_bigint + + unprepare_async_index_by_name :ci_stages, :index_ci_stages_on_id_convert_to_bigint + end +end diff --git a/db/post_migrate/20210805131510_finalize_ci_builds_runner_session_bigint_conversion.rb b/db/post_migrate/20210805131510_finalize_ci_builds_runner_session_bigint_conversion.rb new file mode 100644 index 00000000000..9706f0036f3 --- /dev/null +++ b/db/post_migrate/20210805131510_finalize_ci_builds_runner_session_bigint_conversion.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +class FinalizeCiBuildsRunnerSessionBigintConversion < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'ci_builds_runner_session' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [['build_id'], ['build_id_convert_to_bigint']] + ) + + swap + end + + def down + swap + end + + private + + def swap + add_concurrent_index TABLE_NAME, :build_id_convert_to_bigint, unique: true, name: 'index_ci_builds_runner_session_on_build_id_convert_to_bigint' + + # Add a foreign key on `build_id_convert_to_bigint` before we swap the columns and drop the old FK () + add_concurrent_foreign_key TABLE_NAME, :ci_builds, column: :build_id_convert_to_bigint, on_delete: :cascade, name: 'fk_rails_70707857d3_tmp', reverse_lock_order: true + + with_lock_retries(raise_on_exhaustion: true) do + # We'll need ACCESS EXCLUSIVE lock on the related tables, + # lets make sure it can be acquired from the start + execute "LOCK TABLE ci_builds, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + # Swap column names + temp_name = 'build_id_tmp' + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:build_id)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:build_id_convert_to_bigint)} TO #{quote_column_name(:build_id)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:build_id_convert_to_bigint)}" + + # We need to update the trigger function in order to make PostgreSQL to + # regenerate the execution plan for it. This is to avoid type mismatch errors like + # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)" + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:build_id, :build_id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # Swap defaults + change_column_default TABLE_NAME, :build_id, nil + change_column_default TABLE_NAME, :build_id_convert_to_bigint, 0 + + # Rename the index on the `bigint` column to match the new column name + # (we already hold an exclusive lock, so no need to use DROP INDEX CONCURRENTLY here) + execute 'DROP INDEX index_ci_builds_runner_session_on_build_id' + rename_index TABLE_NAME, 'index_ci_builds_runner_session_on_build_id_convert_to_bigint', 'index_ci_builds_runner_session_on_build_id' + + # Drop original FK on the old int4 `build_id` (fk_rails_70707857d3) + remove_foreign_key TABLE_NAME, name: 'fk_rails_70707857d3' + + # We swapped the columns but the FK for buil_id is still using the temporary name for the build_id_convert_to_bigint column + # So we have to also swap the FK name now that we dropped the other one with the same + rename_constraint(TABLE_NAME, 'fk_rails_70707857d3_tmp', 'fk_rails_70707857d3') + end + end +end diff --git a/db/post_migrate/20210805192450_update_trial_plans_ci_daily_pipeline_schedule_triggers.rb b/db/post_migrate/20210805192450_update_trial_plans_ci_daily_pipeline_schedule_triggers.rb new file mode 100644 index 00000000000..9552058dd73 --- /dev/null +++ b/db/post_migrate/20210805192450_update_trial_plans_ci_daily_pipeline_schedule_triggers.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +class UpdateTrialPlansCiDailyPipelineScheduleTriggers < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + PREMIUM_TRIAL = 'premium_trial' + ULTIMATE_TRIAL = 'ultimate_trial' + EVERY_5_MINUTES = (1.day.in_minutes / 5).to_i + + class Plan < ActiveRecord::Base + self.table_name = 'plans' + self.inheritance_column = :_type_disabled + + has_one :limits, class_name: 'PlanLimits' + end + + class PlanLimits < ActiveRecord::Base + self.table_name = 'plan_limits' + self.inheritance_column = :_type_disabled + + belongs_to :plan + end + + def plan_limits_present? + premium_trial_plan = Plan.find_by(name: PREMIUM_TRIAL) + ultimate_trial_plan = Plan.find_by(name: ULTIMATE_TRIAL) + + premium_trial_plan && premium_trial_plan.limits && ultimate_trial_plan && ultimate_trial_plan.limits + end + + def up + return unless Gitlab.dev_env_or_com? + + if plan_limits_present? + create_or_update_plan_limit('ci_daily_pipeline_schedule_triggers', PREMIUM_TRIAL, EVERY_5_MINUTES) + create_or_update_plan_limit('ci_daily_pipeline_schedule_triggers', ULTIMATE_TRIAL, EVERY_5_MINUTES) + end + end + + def down + return unless Gitlab.dev_env_or_com? + + if plan_limits_present? + create_or_update_plan_limit('ci_daily_pipeline_schedule_triggers', PREMIUM_TRIAL, 0) + create_or_update_plan_limit('ci_daily_pipeline_schedule_triggers', ULTIMATE_TRIAL, 0) + end + end +end diff --git a/db/post_migrate/20210806011811_schedule_backfill_draft_column_on_merge_requests_rerun.rb b/db/post_migrate/20210806011811_schedule_backfill_draft_column_on_merge_requests_rerun.rb new file mode 100644 index 00000000000..90bca79624d --- /dev/null +++ b/db/post_migrate/20210806011811_schedule_backfill_draft_column_on_merge_requests_rerun.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ScheduleBackfillDraftColumnOnMergeRequestsRerun < ActiveRecord::Migration[6.1] + def up + # noop + # + end + + def down + # noop + # + end +end diff --git a/db/post_migrate/20210809123658_orphaned_invite_tokens_cleanup.rb b/db/post_migrate/20210809123658_orphaned_invite_tokens_cleanup.rb new file mode 100644 index 00000000000..f774db73eaa --- /dev/null +++ b/db/post_migrate/20210809123658_orphaned_invite_tokens_cleanup.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class OrphanedInviteTokensCleanup < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TMP_INDEX_NAME = 'tmp_idx_orphaned_invite_tokens' + QUERY_CONDITION = "invite_token IS NOT NULL and invite_accepted_at IS NOT NULL and invite_accepted_at < created_at" + + def up + membership = define_batchable_model('members') + + add_concurrent_index('members', :id, where: QUERY_CONDITION, name: TMP_INDEX_NAME) + + membership.where(QUERY_CONDITION).pluck(:id).each_slice(10) do |group| + membership.where(id: group).where(QUERY_CONDITION).update_all(invite_token: nil) + end + + remove_concurrent_index_by_name('members', TMP_INDEX_NAME) + end + + def down + remove_concurrent_index_by_name('members', TMP_INDEX_NAME) if index_exists_by_name?('members', TMP_INDEX_NAME) + + # This migration is irreversible + end +end diff --git a/db/post_migrate/20210809143931_finalize_job_id_conversion_to_bigint_for_ci_job_artifacts.rb b/db/post_migrate/20210809143931_finalize_job_id_conversion_to_bigint_for_ci_job_artifacts.rb new file mode 100644 index 00000000000..bb12045b1de --- /dev/null +++ b/db/post_migrate/20210809143931_finalize_job_id_conversion_to_bigint_for_ci_job_artifacts.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +class FinalizeJobIdConversionToBigintForCiJobArtifacts < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + TABLE_NAME = 'ci_job_artifacts' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [%w[id job_id], %w[id_convert_to_bigint job_id_convert_to_bigint]] + ) + + swap + end + + def down + swap + end + + private + + def swap + # This is to replace the existing "index_ci_job_artifacts_on_expire_at_and_job_id" btree (expire_at, job_id) + add_concurrent_index TABLE_NAME, [:expire_at, :job_id_convert_to_bigint], name: 'index_ci_job_artifacts_on_expire_at_and_job_id_bigint' + # This is to replace the existing "index_ci_job_artifacts_on_job_id_and_file_type" btree (job_id, file_type) + add_concurrent_index TABLE_NAME, [:job_id_convert_to_bigint, :file_type], name: 'index_ci_job_artifacts_on_job_id_and_file_type_bigint', unique: true + + # # Add a FK on `job_id_convert_to_bigint` to `ci_builds(id)`, the old FK (fk_rails_c5137cb2c1) + # # is removed below since it won't be dropped automatically. + fk_ci_builds_job_id = concurrent_foreign_key_name(TABLE_NAME, :job_id, prefix: 'fk_rails_') + fk_ci_builds_job_id_tmp = "#{fk_ci_builds_job_id}_tmp" + + add_concurrent_foreign_key TABLE_NAME, :ci_builds, + column: :job_id_convert_to_bigint, + name: fk_ci_builds_job_id_tmp, + on_delete: :cascade, + reverse_lock_order: true + + with_lock_retries(raise_on_exhaustion: true) do + # We'll need ACCESS EXCLUSIVE lock on the related tables, + # lets make sure it can be acquired from the start + + execute "LOCK TABLE ci_builds, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + temp_name = 'job_id_tmp' + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:job_id)} TO #{quote_column_name(temp_name)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:job_id_convert_to_bigint)} TO #{quote_column_name(:job_id)}" + execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:job_id_convert_to_bigint)}" + + # We need to update the trigger function in order to make PostgreSQL to + # regenerate the execution plan for it. This is to avoid type mismatch errors like + # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)" + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name([:id, :job_id], [:id_convert_to_bigint, :job_id_convert_to_bigint]) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + # Swap defaults + change_column_default TABLE_NAME, :job_id, nil + change_column_default TABLE_NAME, :job_id_convert_to_bigint, 0 + + # Rename the rest of the indexes (we already hold an exclusive lock, so no need to use DROP INDEX CONCURRENTLY here + execute 'DROP INDEX index_ci_job_artifacts_on_expire_at_and_job_id' + rename_index TABLE_NAME, 'index_ci_job_artifacts_on_expire_at_and_job_id_bigint', 'index_ci_job_artifacts_on_expire_at_and_job_id' + execute 'DROP INDEX index_ci_job_artifacts_on_job_id_and_file_type' + rename_index TABLE_NAME, 'index_ci_job_artifacts_on_job_id_and_file_type_bigint', 'index_ci_job_artifacts_on_job_id_and_file_type' + + # Drop original FK on the old int4 `job_id` (fk_rails_c5137cb2c1) + remove_foreign_key TABLE_NAME, name: fk_ci_builds_job_id + + # We swapped the columns but the FK for job_id is still using the temporary name for the job_id_convert_to_bigint column + # So we have to also swap the FK name now that we dropped the other one with the same + rename_constraint(TABLE_NAME, fk_ci_builds_job_id_tmp, fk_ci_builds_job_id) + end + end +end diff --git a/db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb b/db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb new file mode 100644 index 00000000000..106d1430dca --- /dev/null +++ b/db/post_migrate/20210811214811_schedule_copy_ci_builds_columns_to_security_scans.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ScheduleCopyCiBuildsColumnsToSecurityScans < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + # no-op as we found an issue with bg migration, we fixed it and rescheduling it again. + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210812145010_reset_job_token_scope_enabled_again.rb b/db/post_migrate/20210812145010_reset_job_token_scope_enabled_again.rb new file mode 100644 index 00000000000..6b4c69efa2b --- /dev/null +++ b/db/post_migrate/20210812145010_reset_job_token_scope_enabled_again.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class ResetJobTokenScopeEnabledAgain < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + with_lock_retries do + remove_column :project_ci_cd_settings, :job_token_scope_enabled + add_column :project_ci_cd_settings, :job_token_scope_enabled, :boolean, default: false, null: false + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210813195518_schedule_recalculate_uuid_on_vulnerabilities_occurrences3.rb b/db/post_migrate/20210813195518_schedule_recalculate_uuid_on_vulnerabilities_occurrences3.rb new file mode 100644 index 00000000000..c59c71708ca --- /dev/null +++ b/db/post_migrate/20210813195518_schedule_recalculate_uuid_on_vulnerabilities_occurrences3.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ScheduleRecalculateUuidOnVulnerabilitiesOccurrences3 < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + MIGRATION = 'RecalculateVulnerabilitiesOccurrencesUuid' + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 2_500 + + disable_ddl_transaction! + + def up + # Make sure that RemoveDuplicateVulnerabilitiesFindings has finished running + # so that we don't run into duplicate UUID issues + Gitlab::BackgroundMigration.steal('RemoveDuplicateVulnerabilitiesFindings') + + say "Scheduling #{MIGRATION} jobs" + queue_background_migration_jobs_by_range_at_intervals( + define_batchable_model('vulnerability_occurrences'), + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb b/db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb new file mode 100644 index 00000000000..b8f843fdd50 --- /dev/null +++ b/db/post_migrate/20210816183304_schedule_copy_ci_builds_columns_to_security_scans2.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class ScheduleCopyCiBuildsColumnsToSecurityScans2 < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + INTERVAL = 2.minutes.to_i + BATCH_SIZE = 5_000 + MIGRATION = 'CopyCiBuildsColumnsToSecurityScans' + + disable_ddl_transaction! + + class SecurityScan < ActiveRecord::Base + include EachBatch + + self.table_name = 'security_scans' + end + + def up + SecurityScan.reset_column_information + + delete_job_tracking(MIGRATION, status: %w[pending succeeded]) + + queue_background_migration_jobs_by_range_at_intervals( + SecurityScan, + MIGRATION, + INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # noop + end +end |