diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2024-01-16 13:42:19 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2024-01-16 13:42:19 +0300 |
commit | 84d1bd786125c1c14a3ba5f63e38a4cc736a9027 (patch) | |
tree | f550fa965f507077e20dbb6d61a8269a99ef7107 /db/post_migrate | |
parent | 3a105e36e689f7b75482236712f1a47fd5a76814 (diff) |
Add latest changes from gitlab-org/gitlab@16-8-stable-eev16.8.0-rc42
Diffstat (limited to 'db/post_migrate')
74 files changed, 1910 insertions, 0 deletions
diff --git a/db/post_migrate/20231204042048_finalize_system_note_metadata_bigint_conversion.rb b/db/post_migrate/20231204042048_finalize_system_note_metadata_bigint_conversion.rb new file mode 100644 index 00000000000..cd6550d6bad --- /dev/null +++ b/db/post_migrate/20231204042048_finalize_system_note_metadata_bigint_conversion.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class FinalizeSystemNoteMetadataBigintConversion < Gitlab::Database::Migration[2.2] + include Gitlab::Database::MigrationHelpers::ConvertToBigint + + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + milestone '16.8' + + TABLE_NAME = :system_note_metadata + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [['id'], ['id_convert_to_bigint']] + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20231204095802_change_i_code_review_create_mr_keys_from_redis_hll_to_redis.rb b/db/post_migrate/20231204095802_change_i_code_review_create_mr_keys_from_redis_hll_to_redis.rb new file mode 100644 index 00000000000..d006e0f3121 --- /dev/null +++ b/db/post_migrate/20231204095802_change_i_code_review_create_mr_keys_from_redis_hll_to_redis.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class ChangeICodeReviewCreateMrKeysFromRedisHllToRedis < Gitlab::Database::Migration[2.2] + milestone '16.8' + + def up + # no-op + # + # Removed due to https://gitlab.com/gitlab-com/gl-infra/production/-/issues/17321 + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20231205200847_rollback_user_foreign_key_from_in_product_marketing_emails.rb b/db/post_migrate/20231205200847_rollback_user_foreign_key_from_in_product_marketing_emails.rb new file mode 100644 index 00000000000..fe4509204db --- /dev/null +++ b/db/post_migrate/20231205200847_rollback_user_foreign_key_from_in_product_marketing_emails.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RollbackUserForeignKeyFromInProductMarketingEmails < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + def up + with_lock_retries do + remove_foreign_key_if_exists :in_product_marketing_emails, :users, name: 'fk_35c9101b63' + end + end + + def down + add_concurrent_foreign_key :in_product_marketing_emails, :users, column: :user_id, name: 'fk_35c9101b63', + on_delete: :cascade + end +end diff --git a/db/post_migrate/20231205200925_drop_in_product_marketing_emails.rb b/db/post_migrate/20231205200925_drop_in_product_marketing_emails.rb new file mode 100644 index 00000000000..5a121793333 --- /dev/null +++ b/db/post_migrate/20231205200925_drop_in_product_marketing_emails.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class DropInProductMarketingEmails < Gitlab::Database::Migration[2.2] + milestone '16.8' + + def up + drop_table :in_product_marketing_emails + end + + def down + create_table :in_product_marketing_emails do |t| + t.bigint :user_id, null: false + t.datetime_with_timezone :cta_clicked_at + t.integer :track, null: false, limit: 2 + t.integer :series, null: false, limit: 2 + + t.timestamps_with_timezone + end + + add_index :in_product_marketing_emails, :user_id + add_index :in_product_marketing_emails, %i[user_id track series], unique: true, + name: 'index_in_product_marketing_emails_on_user_track_series' + add_index :in_product_marketing_emails, %i[track series id cta_clicked_at], + name: 'index_in_product_marketing_emails_on_track_series_id_clicked' + end +end diff --git a/db/post_migrate/20231207145335_cleanup_group_level_work_items.rb b/db/post_migrate/20231207145335_cleanup_group_level_work_items.rb new file mode 100644 index 00000000000..d52f0518f44 --- /dev/null +++ b/db/post_migrate/20231207145335_cleanup_group_level_work_items.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +class CleanupGroupLevelWorkItems < Gitlab::Database::Migration[2.2] + milestone '16.8' + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + BATCH_SIZE = 100 + + class MigrationIssue < MigrationRecord + self.table_name = :issues + + include EachBatch + end + + class MigrationNote < MigrationRecord + self.table_name = :notes + + include EachBatch + end + + class MigrationLabelLink < MigrationRecord + self.table_name = :label_links + + include EachBatch + end + + class MigrationTodo < MigrationRecord + self.table_name = :todos + + include EachBatch + end + + def up + MigrationIssue.where(project_id: nil).each_batch(of: BATCH_SIZE) do |batch| + logger.info("deleting #{batch.size} issues at group level: #{batch.pluck(:id)}") + + # cleaning up notes for the batch of issues + MigrationNote.where(noteable_type: 'Issue', noteable_id: batch).each_batch(of: BATCH_SIZE) do |note_batch| + logger.info("deleting #{note_batch.size} notes for issues at group level: #{note_batch.pluck(:id)}") + note_batch.delete_all + end + + # cleaning up label links for the batch of issues + MigrationLabelLink.where(target_type: 'Issue', target_id: batch).each_batch(of: BATCH_SIZE) do |label_link_batch| + logger.info( + "deleting #{label_link_batch.size} label links for issues at group level: #{label_link_batch.pluck(:id)}" + ) + label_link_batch.delete_all + end + + # cleaning up todos for the batch of issues + MigrationTodo.where(target_type: 'Issue', target_id: batch).each_batch(of: BATCH_SIZE) do |todo_batch| + logger.info("deleting #{todo_batch.size} todos for issues at group level: #{todo_batch.pluck(:id)}") + todo_batch.delete_all + end + + batch.delete_all + end + end + + def down + # no-op + end + + private + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end +end diff --git a/db/post_migrate/20231207155340_prepare_iid_namespace_unique_index_in_issues.rb b/db/post_migrate/20231207155340_prepare_iid_namespace_unique_index_in_issues.rb new file mode 100644 index 00000000000..a0149a79f83 --- /dev/null +++ b/db/post_migrate/20231207155340_prepare_iid_namespace_unique_index_in_issues.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PrepareIidNamespaceUniqueIndexInIssues < Gitlab::Database::Migration[2.2] + INDEX_NAME = 'index_issues_on_namespace_id_iid_unique' + + milestone '16.8' + + # TODO: Index to be created synchronously in https://gitlab.com/gitlab-org/gitlab/-/issues/435856 + def up + prepare_async_index :issues, [:namespace_id, :iid], unique: true, name: INDEX_NAME + end + + def down + unprepare_async_index :issues, [:namespace_id, :iid], unique: true, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20231207211418_add_iid_namespace_unique_index_to_issues.rb b/db/post_migrate/20231207211418_add_iid_namespace_unique_index_to_issues.rb new file mode 100644 index 00000000000..4c2439f33ea --- /dev/null +++ b/db/post_migrate/20231207211418_add_iid_namespace_unique_index_to_issues.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIidNamespaceUniqueIndexToIssues < Gitlab::Database::Migration[2.2] + INDEX_NAME = 'index_issues_on_namespace_id_iid_unique' + + disable_ddl_transaction! + + milestone '16.8' + + def up + add_concurrent_index :issues, [:namespace_id, :iid], name: INDEX_NAME, unique: true + end + + def down + remove_concurrent_index_by_name :issues, INDEX_NAME + end +end diff --git a/db/post_migrate/20231207221418_drop_namespace_id_index_on_issues.rb b/db/post_migrate/20231207221418_drop_namespace_id_index_on_issues.rb new file mode 100644 index 00000000000..38df861de00 --- /dev/null +++ b/db/post_migrate/20231207221418_drop_namespace_id_index_on_issues.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class DropNamespaceIdIndexOnIssues < Gitlab::Database::Migration[2.2] + INDEX_NAME = 'index_issues_on_namespace_id' + + disable_ddl_transaction! + + milestone '16.8' + + def up + remove_concurrent_index_by_name :issues, INDEX_NAME + end + + def down + add_concurrent_index :issues, :namespace_id, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20231211154654_add_work_items_rolledup_dates_widget.rb b/db/post_migrate/20231211154654_add_work_items_rolledup_dates_widget.rb new file mode 100644 index 00000000000..b45967920fe --- /dev/null +++ b/db/post_migrate/20231211154654_add_work_items_rolledup_dates_widget.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class AddWorkItemsRolledupDatesWidget < Gitlab::Database::Migration[2.2] + milestone '16.8' + + class WorkItemType < MigrationRecord + self.table_name = 'work_item_types' + end + + class WidgetDefinition < MigrationRecord + self.table_name = 'work_item_widget_definitions' + end + + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + WIDGET_NAME = 'Rolledup dates' + WIDGET_ENUM_VALUE = 19 + WORK_ITEM_TYPES = ['Epic'].freeze + + def up + widgets = WORK_ITEM_TYPES.each_with_object([]) do |type_name, result| + type = WorkItemType.find_by_name_and_namespace_id(type_name, nil) + + unless type + Gitlab::AppLogger.warn("type #{type_name} is missing, not adding widget") + + next + end + + result << { + work_item_type_id: type.id, + name: WIDGET_NAME, + widget_type: WIDGET_ENUM_VALUE + } + end + + return if widgets.empty? + + WidgetDefinition.upsert_all( + widgets, + unique_by: :index_work_item_widget_definitions_on_default_witype_and_name + ) + end + + def down + WidgetDefinition.where(name: WIDGET_NAME).delete_all + end +end diff --git a/db/post_migrate/20231212135235_queue_backfill_vs_code_settings_version.rb b/db/post_migrate/20231212135235_queue_backfill_vs_code_settings_version.rb new file mode 100644 index 00000000000..db2704c3c33 --- /dev/null +++ b/db/post_migrate/20231212135235_queue_backfill_vs_code_settings_version.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueBackfillVsCodeSettingsVersion < Gitlab::Database::Migration[2.2] + milestone '16.7' + + MIGRATION = "BackfillVsCodeSettingsVersion" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :vs_code_settings, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :vs_code_settings, :id, []) + end +end diff --git a/db/post_migrate/20231213014658_remove_plain_text_credit_card_columns.rb b/db/post_migrate/20231213014658_remove_plain_text_credit_card_columns.rb new file mode 100644 index 00000000000..c1ecf63012d --- /dev/null +++ b/db/post_migrate/20231213014658_remove_plain_text_credit_card_columns.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class RemovePlainTextCreditCardColumns < Gitlab::Database::Migration[2.2] + milestone '16.8' + + disable_ddl_transaction! + + FULL_MATCH_INDEX_NAME = 'index_user_credit_card_validations_meta_data_full_match_lower' + FULL_MATCH_FIELDS = 'lower(holder_name), expiration_date, last_digits, credit_card_validated_at' + + PARTIAL_MATCH_INDEX_NAME = 'index_user_credit_card_validations_meta_data_partial_match' + PARTIAL_MATCH_FIELDS = 'expiration_date, last_digits, network, credit_card_validated_at' + + def up + with_lock_retries do + remove_column :user_credit_card_validations, :holder_name, if_exists: true + remove_column :user_credit_card_validations, :network, if_exists: true + remove_column :user_credit_card_validations, :expiration_date, if_exists: true + remove_column :user_credit_card_validations, :last_digits, if_exists: true + end + end + + def down + add_column :user_credit_card_validations, :holder_name, :text, if_not_exists: true + add_column :user_credit_card_validations, :network, :text, if_not_exists: true + add_column :user_credit_card_validations, :expiration_date, :date, if_not_exists: true + add_column :user_credit_card_validations, :last_digits, :integer, limit: 2, if_not_exists: true + + add_text_limit :user_credit_card_validations, :holder_name, 50 + add_text_limit :user_credit_card_validations, :network, 32 + add_check_constraint :user_credit_card_validations, 'last_digits BETWEEN 0 AND 9999', + 'credit_card_last_digits_range' + + add_concurrent_index :user_credit_card_validations, FULL_MATCH_FIELDS, name: FULL_MATCH_INDEX_NAME + add_concurrent_index :user_credit_card_validations, PARTIAL_MATCH_FIELDS, name: PARTIAL_MATCH_INDEX_NAME + end +end diff --git a/db/post_migrate/20231213064314_remove_ci_editor_ai_conversation_messages_table.rb b/db/post_migrate/20231213064314_remove_ci_editor_ai_conversation_messages_table.rb new file mode 100644 index 00000000000..b108805a22d --- /dev/null +++ b/db/post_migrate/20231213064314_remove_ci_editor_ai_conversation_messages_table.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class RemoveCiEditorAiConversationMessagesTable < Gitlab::Database::Migration[2.2] + milestone '16.8' + + def up + drop_table :ci_editor_ai_conversation_messages, if_exists: true + end + + def down + create_table :ci_editor_ai_conversation_messages do |t| + t.bigint :user_id, + null: false + t.bigint :project_id, + null: false + t.timestamps_with_timezone null: false + t.text :role, limit: 100, + null: false + t.text :content, limit: 16384, + null: true + t.text :async_errors, array: true, null: false, default: [] + + t.index [:user_id, :project_id, :created_at], + name: :index_ci_editor_ai_messages_on_user_project_and_created_at + + t.index :project_id, + name: :index_ci_editor_ai_messages_project_id + + t.index :created_at, + name: :index_ci_editor_ai_messages_created_at + end + end +end diff --git a/db/post_migrate/20231213080216_finalize_merge_request_diffs_project_id_backfill.rb b/db/post_migrate/20231213080216_finalize_merge_request_diffs_project_id_backfill.rb new file mode 100644 index 00000000000..cb83b00b2a2 --- /dev/null +++ b/db/post_migrate/20231213080216_finalize_merge_request_diffs_project_id_backfill.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class FinalizeMergeRequestDiffsProjectIdBackfill < Gitlab::Database::Migration[2.2] + milestone '16.8' + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'BackfillMergeRequestDiffsProjectId', + table_name: :merge_request_diffs, + column_name: :id, + job_arguments: [], + finalize: true + ) + end + + def down; end +end diff --git a/db/post_migrate/20231213080238_validate_merge_request_diffs_project_id_null_constraint.rb b/db/post_migrate/20231213080238_validate_merge_request_diffs_project_id_null_constraint.rb new file mode 100644 index 00000000000..6e2d591bd96 --- /dev/null +++ b/db/post_migrate/20231213080238_validate_merge_request_diffs_project_id_null_constraint.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class ValidateMergeRequestDiffsProjectIdNullConstraint < Gitlab::Database::Migration[2.2] + milestone '16.8' + + def up + validate_not_null_constraint :merge_request_diffs, :project_id + end + + def down; end +end diff --git a/db/post_migrate/20231214111617_queue_backfill_owasp_top_ten_of_vulnerability_reads.rb b/db/post_migrate/20231214111617_queue_backfill_owasp_top_ten_of_vulnerability_reads.rb new file mode 100644 index 00000000000..e8b13212df8 --- /dev/null +++ b/db/post_migrate/20231214111617_queue_backfill_owasp_top_ten_of_vulnerability_reads.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class QueueBackfillOwaspTopTenOfVulnerabilityReads < Gitlab::Database::Migration[2.2] + milestone '16.8' + + MIGRATION = "BackfillOwaspTopTenOfVulnerabilityReads" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 10000 + SUB_BATCH_SIZE = 50 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + def up + queue_batched_background_migration( + MIGRATION, + :vulnerability_reads, + :vulnerability_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :vulnerability_reads, :vulnerability_id, []) + end +end diff --git a/db/post_migrate/20231215130625_schedule_index_to_events_author_group_action_target_type_created_at.rb b/db/post_migrate/20231215130625_schedule_index_to_events_author_group_action_target_type_created_at.rb new file mode 100644 index 00000000000..063aff58176 --- /dev/null +++ b/db/post_migrate/20231215130625_schedule_index_to_events_author_group_action_target_type_created_at.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ScheduleIndexToEventsAuthorGroupActionTargetTypeCreatedAt < Gitlab::Database::Migration[2.2] + milestone '16.8' + + INDEX_NAME = 'index_events_author_id_group_id_action_target_type_created_at' + COLUMNS = [:author_id, :group_id, :action, :target_type, :created_at] + + # TODO: Index to be created synchronously in https://gitlab.com/gitlab-org/gitlab/-/issues/435524 + def up + prepare_async_index :events, COLUMNS, name: INDEX_NAME + end + + def down + unprepare_async_index :events, COLUMNS, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20231215151348_finalize_workspaces_config_version_2_migration.rb b/db/post_migrate/20231215151348_finalize_workspaces_config_version_2_migration.rb new file mode 100644 index 00000000000..c76bc15d778 --- /dev/null +++ b/db/post_migrate/20231215151348_finalize_workspaces_config_version_2_migration.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class FinalizeWorkspacesConfigVersion2Migration < Gitlab::Database::Migration[2.2] + milestone '16.8' + MIGRATION = 'UpdateWorkspacesConfigVersion' + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: :workspaces, + column_name: :id, + job_arguments: [], + finalize: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20231217053910_remove_due_date_sourcing_milestone_id_column_from_vulnerabilities.rb b/db/post_migrate/20231217053910_remove_due_date_sourcing_milestone_id_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..5cd41dff0ef --- /dev/null +++ b/db/post_migrate/20231217053910_remove_due_date_sourcing_milestone_id_column_from_vulnerabilities.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class RemoveDueDateSourcingMilestoneIdColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + def up + with_lock_retries do + remove_column :vulnerabilities, :due_date_sourcing_milestone_id + end + end + + def down + unless column_exists?(:vulnerabilities, :due_date_sourcing_milestone_id) + add_column :vulnerabilities, :due_date_sourcing_milestone_id, :bigint + end + + # Add back index and constraint that were dropped in `up` + add_concurrent_index(:vulnerabilities, :due_date_sourcing_milestone_id) + add_concurrent_foreign_key(:vulnerabilities, :milestones, column: :due_date_sourcing_milestone_id, + on_delete: :nullify) + end +end diff --git a/db/post_migrate/20231218074237_add_index_to_ci_job_artifact_states.rb b/db/post_migrate/20231218074237_add_index_to_ci_job_artifact_states.rb new file mode 100644 index 00000000000..bd0a46f14ba --- /dev/null +++ b/db/post_migrate/20231218074237_add_index_to_ci_job_artifact_states.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexToCiJobArtifactStates < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + INDEX_NAME = :index_ci_job_artifact_states_on_job_artifact_id_partition_id + TABLE_NAME = :ci_job_artifact_states + + def up + add_concurrent_index(TABLE_NAME, [:job_artifact_id, :partition_id], name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME) + end +end diff --git a/db/post_migrate/20231218074508_add_index_to_ci_build_trace_metadata.rb b/db/post_migrate/20231218074508_add_index_to_ci_build_trace_metadata.rb new file mode 100644 index 00000000000..574c71034fe --- /dev/null +++ b/db/post_migrate/20231218074508_add_index_to_ci_build_trace_metadata.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexToCiBuildTraceMetadata < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + INDEX_NAME = :index_ci_build_trace_metadata_on_trace_artifact_id_partition_id + TABLE_NAME = :ci_build_trace_metadata + + def up + add_concurrent_index(TABLE_NAME, [:trace_artifact_id, :partition_id], name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME) + end +end diff --git a/db/post_migrate/20231218092401_queue_backfill_partition_id_ci_pipeline_chat_data.rb b/db/post_migrate/20231218092401_queue_backfill_partition_id_ci_pipeline_chat_data.rb new file mode 100644 index 00000000000..ffa8d0dbd21 --- /dev/null +++ b/db/post_migrate/20231218092401_queue_backfill_partition_id_ci_pipeline_chat_data.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class QueueBackfillPartitionIdCiPipelineChatData < Gitlab::Database::Migration[2.2] + milestone '16.8' + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + MIGRATION = 'BackfillPartitionIdCiPipelineChatData' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :ci_pipeline_chat_data, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :ci_pipeline_chat_data, :id, []) + end +end diff --git a/db/post_migrate/20231218121131_remove_start_date_sourcing_milestone_id_column_from_vulnerabilities.rb b/db/post_migrate/20231218121131_remove_start_date_sourcing_milestone_id_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..1ca04e5aed1 --- /dev/null +++ b/db/post_migrate/20231218121131_remove_start_date_sourcing_milestone_id_column_from_vulnerabilities.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class RemoveStartDateSourcingMilestoneIdColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + def up + with_lock_retries do + remove_column :vulnerabilities, :start_date_sourcing_milestone_id + end + end + + def down + unless column_exists?(:vulnerabilities, :start_date_sourcing_milestone_id) + add_column :vulnerabilities, :start_date_sourcing_milestone_id, :bigint + end + + # Add back index and constraint that were dropped in `up` + add_concurrent_index(:vulnerabilities, :start_date_sourcing_milestone_id) + add_concurrent_foreign_key(:vulnerabilities, :milestones, column: :start_date_sourcing_milestone_id, + on_delete: :nullify) + end +end diff --git a/db/post_migrate/20231219044006_drop_note_mentions_temp_index.rb b/db/post_migrate/20231219044006_drop_note_mentions_temp_index.rb new file mode 100644 index 00000000000..948530d4c02 --- /dev/null +++ b/db/post_migrate/20231219044006_drop_note_mentions_temp_index.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class DropNoteMentionsTempIndex < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + INDEX_NAME = 'note_mentions_temp_index' + + def up + remove_concurrent_index_by_name :notes, INDEX_NAME + end + + def down + add_concurrent_index :notes, [:id, :noteable_type], where: "note ~~ '%@%'::text", name: INDEX_NAME + end +end diff --git a/db/post_migrate/20231219075928_finalize_backfill_users_with_defaults.rb b/db/post_migrate/20231219075928_finalize_backfill_users_with_defaults.rb new file mode 100644 index 00000000000..373ea1faaa9 --- /dev/null +++ b/db/post_migrate/20231219075928_finalize_backfill_users_with_defaults.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class FinalizeBackfillUsersWithDefaults < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillUsersWithDefaults" + + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: 'users', + column_name: 'id', + job_arguments: [] + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20231219075929_finalize_backfill_user_preferences_with_defaults.rb b/db/post_migrate/20231219075929_finalize_backfill_user_preferences_with_defaults.rb new file mode 100644 index 00000000000..fb10385ad20 --- /dev/null +++ b/db/post_migrate/20231219075929_finalize_backfill_user_preferences_with_defaults.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class FinalizeBackfillUserPreferencesWithDefaults < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillUserPreferencesWithDefaults" + + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: 'user_preferences', + column_name: 'id', + job_arguments: [] + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20231219083804_add_fk_to_ci_build_trace_metadata_on_partition_id_and_trace_artifact_id.rb b/db/post_migrate/20231219083804_add_fk_to_ci_build_trace_metadata_on_partition_id_and_trace_artifact_id.rb new file mode 100644 index 00000000000..bb7749b8ab5 --- /dev/null +++ b/db/post_migrate/20231219083804_add_fk_to_ci_build_trace_metadata_on_partition_id_and_trace_artifact_id.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class AddFkToCiBuildTraceMetadataOnPartitionIdAndTraceArtifactId < Gitlab::Database::Migration[2.2] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + milestone '16.8' + disable_ddl_transaction! + + SOURCE_TABLE_NAME = :ci_build_trace_metadata + TARGET_TABLE_NAME = :ci_job_artifacts + COLUMN = :trace_artifact_id + TARGET_COLUMN = :id + FK_NAME = :fk_21d25cac1a_p + PARTITION_COLUMN = :partition_id + + def up + return unless should_run? + + add_concurrent_foreign_key( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + column: [PARTITION_COLUMN, COLUMN], + target_column: [PARTITION_COLUMN, TARGET_COLUMN], + validate: false, + reverse_lock_order: true, + on_update: :cascade, + on_delete: :cascade, + name: FK_NAME + ) + + prepare_async_foreign_key_validation(SOURCE_TABLE_NAME, name: FK_NAME) + end + + def down + return unless should_run? + + unprepare_async_foreign_key_validation(SOURCE_TABLE_NAME, name: FK_NAME) + + with_lock_retries do + remove_foreign_key_if_exists( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + name: FK_NAME, + reverse_lock_order: true + ) + end + end + + private + + def should_run? + can_execute_on?(TARGET_TABLE_NAME) + end +end diff --git a/db/post_migrate/20231219084159_add_fk_to_ci_job_artifact_states_on_partition_id_and_job_artifact_id.rb b/db/post_migrate/20231219084159_add_fk_to_ci_job_artifact_states_on_partition_id_and_job_artifact_id.rb new file mode 100644 index 00000000000..d72f10b012f --- /dev/null +++ b/db/post_migrate/20231219084159_add_fk_to_ci_job_artifact_states_on_partition_id_and_job_artifact_id.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class AddFkToCiJobArtifactStatesOnPartitionIdAndJobArtifactId < Gitlab::Database::Migration[2.2] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + milestone '16.8' + disable_ddl_transaction! + + SOURCE_TABLE_NAME = :ci_job_artifact_states + TARGET_TABLE_NAME = :ci_job_artifacts + COLUMN = :job_artifact_id + TARGET_COLUMN = :id + FK_NAME = :fk_rails_80a9cba3b2_p + PARTITION_COLUMN = :partition_id + + def up + return unless should_run? + + add_concurrent_foreign_key( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + column: [PARTITION_COLUMN, COLUMN], + target_column: [PARTITION_COLUMN, TARGET_COLUMN], + validate: false, + reverse_lock_order: true, + on_update: :cascade, + on_delete: :cascade, + name: FK_NAME + ) + + prepare_async_foreign_key_validation(SOURCE_TABLE_NAME, name: FK_NAME) + end + + def down + return unless should_run? + + unprepare_async_foreign_key_validation(SOURCE_TABLE_NAME, name: FK_NAME) + + with_lock_retries do + remove_foreign_key_if_exists( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + name: FK_NAME, + reverse_lock_order: true + ) + end + end + + private + + def should_run? + can_execute_on?(TARGET_TABLE_NAME) + end +end diff --git a/db/post_migrate/20231219092400_backfill_geo_event_log_for_bigint_conversion.rb b/db/post_migrate/20231219092400_backfill_geo_event_log_for_bigint_conversion.rb new file mode 100644 index 00000000000..b5203ff9051 --- /dev/null +++ b/db/post_migrate/20231219092400_backfill_geo_event_log_for_bigint_conversion.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class BackfillGeoEventLogForBigintConversion < Gitlab::Database::Migration[2.2] + restrict_gitlab_migration gitlab_schema: :gitlab_main + milestone '16.8' + + TABLE = :geo_event_log + COLUMNS = %i[geo_event_id] + + def up + backfill_conversion_of_integer_to_bigint(TABLE, COLUMNS, sub_batch_size: 200) + end + + def down + revert_backfill_conversion_of_integer_to_bigint(TABLE, COLUMNS) + end +end diff --git a/db/post_migrate/20231219125907_remove_updated_by_id_column_from_vulnerabilities.rb b/db/post_migrate/20231219125907_remove_updated_by_id_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..ecac9b55f4c --- /dev/null +++ b/db/post_migrate/20231219125907_remove_updated_by_id_column_from_vulnerabilities.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveUpdatedByIdColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + def up + with_lock_retries do + remove_column :vulnerabilities, :updated_by_id + end + end + + def down + add_column :vulnerabilities, :updated_by_id, :bigint unless column_exists?(:vulnerabilities, :updated_by_id) + + # Add back index and constraint that were dropped in `up` + add_concurrent_index(:vulnerabilities, :updated_by_id) + add_concurrent_foreign_key(:vulnerabilities, :users, column: :updated_by_id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20231219132423_remove_epic_id_column_from_vulnerabilities.rb b/db/post_migrate/20231219132423_remove_epic_id_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..fa6379a409a --- /dev/null +++ b/db/post_migrate/20231219132423_remove_epic_id_column_from_vulnerabilities.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveEpicIdColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + def up + with_lock_retries do + remove_column :vulnerabilities, :epic_id + end + end + + def down + add_column :vulnerabilities, :epic_id, :bigint unless column_exists?(:vulnerabilities, :epic_id) + + # Add back index and constraint that were dropped in `up` + add_concurrent_index(:vulnerabilities, :epic_id) + add_concurrent_foreign_key(:vulnerabilities, :epics, column: :epic_id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20231220115201_add_color_work_item_widget.rb b/db/post_migrate/20231220115201_add_color_work_item_widget.rb new file mode 100644 index 00000000000..eb410cb8b0b --- /dev/null +++ b/db/post_migrate/20231220115201_add_color_work_item_widget.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +class AddColorWorkItemWidget < Gitlab::Database::Migration[2.2] + milestone '16.8' + + class WorkItemType < MigrationRecord + self.table_name = 'work_item_types' + end + + class WidgetDefinition < MigrationRecord + self.table_name = 'work_item_widget_definitions' + end + + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + WIDGET_NAME = 'Color' + WIDGET_ENUM_VALUE = 18 + WORK_ITEM_TYPE = 'Epic' + + def up + type = WorkItemType.find_by_name_and_namespace_id(WORK_ITEM_TYPE, nil) + + unless type + Gitlab::AppLogger.warn("type #{WORK_ITEM_TYPE} is missing, not adding widget") + return + end + + widget = { + work_item_type_id: type.id, + name: WIDGET_NAME, + widget_type: WIDGET_ENUM_VALUE + } + + WidgetDefinition.upsert_all( + [widget], + unique_by: :index_work_item_widget_definitions_on_default_witype_and_name + ) + end + + def down + WidgetDefinition.where(name: WIDGET_NAME).delete_all + end +end diff --git a/db/post_migrate/20231220130725_remove_last_edited_at_column_from_vulnerabilities.rb b/db/post_migrate/20231220130725_remove_last_edited_at_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..e3e963ee0c0 --- /dev/null +++ b/db/post_migrate/20231220130725_remove_last_edited_at_column_from_vulnerabilities.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveLastEditedAtColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + milestone '16.8' + + enable_lock_retries! + + def up + remove_column :vulnerabilities, :last_edited_at + end + + def down + add_column :vulnerabilities, :last_edited_at, :datetime_with_timezone + end +end diff --git a/db/post_migrate/20231220150143_remove_organizations_organization_users_organization_id_fk.rb b/db/post_migrate/20231220150143_remove_organizations_organization_users_organization_id_fk.rb new file mode 100644 index 00000000000..75bf5d3ead9 --- /dev/null +++ b/db/post_migrate/20231220150143_remove_organizations_organization_users_organization_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveOrganizationsOrganizationUsersOrganizationIdFk < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + FOREIGN_KEY_NAME = "fk_8471abad75" + + def up + with_lock_retries do + remove_foreign_key_if_exists(:organization_users, :organizations, + name: FOREIGN_KEY_NAME, reverse_lock_order: true) + end + end + + def down + add_concurrent_foreign_key(:organization_users, :organizations, + name: FOREIGN_KEY_NAME, column: :organization_id, + target_column: :id, on_delete: :cascade) + end +end diff --git a/db/post_migrate/20231220150210_remove_users_organization_users_user_id_fk.rb b/db/post_migrate/20231220150210_remove_users_organization_users_user_id_fk.rb new file mode 100644 index 00000000000..365d6b0fc77 --- /dev/null +++ b/db/post_migrate/20231220150210_remove_users_organization_users_user_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveUsersOrganizationUsersUserIdFk < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + FOREIGN_KEY_NAME = "fk_8d9b20725d" + + def up + with_lock_retries do + remove_foreign_key_if_exists(:organization_users, :users, + name: FOREIGN_KEY_NAME, reverse_lock_order: true) + end + end + + def down + add_concurrent_foreign_key(:organization_users, :users, + name: FOREIGN_KEY_NAME, column: :user_id, + target_column: :id, on_delete: :cascade) + end +end diff --git a/db/post_migrate/20231221000703_async_validate_merge_request_diffs_project_id_foreign_key.rb b/db/post_migrate/20231221000703_async_validate_merge_request_diffs_project_id_foreign_key.rb new file mode 100644 index 00000000000..64adf7d3523 --- /dev/null +++ b/db/post_migrate/20231221000703_async_validate_merge_request_diffs_project_id_foreign_key.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AsyncValidateMergeRequestDiffsProjectIdForeignKey < Gitlab::Database::Migration[2.2] + milestone '16.8' + + def up + prepare_async_foreign_key_validation :merge_request_diffs, :project_id + end + + def down + unprepare_async_foreign_key_validation :merge_request_diffs, :project_id + end +end diff --git a/db/post_migrate/20231221223259_queue_drop_vulnerabilities_without_finding_id.rb b/db/post_migrate/20231221223259_queue_drop_vulnerabilities_without_finding_id.rb new file mode 100644 index 00000000000..25aa2912767 --- /dev/null +++ b/db/post_migrate/20231221223259_queue_drop_vulnerabilities_without_finding_id.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueDropVulnerabilitiesWithoutFindingId < Gitlab::Database::Migration[2.2] + milestone '16.8' + + MIGRATION = "DropVulnerabilitiesWithoutFindingId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :vulnerabilities, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :vulnerabilities, :id, []) + end +end diff --git a/db/post_migrate/20231222051050_add_search_vector_index_to_catalog_resources.rb b/db/post_migrate/20231222051050_add_search_vector_index_to_catalog_resources.rb new file mode 100644 index 00000000000..036aadbe10f --- /dev/null +++ b/db/post_migrate/20231222051050_add_search_vector_index_to_catalog_resources.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddSearchVectorIndexToCatalogResources < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '16.8' + + INDEX_NAME = 'index_catalog_resources_on_search_vector_triagram' + + def up + disable_statement_timeout do + execute <<-SQL + CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON catalog_resources + USING GIN (search_vector); + SQL + end + end + + def down + remove_concurrent_index_by_name :catalog_resources, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20231222072237_add_text_limit_to_ci_instance_variables_description.rb b/db/post_migrate/20231222072237_add_text_limit_to_ci_instance_variables_description.rb new file mode 100644 index 00000000000..6ad13a591e4 --- /dev/null +++ b/db/post_migrate/20231222072237_add_text_limit_to_ci_instance_variables_description.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddTextLimitToCiInstanceVariablesDescription < Gitlab::Database::Migration[2.2] + milestone '16.8' + + disable_ddl_transaction! + + TABLE_NAME = :ci_instance_variables + COLUMN_NAME = :description + + def up + add_text_limit(TABLE_NAME, COLUMN_NAME, 255) + end + + def down + remove_text_limit(TABLE_NAME, COLUMN_NAME) + end +end diff --git a/db/post_migrate/20231222114515_remove_tmp_index_members_on_state.rb b/db/post_migrate/20231222114515_remove_tmp_index_members_on_state.rb new file mode 100644 index 00000000000..88a52160fef --- /dev/null +++ b/db/post_migrate/20231222114515_remove_tmp_index_members_on_state.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveTmpIndexMembersOnState < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '16.8' + + INDEX_NAME = 'tmp_index_members_on_state' + + def up + remove_concurrent_index_by_name :members, INDEX_NAME + end + + def down + add_concurrent_index :members, :state, name: INDEX_NAME, where: 'state = 2' + end +end diff --git a/db/post_migrate/20231224083824_remove_last_edited_by_id_column_from_vulnerabilities.rb b/db/post_migrate/20231224083824_remove_last_edited_by_id_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..3df182db156 --- /dev/null +++ b/db/post_migrate/20231224083824_remove_last_edited_by_id_column_from_vulnerabilities.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveLastEditedByIdColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + def up + with_lock_retries do + remove_column :vulnerabilities, :last_edited_by_id + end + end + + def down + add_column :vulnerabilities, :last_edited_by_id, :bigint unless column_exists?(:vulnerabilities, :last_edited_by_id) + + # Add back index and constraint that were dropped in `up` + add_concurrent_index(:vulnerabilities, :last_edited_by_id) + add_concurrent_foreign_key(:vulnerabilities, :users, column: :last_edited_by_id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20231225070431_async_validate_finding_id_on_vulnerabilities.rb b/db/post_migrate/20231225070431_async_validate_finding_id_on_vulnerabilities.rb new file mode 100644 index 00000000000..dfab0d6eab0 --- /dev/null +++ b/db/post_migrate/20231225070431_async_validate_finding_id_on_vulnerabilities.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AsyncValidateFindingIdOnVulnerabilities < Gitlab::Database::Migration[2.2] + # obtained by running `\d vulnerabilities` on https://console.postgres.ai + FK_NAME = :fk_4e64972902 + + milestone '16.8' + + # TODO: FK to be validated synchronously in https://gitlab.com/gitlab-org/gitlab/-/issues/425409 + def up + prepare_async_foreign_key_validation :vulnerabilities, :finding_id, name: FK_NAME + end + + def down + unprepare_async_foreign_key_validation :vulnerabilities, :finding_id, name: FK_NAME + end +end diff --git a/db/post_migrate/20231225133628_remove_due_date_column_from_vulnerabilities.rb b/db/post_migrate/20231225133628_remove_due_date_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..2f77a5a82f4 --- /dev/null +++ b/db/post_migrate/20231225133628_remove_due_date_column_from_vulnerabilities.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveDueDateColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + milestone '16.8' + + enable_lock_retries! + + def up + remove_column :vulnerabilities, :due_date + end + + def down + add_column :vulnerabilities, :due_date, :date + end +end diff --git a/db/post_migrate/20231228043653_rename_old_index_to_new_index_in_catalog_resources.rb b/db/post_migrate/20231228043653_rename_old_index_to_new_index_in_catalog_resources.rb new file mode 100644 index 00000000000..6569e624c78 --- /dev/null +++ b/db/post_migrate/20231228043653_rename_old_index_to_new_index_in_catalog_resources.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class RenameOldIndexToNewIndexInCatalogResources < Gitlab::Database::Migration[2.2] + milestone '16.8' + + enable_lock_retries! + + OLD_INDEX_NAME = 'index_catalog_resources_on_search_vector_triagram' + NEW_INDEX_NAME = 'index_catalog_resources_on_search_vector' + + def change + rename_index :catalog_resources, OLD_INDEX_NAME, NEW_INDEX_NAME + end +end diff --git a/db/post_migrate/20240101133628_remove_start_date_column_from_vulnerabilities.rb b/db/post_migrate/20240101133628_remove_start_date_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..747ff35377b --- /dev/null +++ b/db/post_migrate/20240101133628_remove_start_date_column_from_vulnerabilities.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveStartDateColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + milestone '16.8' + + enable_lock_retries! + + def up + remove_column :vulnerabilities, :start_date + end + + def down + add_column :vulnerabilities, :start_date, :date + end +end diff --git a/db/post_migrate/20240102065444_remove_milestone_id_column_from_vulnerabilities.rb b/db/post_migrate/20240102065444_remove_milestone_id_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..f74c15a97fb --- /dev/null +++ b/db/post_migrate/20240102065444_remove_milestone_id_column_from_vulnerabilities.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveMilestoneIdColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + def up + with_lock_retries do + remove_column :vulnerabilities, :milestone_id + end + end + + def down + add_column :vulnerabilities, :milestone_id, :bigint unless column_exists?(:vulnerabilities, :milestone_id) + + # Add back index and constraint that were dropped in `up` + add_concurrent_index(:vulnerabilities, :milestone_id) + add_concurrent_foreign_key(:vulnerabilities, :milestones, column: :milestone_id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20240102101409_validate_finding_id_on_vulnerabilities.rb b/db/post_migrate/20240102101409_validate_finding_id_on_vulnerabilities.rb new file mode 100644 index 00000000000..deed2462f93 --- /dev/null +++ b/db/post_migrate/20240102101409_validate_finding_id_on_vulnerabilities.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class ValidateFindingIdOnVulnerabilities < Gitlab::Database::Migration[2.2] + # obtained by running `\d vulnerabilities` on https://console.postgres.ai + FK_NAME = :fk_4e64972902 + + milestone '16.8' + + # validated asynchronously in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131979 + def up + validate_foreign_key :vulnerabilities, :finding_id, name: FK_NAME + end + + def down + # Can be safely a no-op if we don't roll back the inconsistent data. + # https://docs.gitlab.com/ee/development/database/add_foreign_key_to_existing_column.html#add-a-migration-to-validate-the-fk-synchronously + end +end diff --git a/db/post_migrate/20240104085448_queue_update_workspaces_config_version3.rb b/db/post_migrate/20240104085448_queue_update_workspaces_config_version3.rb new file mode 100644 index 00000000000..574b6cdb4c2 --- /dev/null +++ b/db/post_migrate/20240104085448_queue_update_workspaces_config_version3.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class QueueUpdateWorkspacesConfigVersion3 < Gitlab::Database::Migration[2.2] + milestone '16.8' + + MIGRATION = "UpdateWorkspacesConfigVersion3" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + def up + queue_batched_background_migration( + MIGRATION, + :workspaces, + :config_version, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :workspaces, :config_version, []) + end +end diff --git a/db/post_migrate/20240104091627_validate_foreign_key_ci_build_trace_metadata.rb b/db/post_migrate/20240104091627_validate_foreign_key_ci_build_trace_metadata.rb new file mode 100644 index 00000000000..16e97bd5e6a --- /dev/null +++ b/db/post_migrate/20240104091627_validate_foreign_key_ci_build_trace_metadata.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class ValidateForeignKeyCiBuildTraceMetadata < Gitlab::Database::Migration[2.2] + milestone '16.8' + + # We first need to introduce this FK for self-managed + def up + # no-op + end + + def down + # Can be safely a no-op if we don't roll back the inconsistent data. + end +end diff --git a/db/post_migrate/20240104091858_validate_foreign_key_ci_job_artifact_state.rb b/db/post_migrate/20240104091858_validate_foreign_key_ci_job_artifact_state.rb new file mode 100644 index 00000000000..bd705fd07a5 --- /dev/null +++ b/db/post_migrate/20240104091858_validate_foreign_key_ci_job_artifact_state.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class ValidateForeignKeyCiJobArtifactState < Gitlab::Database::Migration[2.2] + milestone '16.8' + + # We first need to introduce this FK for self-managed + def up + # no-op + end + + def down + # Can be safely a no-op if we don't roll back the inconsistent data. + end +end diff --git a/db/post_migrate/20240104101601_finalize_backfill_partition_id_ci_pipeline_chat_data.rb b/db/post_migrate/20240104101601_finalize_backfill_partition_id_ci_pipeline_chat_data.rb new file mode 100644 index 00000000000..45da0e9c12a --- /dev/null +++ b/db/post_migrate/20240104101601_finalize_backfill_partition_id_ci_pipeline_chat_data.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class FinalizeBackfillPartitionIdCiPipelineChatData < Gitlab::Database::Migration[2.2] + milestone '16.8' + MIGRATION = 'BackfillPartitionIdCiPipelineChatData' + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: :ci_pipeline_chat_data, + column_name: :id, + job_arguments: [], + finalize: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20240104145320_remove_partition_id_default_value_for_ci_pipeline_chat_data.rb b/db/post_migrate/20240104145320_remove_partition_id_default_value_for_ci_pipeline_chat_data.rb new file mode 100644 index 00000000000..3aedf6e66a1 --- /dev/null +++ b/db/post_migrate/20240104145320_remove_partition_id_default_value_for_ci_pipeline_chat_data.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class RemovePartitionIdDefaultValueForCiPipelineChatData < Gitlab::Database::Migration[2.2] + milestone '16.8' + enable_lock_retries! + + TABLE_NAME = :ci_pipeline_chat_data + COLUM_NAME = :partition_id + + def change + change_column_default(TABLE_NAME, COLUM_NAME, from: 100, to: nil) + end +end diff --git a/db/post_migrate/20240104155616_finalize_backfill_default_branch_protection_namespace_settings.rb b/db/post_migrate/20240104155616_finalize_backfill_default_branch_protection_namespace_settings.rb new file mode 100644 index 00000000000..b4a23028f21 --- /dev/null +++ b/db/post_migrate/20240104155616_finalize_backfill_default_branch_protection_namespace_settings.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class FinalizeBackfillDefaultBranchProtectionNamespaceSettings < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = 'BackfillBranchProtectionNamespaceSetting' + + milestone '16.8' + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: :namespace_settings, + column_name: :namespace_id, + job_arguments: [], + finalize: true + ) + end + + def down + # noop + end +end diff --git a/db/post_migrate/20240104223119_add_index_owasp_top_10_with_project_id_on_vulnerability_reads.rb b/db/post_migrate/20240104223119_add_index_owasp_top_10_with_project_id_on_vulnerability_reads.rb new file mode 100644 index 00000000000..99a2dccc8f5 --- /dev/null +++ b/db/post_migrate/20240104223119_add_index_owasp_top_10_with_project_id_on_vulnerability_reads.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexOwaspTop10WithProjectIdOnVulnerabilityReads < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + INDEX_NAME = 'index_vuln_reads_on_project_id_owasp_top_10' + + def up + add_concurrent_index :vulnerability_reads, [:project_id, :owasp_top_10], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :vulnerability_reads, INDEX_NAME + end +end diff --git a/db/post_migrate/20240105000001_cleanup_workspaces_url_domain_to_dns_zone_rename.rb b/db/post_migrate/20240105000001_cleanup_workspaces_url_domain_to_dns_zone_rename.rb new file mode 100644 index 00000000000..d37d0c9a5af --- /dev/null +++ b/db/post_migrate/20240105000001_cleanup_workspaces_url_domain_to_dns_zone_rename.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CleanupWorkspacesUrlDomainToDnsZoneRename < Gitlab::Database::Migration[2.2] + milestone '16.8' + + disable_ddl_transaction! + + def up + cleanup_concurrent_column_rename :workspaces, :url_domain, :dns_zone + end + + def down + undo_cleanup_concurrent_column_rename :workspaces, :url_domain, :dns_zone + end +end diff --git a/db/post_migrate/20240105120320_disable_fastupdate_on_issue_search_data.rb b/db/post_migrate/20240105120320_disable_fastupdate_on_issue_search_data.rb new file mode 100644 index 00000000000..b7271fa5f51 --- /dev/null +++ b/db/post_migrate/20240105120320_disable_fastupdate_on_issue_search_data.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class DisableFastupdateOnIssueSearchData < Gitlab::Database::Migration[2.2] + milestone '16.8' + + disable_ddl_transaction! + + NUM_PARTITIONS = 64 + + def up + each_index_name do |index_name| + with_lock_retries do + execute <<~SQL + ALTER INDEX #{index_name} SET ( fastupdate = false ) ; + SQL + end + end + end + + def down + each_index_name do |index_name| + with_lock_retries do + execute <<~SQL + ALTER INDEX #{index_name} RESET ( fastupdate ) ; + SQL + end + end + end + + private + + def each_index_name + NUM_PARTITIONS.times do |partition| + yield "gitlab_partitions_static.issue_search_data_#{format('%02d', partition)}_search_vector_idx" + end + end +end diff --git a/db/post_migrate/20240105144908_queue_backfill_issue_search_data_namespace_id.rb b/db/post_migrate/20240105144908_queue_backfill_issue_search_data_namespace_id.rb new file mode 100644 index 00000000000..f12041e613f --- /dev/null +++ b/db/post_migrate/20240105144908_queue_backfill_issue_search_data_namespace_id.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueBackfillIssueSearchDataNamespaceId < Gitlab::Database::Migration[2.2] + milestone '16.8' + + MIGRATION = "BackfillIssueSearchDataNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 1_000 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :issues, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :issues, :id, []) + end +end diff --git a/db/post_migrate/20240107154747_sent_notifications_self_install_finalize_bbm.rb b/db/post_migrate/20240107154747_sent_notifications_self_install_finalize_bbm.rb new file mode 100644 index 00000000000..78efdf36d64 --- /dev/null +++ b/db/post_migrate/20240107154747_sent_notifications_self_install_finalize_bbm.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class SentNotificationsSelfInstallFinalizeBbm < Gitlab::Database::Migration[2.2] + include Gitlab::Database::MigrationHelpers::ConvertToBigint + + milestone '16.8' + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + TABLE_NAME = 'sent_notifications' + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [['id'], ['id_convert_to_bigint']] + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20240107154805_sent_notifications_self_install_id_swap.rb b/db/post_migrate/20240107154805_sent_notifications_self_install_id_swap.rb new file mode 100644 index 00000000000..c6e31ca568e --- /dev/null +++ b/db/post_migrate/20240107154805_sent_notifications_self_install_id_swap.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +class SentNotificationsSelfInstallIdSwap < Gitlab::Database::Migration[2.2] + include Gitlab::Database::MigrationHelpers::ConvertToBigint + + milestone '16.8' + + disable_ddl_transaction! + + TABLE_NAME = :sent_notifications + COLUMN_NAME = :id_convert_to_bigint + INDEX_NAME = :index_sent_notifications_on_id_convert_to_bigint + + def up + return if com_or_dev_or_test_but_not_jh? + return if temp_column_removed?(TABLE_NAME, :id) + return if columns_swapped?(TABLE_NAME, :id) + + swap + end + + def down + return if com_or_dev_or_test_but_not_jh? + return if temp_column_removed?(TABLE_NAME, :id) + return unless columns_swapped?(TABLE_NAME, :id) + + swap + end + + def swap + add_concurrent_index TABLE_NAME, COLUMN_NAME, unique: true, name: INDEX_NAME + + with_lock_retries(raise_on_exhaustion: true) do + execute "LOCK TABLE #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + # Swap Columns + temp_name = quote_column_name(:id_tmp) + id_name = quote_column_name(:id) + id_convert_to_bigint_name = quote_column_name(COLUMN_NAME) + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_name} TO #{temp_name}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_convert_to_bigint_name} TO #{id_name}" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{temp_name} TO #{id_convert_to_bigint_name}" + + # Reset trigger + function_name = Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME, connection: connection) + .name(:id, :id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + execute "ALTER SEQUENCE #{TABLE_NAME}_id_seq OWNED BY #{TABLE_NAME}.id" + change_column_default TABLE_NAME, :id, -> { "nextval('#{TABLE_NAME}_id_seq'::regclass)" } + change_column_default TABLE_NAME, :id_convert_to_bigint, 0 + + execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT #{TABLE_NAME}_pkey CASCADE" + rename_index TABLE_NAME, INDEX_NAME, "#{TABLE_NAME}_pkey" + execute "ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT #{TABLE_NAME}_pkey PRIMARY KEY USING INDEX #{TABLE_NAME}_pkey" + end + end +end diff --git a/db/post_migrate/20240108072319_add_fk_to_ci_build_trace_metadata_on_partition_id_and_trace_artifact_id2.rb b/db/post_migrate/20240108072319_add_fk_to_ci_build_trace_metadata_on_partition_id_and_trace_artifact_id2.rb new file mode 100644 index 00000000000..23219a9d90a --- /dev/null +++ b/db/post_migrate/20240108072319_add_fk_to_ci_build_trace_metadata_on_partition_id_and_trace_artifact_id2.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class AddFkToCiBuildTraceMetadataOnPartitionIdAndTraceArtifactId2 < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + SOURCE_TABLE_NAME = :ci_build_trace_metadata + TARGET_TABLE_NAME = :ci_job_artifacts + COLUMN = :trace_artifact_id + TARGET_COLUMN = :id + FK_NAME = :fk_21d25cac1a_p + PARTITION_COLUMN = :partition_id + + def up + add_concurrent_foreign_key( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + column: [PARTITION_COLUMN, COLUMN], + target_column: [PARTITION_COLUMN, TARGET_COLUMN], + validate: true, + reverse_lock_order: true, + on_update: :cascade, + on_delete: :cascade, + name: FK_NAME + ) + end + + def down + with_lock_retries do + remove_foreign_key_if_exists( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + name: FK_NAME, + reverse_lock_order: true + ) + end + end +end diff --git a/db/post_migrate/20240108072546_add_fk_to_ci_job_artifact_states_on_partition_id_and_job_artifact_id2.rb b/db/post_migrate/20240108072546_add_fk_to_ci_job_artifact_states_on_partition_id_and_job_artifact_id2.rb new file mode 100644 index 00000000000..9b74f7019b9 --- /dev/null +++ b/db/post_migrate/20240108072546_add_fk_to_ci_job_artifact_states_on_partition_id_and_job_artifact_id2.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class AddFkToCiJobArtifactStatesOnPartitionIdAndJobArtifactId2 < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + SOURCE_TABLE_NAME = :ci_job_artifact_states + TARGET_TABLE_NAME = :ci_job_artifacts + COLUMN = :job_artifact_id + TARGET_COLUMN = :id + FK_NAME = :fk_rails_80a9cba3b2_p + PARTITION_COLUMN = :partition_id + + def up + add_concurrent_foreign_key( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + column: [PARTITION_COLUMN, COLUMN], + target_column: [PARTITION_COLUMN, TARGET_COLUMN], + validate: true, + reverse_lock_order: true, + on_update: :cascade, + on_delete: :cascade, + name: FK_NAME + ) + end + + def down + with_lock_retries do + remove_foreign_key_if_exists( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + name: FK_NAME, + reverse_lock_order: true + ) + end + end +end diff --git a/db/post_migrate/20240108082419_queue_backfill_partition_id_ci_pipeline_metadata.rb b/db/post_migrate/20240108082419_queue_backfill_partition_id_ci_pipeline_metadata.rb new file mode 100644 index 00000000000..6bcaf376a05 --- /dev/null +++ b/db/post_migrate/20240108082419_queue_backfill_partition_id_ci_pipeline_metadata.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class QueueBackfillPartitionIdCiPipelineMetadata < Gitlab::Database::Migration[2.2] + milestone '16.8' + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + MIGRATION = 'BackfillPartitionIdCiPipelineMetadata' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 250 + + def up + queue_batched_background_migration( + MIGRATION, + :ci_pipeline_metadata, + :pipeline_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :ci_pipeline_metadata, :pipeline_id, []) + end +end diff --git a/db/post_migrate/20240108125135_remove_internal_ids_triggers.rb b/db/post_migrate/20240108125135_remove_internal_ids_triggers.rb new file mode 100644 index 00000000000..861aad2b288 --- /dev/null +++ b/db/post_migrate/20240108125135_remove_internal_ids_triggers.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +class RemoveInternalIdsTriggers < Gitlab::Database::Migration[2.2] + include Gitlab::Database::SchemaHelpers + + milestone '16.8' + disable_ddl_transaction! + + TRIGGER_ON_INSERT = 'trigger_copy_usage_on_internal_ids_on_insert' + TRIGGER_ON_UPDATE = 'trigger_copy_usage_on_internal_ids_on_update' + INSERT_OR_UPDATE_FUNCTION_NAME = 'insert_or_update_internal_ids_usage' + + def up + drop_trigger(:internal_ids, TRIGGER_ON_INSERT) + drop_trigger(:internal_ids, TRIGGER_ON_UPDATE) + drop_function(INSERT_OR_UPDATE_FUNCTION_NAME) + end + + def down + execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{INSERT_OR_UPDATE_FUNCTION_NAME}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + DECLARE + namespace_type varchar; + copy_usage smallint; + BEGIN + IF (NEW.usage = 0) THEN + copy_usage = 4; + + -- we only care about group level internal_ids so we check namespace type here + namespace_type = (SELECT type FROM namespaces WHERE id = NEW.namespace_id); + IF (namespace_type <> 'Group') THEN + RETURN NULL; + END IF; + ELSIF (NEW.usage = 4) THEN + copy_usage = 0; + ELSE + RETURN NULL; + END IF; + + -- if value is the same there is nothing to update + IF (OLD.last_value = NEW.last_value AND (TG_OP = 'INSERT' OR TG_OP = 'UPDATE')) THEN + RETURN NULL; + END IF; + + INSERT INTO internal_ids (usage, last_value, namespace_id) + VALUES (copy_usage, NEW.last_value, NEW.namespace_id) + ON CONFLICT (usage, namespace_id) WHERE namespace_id IS NOT NULL + DO UPDATE SET last_value = NEW.last_value; + + RETURN NULL; + END + $$ + SQL + + execute(<<~SQL) + CREATE TRIGGER #{TRIGGER_ON_INSERT} + AFTER INSERT ON internal_ids + FOR EACH ROW + WHEN (((NEW.usage = 0) OR (NEW.usage = 4)) AND NEW.namespace_id IS NOT NULL) + EXECUTE FUNCTION #{INSERT_OR_UPDATE_FUNCTION_NAME}(); + SQL + + execute(<<~SQL) + CREATE TRIGGER #{TRIGGER_ON_UPDATE} + AFTER UPDATE ON internal_ids + FOR EACH ROW + WHEN (((NEW.usage = 0) OR (NEW.usage = 4)) AND NEW.namespace_id IS NOT NULL) + EXECUTE FUNCTION #{INSERT_OR_UPDATE_FUNCTION_NAME}(); + SQL + end +end diff --git a/db/post_migrate/20240108125335_add_temporary_index_internal_ids_on_id_and_usage.rb b/db/post_migrate/20240108125335_add_temporary_index_internal_ids_on_id_and_usage.rb new file mode 100644 index 00000000000..798a42d2a1d --- /dev/null +++ b/db/post_migrate/20240108125335_add_temporary_index_internal_ids_on_id_and_usage.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddTemporaryIndexInternalIdsOnIdAndUsage < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + INDEX_NAME = "tmp_index_internal_ids_on_id_and_usage" + EPICS_USAGE = 4 # see Enums::InternalId#usage_resources[:epics] + + def up + add_concurrent_index :internal_ids, :id, name: INDEX_NAME, where: "usage = #{EPICS_USAGE}" + end + + def down + remove_concurrent_index_by_name :internal_ids, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20240108181808_remove_package_registry_duplicated_indexes.rb b/db/post_migrate/20240108181808_remove_package_registry_duplicated_indexes.rb new file mode 100644 index 00000000000..96380b94d50 --- /dev/null +++ b/db/post_migrate/20240108181808_remove_package_registry_duplicated_indexes.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class RemovePackageRegistryDuplicatedIndexes < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '16.8' + + DUPLICATED_INDEXES = [ + { + name: :index_packages_debian_group_distributions_on_group_id, + table: :packages_debian_group_distributions, + column: :group_id + }, + { + name: :index_packages_debian_project_distributions_on_project_id, + table: :packages_debian_project_distributions, + column: :project_id + }, + { + name: :index_packages_tags_on_package_id, + table: :packages_tags, + column: :package_id + } + ] + + def up + DUPLICATED_INDEXES.each do |index| + remove_concurrent_index_by_name(index[:table], index[:name]) + end + end + + def down + DUPLICATED_INDEXES.each do |index| + add_concurrent_index(index[:table], index[:column], name: index[:name]) + end + end +end diff --git a/db/post_migrate/20240108185335_backfill_internal_ids_with_issues_usage_for_epics.rb b/db/post_migrate/20240108185335_backfill_internal_ids_with_issues_usage_for_epics.rb new file mode 100644 index 00000000000..4131f403212 --- /dev/null +++ b/db/post_migrate/20240108185335_backfill_internal_ids_with_issues_usage_for_epics.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +class BackfillInternalIdsWithIssuesUsageForEpics < Gitlab::Database::Migration[2.2] + milestone '16.8' + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + BATCH_SIZE = 500 + ISSUES_USAGE = 0 # see Enums::InternalId#usage_resources[:issues] + EPICS_USAGE = 4 # see Enums::InternalId#usage_resources[:epics] + + def up + internal_id_model = define_batchable_model('internal_ids') + epic_model = define_batchable_model('epics') + + internal_id_model.where(usage: EPICS_USAGE).each_batch(of: BATCH_SIZE) do |relation| + # Creates a corresponding `usage: :issues` record for every `epics` usage. + # On conflict it means the record was already created when a new epic was created with the newly issues usage. + # In which case to make sure we have the value copied over from epics record. + connection.execute( + <<~SQL + INSERT INTO internal_ids (usage, last_value, namespace_id) + SELECT #{ISSUES_USAGE}, last_value, namespace_id + FROM internal_ids + WHERE internal_ids.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (usage, namespace_id) WHERE namespace_id IS NOT NULL + DO UPDATE SET last_value = GREATEST(EXCLUDED.last_value, internal_ids.last_value) + RETURNING id; + SQL + ) + + relation.delete_all + end + + # there are a couple records in epics table that reference namespaces without a corresponding entry + # in internal_ids, for whatever reason, so this statement addresses that. + epic_model.distinct_each_batch(column: :group_id, of: BATCH_SIZE) do |relation| + connection.execute( + <<~SQL + INSERT INTO internal_ids (usage, last_value, namespace_id) + SELECT #{ISSUES_USAGE}, max(iid) as last_value, group_id + FROM epics + WHERE group_id IN(#{relation.to_sql}) + GROUP BY group_id + ON CONFLICT (usage, namespace_id) WHERE namespace_id IS NOT NULL + DO NOTHING + RETURNING id; + SQL + ) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20240108215335_remove_internal_ids_tmp_index.rb b/db/post_migrate/20240108215335_remove_internal_ids_tmp_index.rb new file mode 100644 index 00000000000..e6ed936afc3 --- /dev/null +++ b/db/post_migrate/20240108215335_remove_internal_ids_tmp_index.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class RemoveInternalIdsTmpIndex < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + INDEX_NAME = "tmp_index_internal_ids_on_id_and_usage" + EPICS_USAGE = 4 # see Enums::InternalId#usage_resources[:epics] + + def up + remove_concurrent_index_by_name :internal_ids, name: INDEX_NAME + end + + def down + add_concurrent_index :internal_ids, :id, name: INDEX_NAME, where: "usage = #{EPICS_USAGE}" + end +end diff --git a/db/post_migrate/20240109025151_create_index_on_id_convert_to_bigint_for_system_note_metadata_async.rb b/db/post_migrate/20240109025151_create_index_on_id_convert_to_bigint_for_system_note_metadata_async.rb new file mode 100644 index 00000000000..281704a1620 --- /dev/null +++ b/db/post_migrate/20240109025151_create_index_on_id_convert_to_bigint_for_system_note_metadata_async.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class CreateIndexOnIdConvertToBigintForSystemNoteMetadataAsync < Gitlab::Database::Migration[2.2] + milestone '16.8' + + TABLE_NAME = :system_note_metadata + INDEX_NAME = 'index_system_note_metadata_pkey_on_id_convert_to_bigint' + + def up + prepare_async_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: INDEX_NAME + end + + def down + unprepare_async_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20240109090354_queue_backfill_partition_id_ci_pipeline_artifact.rb b/db/post_migrate/20240109090354_queue_backfill_partition_id_ci_pipeline_artifact.rb new file mode 100644 index 00000000000..30959a299fb --- /dev/null +++ b/db/post_migrate/20240109090354_queue_backfill_partition_id_ci_pipeline_artifact.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class QueueBackfillPartitionIdCiPipelineArtifact < Gitlab::Database::Migration[2.2] + milestone '16.8' + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + MIGRATION = 'BackfillPartitionIdCiPipelineArtifact' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :ci_pipeline_artifacts, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :ci_pipeline_artifacts, :id, []) + end +end diff --git a/db/post_migrate/20240109165047_add_index_project_id_component_version_id_id_on_sbom_occurrences.rb b/db/post_migrate/20240109165047_add_index_project_id_component_version_id_id_on_sbom_occurrences.rb new file mode 100644 index 00000000000..123d665f625 --- /dev/null +++ b/db/post_migrate/20240109165047_add_index_project_id_component_version_id_id_on_sbom_occurrences.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexProjectIdComponentVersionIdIdOnSbomOccurrences < Gitlab::Database::Migration[2.2] + INDEX_NAME = 'index_sbom_occurr_on_project_id_and_component_version_id_and_id' + + disable_ddl_transaction! + milestone '16.8' + + def up + add_concurrent_index :sbom_occurrences, [:project_id, :component_version_id, :id], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :sbom_occurrence, INDEX_NAME + end +end diff --git a/db/post_migrate/20240110090352_queue_backfill_partition_id_ci_pipeline_config.rb b/db/post_migrate/20240110090352_queue_backfill_partition_id_ci_pipeline_config.rb new file mode 100644 index 00000000000..d5ae025b385 --- /dev/null +++ b/db/post_migrate/20240110090352_queue_backfill_partition_id_ci_pipeline_config.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class QueueBackfillPartitionIdCiPipelineConfig < Gitlab::Database::Migration[2.2] + milestone '16.8' + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + MIGRATION = 'BackfillPartitionIdCiPipelineConfig' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :ci_pipelines_config, + :pipeline_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :ci_pipelines_config, :pipeline_id, []) + end +end diff --git a/db/post_migrate/20240110093654_schedule_index_removal_ci_build_trace_metadata.rb b/db/post_migrate/20240110093654_schedule_index_removal_ci_build_trace_metadata.rb new file mode 100644 index 00000000000..8f2faa32c81 --- /dev/null +++ b/db/post_migrate/20240110093654_schedule_index_removal_ci_build_trace_metadata.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class ScheduleIndexRemovalCiBuildTraceMetadata < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + INDEX_NAME = :index_ci_build_trace_metadata_on_trace_artifact_id + TABLE_NAME = :ci_build_trace_metadata + COLUMN_NAME = :trace_artifact_id + + def up + prepare_async_index_removal(TABLE_NAME, COLUMN_NAME, name: INDEX_NAME) + end + + def down + unprepare_async_index(TABLE_NAME, COLUMN_NAME, name: INDEX_NAME) + end +end diff --git a/db/post_migrate/20240110094002_drop_index_from_ci_job_artifact_state.rb b/db/post_migrate/20240110094002_drop_index_from_ci_job_artifact_state.rb new file mode 100644 index 00000000000..0b92c73ad55 --- /dev/null +++ b/db/post_migrate/20240110094002_drop_index_from_ci_job_artifact_state.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class DropIndexFromCiJobArtifactState < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + INDEX_NAME = :index_ci_job_artifact_states_on_job_artifact_id + TABLE_NAME = :ci_job_artifact_states + + def up + remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME) + end + + def down + add_concurrent_index(TABLE_NAME, :job_artifact_id, name: INDEX_NAME) + end +end diff --git a/db/post_migrate/20240110094510_remove_fk_from_ci_job_artifact_state.rb b/db/post_migrate/20240110094510_remove_fk_from_ci_job_artifact_state.rb new file mode 100644 index 00000000000..bbe1f39e3ed --- /dev/null +++ b/db/post_migrate/20240110094510_remove_fk_from_ci_job_artifact_state.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class RemoveFkFromCiJobArtifactState < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + SOURCE_TABLE_NAME = :ci_job_artifact_states + TARGET_TABLE_NAME = :ci_job_artifacts + COLUMN = :job_artifact_id + TARGET_COLUMN = :id + FK_NAME = :fk_rails_80a9cba3b2 + + def up + with_lock_retries do + remove_foreign_key_if_exists( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + name: FK_NAME, + reverse_lock_order: true + ) + end + end + + def down + add_concurrent_foreign_key( + SOURCE_TABLE_NAME, + TARGET_TABLE_NAME, + column: COLUMN, + target_column: TARGET_COLUMN, + validate: true, + reverse_lock_order: true, + on_delete: :cascade, + name: FK_NAME + ) + end +end diff --git a/db/post_migrate/20240110132029_re_remove_requirements_ignored_columns.rb b/db/post_migrate/20240110132029_re_remove_requirements_ignored_columns.rb new file mode 100644 index 00000000000..e6d85761a14 --- /dev/null +++ b/db/post_migrate/20240110132029_re_remove_requirements_ignored_columns.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class ReRemoveRequirementsIgnoredColumns < Gitlab::Database::Migration[2.2] + milestone '16.8' + + disable_ddl_transaction! + + CONSTRAINT_NAME = 'check_785ae25b9d' + NAME_INDEX = 'index_requirements_on_title_trigram' + FOREIGN_KEY = 'fk_rails_33fed8aa4e' + + def up + remove_column(:requirements, :created_at, if_exists: true) + remove_column(:requirements, :updated_at, if_exists: true) + remove_column(:requirements, :cached_markdown_version, if_exists: true) + remove_column(:requirements, :state, if_exists: true) + remove_column(:requirements, :title, if_exists: true) + remove_column(:requirements, :title_html, if_exists: true) + remove_column(:requirements, :description, if_exists: true) + remove_column(:requirements, :description_html, if_exists: true) + with_lock_retries do + remove_column(:requirements, :author_id, if_exists: true) + end + end + + def down + add_column(:requirements, :created_at, :datetime_with_timezone, if_not_exists: true) + add_column(:requirements, :updated_at, :datetime_with_timezone, if_not_exists: true) + add_column(:requirements, :author_id, :integer, if_not_exists: true) + add_column(:requirements, :cached_markdown_version, :integer, if_not_exists: true) + add_column(:requirements, :state, :smallint, default: 1, if_not_exists: true) + add_column(:requirements, :title, :string, limit: 255, if_not_exists: true) + add_column(:requirements, :title_html, :text, if_not_exists: true) + add_column(:requirements, :description, :text, if_not_exists: true) + add_column(:requirements, :description_html, :text, if_not_exists: true) + + add_check_constraint(:requirements, "char_length(description) <= 10000", CONSTRAINT_NAME) + + add_concurrent_foreign_key(:requirements, :users, column: :author_id, name: FOREIGN_KEY, on_delete: :nullify) + + add_concurrent_index(:requirements, :created_at) + add_concurrent_index(:requirements, :updated_at) + add_concurrent_index(:requirements, :author_id) + add_concurrent_index(:requirements, :state) + add_concurrent_index(:requirements, :title, name: NAME_INDEX, using: :gin, opclass: { name: :gin_trgm_ops }) + end +end |