From ee664acb356f8123f4f6b00b73c1e1cf0866c7fb Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 20 Oct 2022 09:40:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@15-5-stable-ee --- ...02011_schedule_destroy_invalid_group_members.rb | 14 +---- .../20220831090454_cleanup_web_hooks_service_id.rb | 13 ++++ ...725_schedule_destroy_invalid_project_members.rb | 14 +---- .../20220905120848_backfill_epic_cache_counts.rb | 29 +++++++++ ...curity_orchestration_policy_configuration_id.rb | 38 ++++++++++++ ...dex_for_vulnerability_reads_container_images.rb | 19 ++++++ ...0220916054103_remove_experience_level_column.rb | 13 ++++ ..._index_id_partition_id_to_ci_builds_metadata.rb | 16 +++++ ...x_build_id_partition_id_to_ci_build_metadata.rb | 16 +++++ ...p_unused_fields_from_merge_request_assignees.rb | 40 ++++++++++++ ...p_unused_fields_from_merge_request_reviewers.rb | 26 ++++++++ ..._callouts_on_attention_request_feature_names.rb | 18 ++++++ ...builds_metadata_for_partitioning_primary_key.rb | 36 +++++++++++ ...chedule_index_removal_for_ci_builds_metadata.rb | 16 +++++ .../20220920124709_backfill_internal_on_notes.rb | 26 ++++++++ .../20220920135356_tiebreak_user_type_index.rb | 18 ++++++ .../20220920141001_add_index_on_internal_notes.rb | 15 +++++ ...920213504_finalize_task_system_note_renaming.rb | 11 ++++ ...24_remove_task_system_note_rename_temp_index.rb | 11 ++++ ...21093355_schedule_backfill_namespace_details.rb | 29 +++++++++ ...d_ci_pipeline_variables_raw_with_new_default.rb | 10 +++ ..._add_ci_group_variables_raw_with_new_default.rb | 10 +++ ...d_ci_instance_variables_raw_with_new_default.rb | 10 +++ ...nd_add_ci_job_variables_raw_with_new_default.rb | 10 +++ ...line_schedule_variables_raw_with_new_default.rb | 10 +++ ...ve_and_add_ci_variables_raw_with_new_default.rb | 10 +++ ...220921144258_remove_orphan_group_token_users.rb | 40 ++++++++++++ ...922090236_remove_trace_column_from_ci_builds.rb | 9 +++ ...dule_reset_duplicate_ci_runners_token_values.rb | 28 +++++++++ ..._duplicate_ci_runners_token_encrypted_values.rb | 28 +++++++++ ...x_for_requested_non_invited_awaiting_members.rb | 17 +++++ ...quest_reviewers_on_attention_requested_state.rb | 18 ++++++ ...m_note_metadata_on_attention_request_actions.rb | 17 +++++ ...e_update_ci_pipeline_artifacts_locked_status.rb | 27 ++++++++ ...20929081645_tmp_idx_null_member_namespace_id.rb | 17 +++++ ...929091500_add_tmp_index_vulns_on_report_type.rb | 22 +++++++ ...01947_delete_remove_invalid_member_migration.rb | 19 ++++++ ...20929194453_add_index_releases_project_id_id.rb | 15 +++++ ..._delete_orphaned_operational_vulnerabilities.rb | 25 ++++++++ ..._unique_index_build_id_to_ci_builds_metadata.rb | 16 +++++ ...ed_on_default_branch_to_vulnerabilities_read.rb | 18 ++++++ ..._index_members_on_id_where_namespace_id_null.rb | 17 +++++ ...21004094814_schedule_destroy_invalid_members.rb | 27 ++++++++ ...on_scan_finding_approval_merge_request_rules.rb | 17 +++++ ...partial_trigram_indexes_for_issues_attempt_2.rb | 25 ++++++++ ...task_note_rename_background_migration_values.rb | 72 ++++++++++++++++++++++ ...4914_migrate_security_findings_delete_queues.rb | 16 +++++ ...uthor_id_target_project_id_on_merge_requests.rb | 15 +++++ ...ompliance_framework_id_to_namespace_settings.rb | 7 +++ ...uthor_id_target_project_id_on_merge_requests.rb | 15 +++++ ..._settings_on_default_compliance_framework_id.rb | 15 +++++ ...ompliance_framework_fk_to_namespace_settings.rb | 16 +++++ ...nd_add_ci_job_variables_raw_with_new_default.rb | 12 ++++ ...espace_settings_cohort_free_user_cap_columns.rb | 15 +++++ ...13154159_update_invalid_dormant_user_setting.rb | 17 +++++ 55 files changed, 1058 insertions(+), 22 deletions(-) create mode 100644 db/post_migrate/20220831090454_cleanup_web_hooks_service_id.rb create mode 100644 db/post_migrate/20220905120848_backfill_epic_cache_counts.rb create mode 100644 db/post_migrate/20220907122648_populate_security_orchestration_policy_configuration_id.rb create mode 100644 db/post_migrate/20220915192521_prepare_async_trigram_index_for_vulnerability_reads_container_images.rb create mode 100644 db/post_migrate/20220916054103_remove_experience_level_column.rb create mode 100644 db/post_migrate/20220916065418_add_index_id_partition_id_to_ci_builds_metadata.rb create mode 100644 db/post_migrate/20220916070101_add_index_build_id_partition_id_to_ci_build_metadata.rb create mode 100644 db/post_migrate/20220919023208_drop_unused_fields_from_merge_request_assignees.rb create mode 100644 db/post_migrate/20220919041604_drop_unused_fields_from_merge_request_reviewers.rb create mode 100644 db/post_migrate/20220919050555_drop_tmp_index_user_callouts_on_attention_request_feature_names.rb create mode 100644 db/post_migrate/20220920081631_prepare_ci_builds_metadata_for_partitioning_primary_key.rb create mode 100644 db/post_migrate/20220920122121_schedule_index_removal_for_ci_builds_metadata.rb create mode 100644 db/post_migrate/20220920124709_backfill_internal_on_notes.rb create mode 100644 db/post_migrate/20220920135356_tiebreak_user_type_index.rb create mode 100644 db/post_migrate/20220920141001_add_index_on_internal_notes.rb create mode 100644 db/post_migrate/20220920213504_finalize_task_system_note_renaming.rb create mode 100644 db/post_migrate/20220920214524_remove_task_system_note_rename_temp_index.rb create mode 100644 db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb create mode 100644 db/post_migrate/20220921111823_remove_and_add_ci_pipeline_variables_raw_with_new_default.rb create mode 100644 db/post_migrate/20220921111831_remove_and_add_ci_group_variables_raw_with_new_default.rb create mode 100644 db/post_migrate/20220921111840_remove_and_add_ci_instance_variables_raw_with_new_default.rb create mode 100644 db/post_migrate/20220921111849_remove_and_add_ci_job_variables_raw_with_new_default.rb create mode 100644 db/post_migrate/20220921111857_remove_and_add_ci_pipeline_schedule_variables_raw_with_new_default.rb create mode 100644 db/post_migrate/20220921111907_remove_and_add_ci_variables_raw_with_new_default.rb create mode 100644 db/post_migrate/20220921144258_remove_orphan_group_token_users.rb create mode 100644 db/post_migrate/20220922090236_remove_trace_column_from_ci_builds.rb create mode 100644 db/post_migrate/20220922143143_schedule_reset_duplicate_ci_runners_token_values.rb create mode 100644 db/post_migrate/20220922143634_schedule_reset_duplicate_ci_runners_token_encrypted_values.rb create mode 100644 db/post_migrate/20220922204106_remove_index_for_requested_non_invited_awaiting_members.rb create mode 100644 db/post_migrate/20220923052531_remove_tmp_index_merge_request_reviewers_on_attention_requested_state.rb create mode 100644 db/post_migrate/20220923060226_remove_tmp_index_system_note_metadata_on_attention_request_actions.rb create mode 100644 db/post_migrate/20220928225711_schedule_update_ci_pipeline_artifacts_locked_status.rb create mode 100644 db/post_migrate/20220929081645_tmp_idx_null_member_namespace_id.rb create mode 100644 db/post_migrate/20220929091500_add_tmp_index_vulns_on_report_type.rb create mode 100644 db/post_migrate/20220929101947_delete_remove_invalid_member_migration.rb create mode 100644 db/post_migrate/20220929194453_add_index_releases_project_id_id.rb create mode 100644 db/post_migrate/20220929213730_schedule_delete_orphaned_operational_vulnerabilities.rb create mode 100644 db/post_migrate/20220930110127_remove_unique_index_build_id_to_ci_builds_metadata.rb create mode 100644 db/post_migrate/20221003192827_add_index_resolved_on_default_branch_to_vulnerabilities_read.rb create mode 100644 db/post_migrate/20221004092038_tmp_index_members_on_id_where_namespace_id_null.rb create mode 100644 db/post_migrate/20221004094814_schedule_destroy_invalid_members.rb create mode 100644 db/post_migrate/20221005103000_add_index_merge_request_id_on_scan_finding_approval_merge_request_rules.rb create mode 100644 db/post_migrate/20221006083240_prepare_partial_trigram_indexes_for_issues_attempt_2.rb create mode 100644 db/post_migrate/20221006172302_adjust_task_note_rename_background_migration_values.rb create mode 100644 db/post_migrate/20221010074914_migrate_security_findings_delete_queues.rb create mode 100644 db/post_migrate/20221010121510_prepare_async_index_author_id_target_project_id_on_merge_requests.rb create mode 100644 db/post_migrate/20221010121807_add_compliance_framework_id_to_namespace_settings.rb create mode 100644 db/post_migrate/20221010122138_add_async_index_author_id_target_project_id_on_merge_requests.rb create mode 100644 db/post_migrate/20221010122352_add_index_to_namespace_settings_on_default_compliance_framework_id.rb create mode 100644 db/post_migrate/20221010123040_add_compliance_framework_fk_to_namespace_settings.rb create mode 100644 db/post_migrate/20221011113928_redo_remove_and_add_ci_job_variables_raw_with_new_default.rb create mode 100644 db/post_migrate/20221012182701_remove_namespace_settings_cohort_free_user_cap_columns.rb create mode 100644 db/post_migrate/20221013154159_update_invalid_dormant_user_setting.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20220809002011_schedule_destroy_invalid_group_members.rb b/db/post_migrate/20220809002011_schedule_destroy_invalid_group_members.rb index 3db21c290b9..e5d97d890ab 100644 --- a/db/post_migrate/20220809002011_schedule_destroy_invalid_group_members.rb +++ b/db/post_migrate/20220809002011_schedule_destroy_invalid_group_members.rb @@ -12,19 +12,11 @@ class ScheduleDestroyInvalidGroupMembers < Gitlab::Database::Migration[2.0] disable_ddl_transaction! def up - queue_batched_background_migration( - MIGRATION, - :members, - :id, - job_interval: DELAY_INTERVAL, - batch_size: BATCH_SIZE, - max_batch_size: MAX_BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE, - gitlab_schema: :gitlab_main - ) + # no-op + # We want to no-op this due to potential inconsistencies in SM upgrade path end def down - delete_batched_background_migration(MIGRATION, :members, :id, []) + # no-op end end diff --git a/db/post_migrate/20220831090454_cleanup_web_hooks_service_id.rb b/db/post_migrate/20220831090454_cleanup_web_hooks_service_id.rb new file mode 100644 index 00000000000..7beb89038e5 --- /dev/null +++ b/db/post_migrate/20220831090454_cleanup_web_hooks_service_id.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class CleanupWebHooksServiceId < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + cleanup_concurrent_column_rename :web_hooks, :service_id, :integration_id + end + + def down + undo_cleanup_concurrent_column_rename :web_hooks, :service_id, :integration_id + end +end diff --git a/db/post_migrate/20220901035725_schedule_destroy_invalid_project_members.rb b/db/post_migrate/20220901035725_schedule_destroy_invalid_project_members.rb index bc90232f855..9de3ac34669 100644 --- a/db/post_migrate/20220901035725_schedule_destroy_invalid_project_members.rb +++ b/db/post_migrate/20220901035725_schedule_destroy_invalid_project_members.rb @@ -10,19 +10,11 @@ class ScheduleDestroyInvalidProjectMembers < Gitlab::Database::Migration[2.0] restrict_gitlab_migration gitlab_schema: :gitlab_main def up - queue_batched_background_migration( - MIGRATION, - :members, - :id, - job_interval: DELAY_INTERVAL, - batch_size: BATCH_SIZE, - max_batch_size: MAX_BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE, - gitlab_schema: :gitlab_main - ) + # no-op + # We want to no-op this due to potential inconsistencies in SM upgrade path end def down - delete_batched_background_migration(MIGRATION, :members, :id, []) + # no-op end end diff --git a/db/post_migrate/20220905120848_backfill_epic_cache_counts.rb b/db/post_migrate/20220905120848_backfill_epic_cache_counts.rb new file mode 100644 index 00000000000..32fe04eb99c --- /dev/null +++ b/db/post_migrate/20220905120848_backfill_epic_cache_counts.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class BackfillEpicCacheCounts < Gitlab::Database::Migration[2.0] + MIGRATION = 'BackfillEpicCacheCounts' + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 200 + MAX_BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 20 + + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :epics, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main + ) + end + + def down + delete_batched_background_migration(MIGRATION, :epics, :id, []) + end +end diff --git a/db/post_migrate/20220907122648_populate_security_orchestration_policy_configuration_id.rb b/db/post_migrate/20220907122648_populate_security_orchestration_policy_configuration_id.rb new file mode 100644 index 00000000000..441113c0ba3 --- /dev/null +++ b/db/post_migrate/20220907122648_populate_security_orchestration_policy_configuration_id.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class PopulateSecurityOrchestrationPolicyConfigurationId < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_main + + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 500 + MERGE_REQUEST_MIGRATION = 'PopulateApprovalMergeRequestRulesWithSecurityOrchestration' + PROJECT_MIGRATION = 'PopulateApprovalProjectRulesWithSecurityOrchestration' + INTERVAL = 2.minutes + + def up + return unless Gitlab.ee? + + queue_batched_background_migration( + PROJECT_MIGRATION, + :approval_project_rules, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + + queue_batched_background_migration( + MERGE_REQUEST_MIGRATION, + :approval_merge_request_rules, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(PROJECT_MIGRATION, :approval_project_rules, :id, []) + delete_batched_background_migration(MERGE_REQUEST_MIGRATION, :approval_merge_request_rules, :id, []) + end +end diff --git a/db/post_migrate/20220915192521_prepare_async_trigram_index_for_vulnerability_reads_container_images.rb b/db/post_migrate/20220915192521_prepare_async_trigram_index_for_vulnerability_reads_container_images.rb new file mode 100644 index 00000000000..18ed1b2cd2b --- /dev/null +++ b/db/post_migrate/20220915192521_prepare_async_trigram_index_for_vulnerability_reads_container_images.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class PrepareAsyncTrigramIndexForVulnerabilityReadsContainerImages < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'index_vulnerability_reads_on_location_image_trigram' + REPORT_TYPES = { container_scanning: 2, cluster_image_scanning: 7 }.freeze + + def up + prepare_async_index :vulnerability_reads, :location_image, + name: INDEX_NAME, + using: :gin, opclass: { location_image: :gin_trgm_ops }, + where: "report_type = ANY (ARRAY[#{REPORT_TYPES.values.join(', ')}]) AND location_image IS NOT NULL" + end + + def down + unprepare_async_index :vulnerability_reads, :location_image, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220916054103_remove_experience_level_column.rb b/db/post_migrate/20220916054103_remove_experience_level_column.rb new file mode 100644 index 00000000000..dff5a9a6580 --- /dev/null +++ b/db/post_migrate/20220916054103_remove_experience_level_column.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class RemoveExperienceLevelColumn < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def up + remove_column :user_preferences, :experience_level + end + + def down + add_column :user_preferences, :experience_level, :integer, limit: 2 + end +end diff --git a/db/post_migrate/20220916065418_add_index_id_partition_id_to_ci_builds_metadata.rb b/db/post_migrate/20220916065418_add_index_id_partition_id_to_ci_builds_metadata.rb new file mode 100644 index 00000000000..c781f2a9ef9 --- /dev/null +++ b/db/post_migrate/20220916065418_add_index_id_partition_id_to_ci_builds_metadata.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexIdPartitionIdToCiBuildsMetadata < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + TABLE_NAME = :ci_builds_metadata + INDEX_NAME = :index_ci_builds_metadata_on_id_partition_id_unique + + def up + add_concurrent_index(TABLE_NAME, %i[id partition_id], unique: true, name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME) + end +end diff --git a/db/post_migrate/20220916070101_add_index_build_id_partition_id_to_ci_build_metadata.rb b/db/post_migrate/20220916070101_add_index_build_id_partition_id_to_ci_build_metadata.rb new file mode 100644 index 00000000000..18c24c47325 --- /dev/null +++ b/db/post_migrate/20220916070101_add_index_build_id_partition_id_to_ci_build_metadata.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexBuildIdPartitionIdToCiBuildMetadata < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + TABLE_NAME = :ci_builds_metadata + INDEX_NAME = :index_ci_builds_metadata_on_build_id_partition_id_unique + + def up + add_concurrent_index(TABLE_NAME, %i[build_id partition_id], unique: true, name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME) + end +end diff --git a/db/post_migrate/20220919023208_drop_unused_fields_from_merge_request_assignees.rb b/db/post_migrate/20220919023208_drop_unused_fields_from_merge_request_assignees.rb new file mode 100644 index 00000000000..f1afbb41893 --- /dev/null +++ b/db/post_migrate/20220919023208_drop_unused_fields_from_merge_request_assignees.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class DropUnusedFieldsFromMergeRequestAssignees < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + with_lock_retries do + if column_exists?(:merge_request_assignees, :state) # rubocop:disable Style/IfUnlessModifier + remove_column :merge_request_assignees, :state + end + + if column_exists?(:merge_request_assignees, :updated_state_by_user_id) + remove_column :merge_request_assignees, :updated_state_by_user_id + end + end + end + + def down + with_lock_retries do + unless column_exists?(:merge_request_assignees, :state) + add_column :merge_request_assignees, :state, :smallint, default: 0, null: false + end + + unless column_exists?(:merge_request_assignees, :updated_state_by_user_id) + add_column :merge_request_assignees, :updated_state_by_user_id, :bigint + end + end + + add_concurrent_index :merge_request_assignees, [:user_id, :state], + where: 'state = 2', + name: 'index_on_merge_request_assignees_user_id_and_state' + + add_concurrent_index :merge_request_assignees, :updated_state_by_user_id, + name: 'index_on_merge_request_assignees_updated_state_by_user_id' + + add_concurrent_foreign_key :merge_request_assignees, :users, + column: :updated_state_by_user_id, + on_delete: :nullify + end +end diff --git a/db/post_migrate/20220919041604_drop_unused_fields_from_merge_request_reviewers.rb b/db/post_migrate/20220919041604_drop_unused_fields_from_merge_request_reviewers.rb new file mode 100644 index 00000000000..d9f98557f75 --- /dev/null +++ b/db/post_migrate/20220919041604_drop_unused_fields_from_merge_request_reviewers.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class DropUnusedFieldsFromMergeRequestReviewers < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + with_lock_retries do + if column_exists?(:merge_request_reviewers, :updated_state_by_user_id) + remove_column :merge_request_reviewers, :updated_state_by_user_id + end + end + end + + def down + with_lock_retries do + unless column_exists?(:merge_request_reviewers, :updated_state_by_user_id) + add_column :merge_request_reviewers, :updated_state_by_user_id, :bigint + end + end + + add_concurrent_index :merge_request_reviewers, :updated_state_by_user_id, + name: 'index_on_merge_request_reviewers_updated_state_by_user_id' + + add_concurrent_foreign_key :merge_request_reviewers, :users, column: :updated_state_by_user_id, on_delete: :nullify + end +end diff --git a/db/post_migrate/20220919050555_drop_tmp_index_user_callouts_on_attention_request_feature_names.rb b/db/post_migrate/20220919050555_drop_tmp_index_user_callouts_on_attention_request_feature_names.rb new file mode 100644 index 00000000000..e456a91fe3f --- /dev/null +++ b/db/post_migrate/20220919050555_drop_tmp_index_user_callouts_on_attention_request_feature_names.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class DropTmpIndexUserCalloutsOnAttentionRequestFeatureNames < Gitlab::Database::Migration[2.0] + INDEX_NAME = "tmp_index_user_callouts_on_attention_request_feature_names" + ATTENTION_REQUEST_CALLOUTS = [47, 48] + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :user_callouts, INDEX_NAME + end + + def down + add_concurrent_index :user_callouts, [:id], + where: "feature_name IN (#{ATTENTION_REQUEST_CALLOUTS.join(',')})", + name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220920081631_prepare_ci_builds_metadata_for_partitioning_primary_key.rb b/db/post_migrate/20220920081631_prepare_ci_builds_metadata_for_partitioning_primary_key.rb new file mode 100644 index 00000000000..90727178624 --- /dev/null +++ b/db/post_migrate/20220920081631_prepare_ci_builds_metadata_for_partitioning_primary_key.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class PrepareCiBuildsMetadataForPartitioningPrimaryKey < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + TABLE_NAME = 'ci_builds_metadata' + PRIMARY_KEY = 'ci_builds_metadata_pkey' + NEW_INDEX_NAME = 'index_ci_builds_metadata_on_id_partition_id_unique' + OLD_INDEX_NAME = 'index_ci_builds_metadata_on_id_unique' + + def up + with_lock_retries(raise_on_exhaustion: true) do + execute("ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT #{PRIMARY_KEY} CASCADE") + + rename_index(TABLE_NAME, NEW_INDEX_NAME, PRIMARY_KEY) + + execute("ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT #{PRIMARY_KEY} " \ + "PRIMARY KEY USING INDEX #{PRIMARY_KEY}") + end + end + + # rolling back this migration is time consuming with the creation of these two indexes + def down + add_concurrent_index(TABLE_NAME, :id, unique: true, name: OLD_INDEX_NAME) + add_concurrent_index(TABLE_NAME, [:id, :partition_id], unique: true, name: NEW_INDEX_NAME) + + with_lock_retries(raise_on_exhaustion: true) do + execute("ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT #{PRIMARY_KEY} CASCADE") + + rename_index(TABLE_NAME, OLD_INDEX_NAME, PRIMARY_KEY) + + execute("ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT #{PRIMARY_KEY} " \ + "PRIMARY KEY USING INDEX #{PRIMARY_KEY}") + end + end +end diff --git a/db/post_migrate/20220920122121_schedule_index_removal_for_ci_builds_metadata.rb b/db/post_migrate/20220920122121_schedule_index_removal_for_ci_builds_metadata.rb new file mode 100644 index 00000000000..57475a08c93 --- /dev/null +++ b/db/post_migrate/20220920122121_schedule_index_removal_for_ci_builds_metadata.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class ScheduleIndexRemovalForCiBuildsMetadata < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + TABLE_NAME = :ci_builds_metadata + INDEX_NAME = :index_ci_builds_metadata_on_build_id + + def up + prepare_async_index_removal(TABLE_NAME, :build_id, name: INDEX_NAME) + end + + def down + unprepare_async_index(TABLE_NAME, :build_id, name: INDEX_NAME) + end +end diff --git a/db/post_migrate/20220920124709_backfill_internal_on_notes.rb b/db/post_migrate/20220920124709_backfill_internal_on_notes.rb new file mode 100644 index 00000000000..0d737195907 --- /dev/null +++ b/db/post_migrate/20220920124709_backfill_internal_on_notes.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class BackfillInternalOnNotes < Gitlab::Database::Migration[2.0] + MIGRATION = 'BackfillInternalOnNotes' + DELAY_INTERVAL = 2.minutes + TABLE = :notes + BATCH_SIZE = 2000 + SUB_BATCH_SIZE = 10 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + TABLE, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, TABLE, :id, []) + end +end diff --git a/db/post_migrate/20220920135356_tiebreak_user_type_index.rb b/db/post_migrate/20220920135356_tiebreak_user_type_index.rb new file mode 100644 index 00000000000..778a957086f --- /dev/null +++ b/db/post_migrate/20220920135356_tiebreak_user_type_index.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class TiebreakUserTypeIndex < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + NEW_INDEX_NAME = 'index_users_on_user_type_and_id' + OLD_INDEX_NAME = 'index_users_on_user_type' + + def up + add_concurrent_index :users, [:user_type, :id], name: NEW_INDEX_NAME + remove_concurrent_index_by_name :users, OLD_INDEX_NAME + end + + def down + add_concurrent_index :users, :user_type, name: OLD_INDEX_NAME + remove_concurrent_index_by_name :users, NEW_INDEX_NAME + end +end diff --git a/db/post_migrate/20220920141001_add_index_on_internal_notes.rb b/db/post_migrate/20220920141001_add_index_on_internal_notes.rb new file mode 100644 index 00000000000..466ad7b5a8d --- /dev/null +++ b/db/post_migrate/20220920141001_add_index_on_internal_notes.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexOnInternalNotes < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_notes_on_id_where_internal' + + disable_ddl_transaction! + + def up + add_concurrent_index :notes, :id, where: 'internal = true', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :notes, INDEX_NAME + end +end diff --git a/db/post_migrate/20220920213504_finalize_task_system_note_renaming.rb b/db/post_migrate/20220920213504_finalize_task_system_note_renaming.rb new file mode 100644 index 00000000000..d9307c14ccb --- /dev/null +++ b/db/post_migrate/20220920213504_finalize_task_system_note_renaming.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class FinalizeTaskSystemNoteRenaming < Gitlab::Database::Migration[2.0] + def up + # no-op + end + + def down + # noop + end +end diff --git a/db/post_migrate/20220920214524_remove_task_system_note_rename_temp_index.rb b/db/post_migrate/20220920214524_remove_task_system_note_rename_temp_index.rb new file mode 100644 index 00000000000..d3671d24578 --- /dev/null +++ b/db/post_migrate/20220920214524_remove_task_system_note_rename_temp_index.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class RemoveTaskSystemNoteRenameTempIndex < Gitlab::Database::Migration[2.0] + def up + # no-op + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb b/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb new file mode 100644 index 00000000000..16ce9bd5c85 --- /dev/null +++ b/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class ScheduleBackfillNamespaceDetails < Gitlab::Database::Migration[2.0] + MIGRATION = 'BackfillNamespaceDetails' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 200 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :namespaces, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :namespaces, :id, []) + end +end diff --git a/db/post_migrate/20220921111823_remove_and_add_ci_pipeline_variables_raw_with_new_default.rb b/db/post_migrate/20220921111823_remove_and_add_ci_pipeline_variables_raw_with_new_default.rb new file mode 100644 index 00000000000..fd56a8f320a --- /dev/null +++ b/db/post_migrate/20220921111823_remove_and_add_ci_pipeline_variables_raw_with_new_default.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveAndAddCiPipelineVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def change + remove_column :ci_pipeline_variables, :raw, :boolean, null: false, default: true + add_column :ci_pipeline_variables, :raw, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20220921111831_remove_and_add_ci_group_variables_raw_with_new_default.rb b/db/post_migrate/20220921111831_remove_and_add_ci_group_variables_raw_with_new_default.rb new file mode 100644 index 00000000000..f4b7de3faf4 --- /dev/null +++ b/db/post_migrate/20220921111831_remove_and_add_ci_group_variables_raw_with_new_default.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveAndAddCiGroupVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def change + remove_column :ci_group_variables, :raw, :boolean, null: false, default: true + add_column :ci_group_variables, :raw, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20220921111840_remove_and_add_ci_instance_variables_raw_with_new_default.rb b/db/post_migrate/20220921111840_remove_and_add_ci_instance_variables_raw_with_new_default.rb new file mode 100644 index 00000000000..019db1bbaa3 --- /dev/null +++ b/db/post_migrate/20220921111840_remove_and_add_ci_instance_variables_raw_with_new_default.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveAndAddCiInstanceVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def change + remove_column :ci_instance_variables, :raw, :boolean, null: false, default: true + add_column :ci_instance_variables, :raw, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20220921111849_remove_and_add_ci_job_variables_raw_with_new_default.rb b/db/post_migrate/20220921111849_remove_and_add_ci_job_variables_raw_with_new_default.rb new file mode 100644 index 00000000000..736cd46382d --- /dev/null +++ b/db/post_migrate/20220921111849_remove_and_add_ci_job_variables_raw_with_new_default.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveAndAddCiJobVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def change + remove_column :ci_job_variables, :raw, :boolean, null: false, default: true + add_column :ci_job_variables, :raw, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20220921111857_remove_and_add_ci_pipeline_schedule_variables_raw_with_new_default.rb b/db/post_migrate/20220921111857_remove_and_add_ci_pipeline_schedule_variables_raw_with_new_default.rb new file mode 100644 index 00000000000..159b0303d71 --- /dev/null +++ b/db/post_migrate/20220921111857_remove_and_add_ci_pipeline_schedule_variables_raw_with_new_default.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveAndAddCiPipelineScheduleVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def change + remove_column :ci_pipeline_schedule_variables, :raw, :boolean, null: false, default: true + add_column :ci_pipeline_schedule_variables, :raw, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20220921111907_remove_and_add_ci_variables_raw_with_new_default.rb b/db/post_migrate/20220921111907_remove_and_add_ci_variables_raw_with_new_default.rb new file mode 100644 index 00000000000..129e5b1f020 --- /dev/null +++ b/db/post_migrate/20220921111907_remove_and_add_ci_variables_raw_with_new_default.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveAndAddCiVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def change + remove_column :ci_variables, :raw, :boolean, null: false, default: true + add_column :ci_variables, :raw, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20220921144258_remove_orphan_group_token_users.rb b/db/post_migrate/20220921144258_remove_orphan_group_token_users.rb new file mode 100644 index 00000000000..a2483f611a3 --- /dev/null +++ b/db/post_migrate/20220921144258_remove_orphan_group_token_users.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class RemoveOrphanGroupTokenUsers < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_main + + disable_ddl_transaction! + + class MigrationUser < MigrationRecord + include EachBatch + + self.table_name = 'users' + + scope :project_bots, -> { where(user_type: 6) } + scope :without_memberships, -> { where("NOT EXISTS (SELECT 1 FROM members where members.user_id = users.id)") } + end + + class MigrationPersonalAccessToken < MigrationRecord + self.table_name = 'personal_access_tokens' + end + + def up + delete_worker = 'DeleteUserWorker'.safe_constantize + + MigrationUser.project_bots.each_batch(of: 1000) do |batch| + bot_ids = batch.without_memberships.pluck(:id) + + MigrationPersonalAccessToken.where(user_id: bot_ids).delete_all + + next unless delete_worker && delete_worker.respond_to?(:perform_async) + + bot_ids.each do |bot_id| + delete_worker.perform_async(bot_id, bot_id, skip_authorization: true) + end + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220922090236_remove_trace_column_from_ci_builds.rb b/db/post_migrate/20220922090236_remove_trace_column_from_ci_builds.rb new file mode 100644 index 00000000000..e1871a9799f --- /dev/null +++ b/db/post_migrate/20220922090236_remove_trace_column_from_ci_builds.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveTraceColumnFromCiBuilds < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def change + remove_column :ci_builds, :trace, :text + end +end diff --git a/db/post_migrate/20220922143143_schedule_reset_duplicate_ci_runners_token_values.rb b/db/post_migrate/20220922143143_schedule_reset_duplicate_ci_runners_token_values.rb new file mode 100644 index 00000000000..fd02bda8004 --- /dev/null +++ b/db/post_migrate/20220922143143_schedule_reset_duplicate_ci_runners_token_values.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class ScheduleResetDuplicateCiRunnersTokenValues < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_ci + disable_ddl_transaction! + + MIGRATION = 'ResetDuplicateCiRunnersTokenValues' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 2_000 + MAX_BATCH_SIZE = 100_000 + SUB_BATCH_SIZE = 500 + + def up + queue_batched_background_migration( + MIGRATION, + :ci_runners, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :ci_runners, :id, []) + end +end diff --git a/db/post_migrate/20220922143634_schedule_reset_duplicate_ci_runners_token_encrypted_values.rb b/db/post_migrate/20220922143634_schedule_reset_duplicate_ci_runners_token_encrypted_values.rb new file mode 100644 index 00000000000..bbbf9a1db44 --- /dev/null +++ b/db/post_migrate/20220922143634_schedule_reset_duplicate_ci_runners_token_encrypted_values.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class ScheduleResetDuplicateCiRunnersTokenEncryptedValues < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_ci + disable_ddl_transaction! + + MIGRATION = 'ResetDuplicateCiRunnersTokenEncryptedValues' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 2_000 + MAX_BATCH_SIZE = 100_000 + SUB_BATCH_SIZE = 500 + + def up + queue_batched_background_migration( + MIGRATION, + :ci_runners, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :ci_runners, :id, []) + end +end diff --git a/db/post_migrate/20220922204106_remove_index_for_requested_non_invited_awaiting_members.rb b/db/post_migrate/20220922204106_remove_index_for_requested_non_invited_awaiting_members.rb new file mode 100644 index 00000000000..033b04a75f8 --- /dev/null +++ b/db/post_migrate/20220922204106_remove_index_for_requested_non_invited_awaiting_members.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class RemoveIndexForRequestedNonInvitedAwaitingMembers < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_members_on_non_requested_non_invited_and_state_awaiting' + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :members, INDEX_NAME + end + + def down + clause = '((requested_at IS NULL) AND (invite_token IS NULL) AND (access_level > 5) AND (state = 1))' + + add_concurrent_index :members, :source_id, where: clause, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220923052531_remove_tmp_index_merge_request_reviewers_on_attention_requested_state.rb b/db/post_migrate/20220923052531_remove_tmp_index_merge_request_reviewers_on_attention_requested_state.rb new file mode 100644 index 00000000000..27d9c1641bc --- /dev/null +++ b/db/post_migrate/20220923052531_remove_tmp_index_merge_request_reviewers_on_attention_requested_state.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class RemoveTmpIndexMergeRequestReviewersOnAttentionRequestedState < Gitlab::Database::Migration[2.0] + INDEX_NAME = "tmp_index_merge_request_reviewers_on_attention_requested_state" + ATTENTION_REQUESTED_STATE = 2 + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :merge_request_reviewers, INDEX_NAME + end + + def down + add_concurrent_index :merge_request_reviewers, [:id], + where: "state = #{ATTENTION_REQUESTED_STATE}", + name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220923060226_remove_tmp_index_system_note_metadata_on_attention_request_actions.rb b/db/post_migrate/20220923060226_remove_tmp_index_system_note_metadata_on_attention_request_actions.rb new file mode 100644 index 00000000000..40e8c1cfdb5 --- /dev/null +++ b/db/post_migrate/20220923060226_remove_tmp_index_system_note_metadata_on_attention_request_actions.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class RemoveTmpIndexSystemNoteMetadataOnAttentionRequestActions < Gitlab::Database::Migration[2.0] + INDEX_NAME = "tmp_index_system_note_metadata_on_attention_request_actions" + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :system_note_metadata, INDEX_NAME + end + + def down + add_concurrent_index :system_note_metadata, [:id], + where: "action IN ('attention_requested', 'attention_request_removed')", + name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220928225711_schedule_update_ci_pipeline_artifacts_locked_status.rb b/db/post_migrate/20220928225711_schedule_update_ci_pipeline_artifacts_locked_status.rb new file mode 100644 index 00000000000..0d7a5dc4bec --- /dev/null +++ b/db/post_migrate/20220928225711_schedule_update_ci_pipeline_artifacts_locked_status.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class ScheduleUpdateCiPipelineArtifactsLockedStatus < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + MIGRATION = 'UpdateCiPipelineArtifactsUnknownLockedStatus' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 500 + + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + def up + queue_batched_background_migration( + MIGRATION, + :ci_pipeline_artifacts, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :ci_pipeline_artifacts, :id, []) + end +end diff --git a/db/post_migrate/20220929081645_tmp_idx_null_member_namespace_id.rb b/db/post_migrate/20220929081645_tmp_idx_null_member_namespace_id.rb new file mode 100644 index 00000000000..58d6c653e6f --- /dev/null +++ b/db/post_migrate/20220929081645_tmp_idx_null_member_namespace_id.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class TmpIdxNullMemberNamespaceId < Gitlab::Database::Migration[2.0] + TMP_INDEX_FOR_NULL_MEMBER_NAMESPACE_ID = 'tmp_index_for_null_member_namespace_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :members, :member_namespace_id, + name: TMP_INDEX_FOR_NULL_MEMBER_NAMESPACE_ID, + where: 'member_namespace_id IS NULL' + end + + def down + remove_concurrent_index_by_name :members, name: TMP_INDEX_FOR_NULL_MEMBER_NAMESPACE_ID + end +end diff --git a/db/post_migrate/20220929091500_add_tmp_index_vulns_on_report_type.rb b/db/post_migrate/20220929091500_add_tmp_index_vulns_on_report_type.rb new file mode 100644 index 00000000000..19100e1d0cc --- /dev/null +++ b/db/post_migrate/20220929091500_add_tmp_index_vulns_on_report_type.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +class AddTmpIndexVulnsOnReportType < Gitlab::Database::Migration[2.0] + # Temporary index to perform migration removing invalid vulnerabilities + INDEX_NAME = 'tmp_idx_vulnerabilities_on_id_where_report_type_7_99' + + REPORT_TYPES = { + cluster_image_scanning: 7, + custom: 99 + } + + disable_ddl_transaction! + + def up + add_concurrent_index :vulnerabilities, :id, + where: "report_type IN (#{REPORT_TYPES.values.join(', ')})", + name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :vulnerabilities, INDEX_NAME + end +end diff --git a/db/post_migrate/20220929101947_delete_remove_invalid_member_migration.rb b/db/post_migrate/20220929101947_delete_remove_invalid_member_migration.rb new file mode 100644 index 00000000000..6fb358e57ab --- /dev/null +++ b/db/post_migrate/20220929101947_delete_remove_invalid_member_migration.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class DeleteRemoveInvalidMemberMigration < Gitlab::Database::Migration[2.0] + PROJECT_MEMBER_MIGRATION = 'ScheduleDestroyInvalidProjectMembers' + GROUP_MEMBER_MIGRATION = 'ScheduleDestroyInvalidGroupMembers' + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + delete_batched_background_migration(PROJECT_MEMBER_MIGRATION, :members, :id, []) + delete_batched_background_migration(GROUP_MEMBER_MIGRATION, :members, :id, []) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20220929194453_add_index_releases_project_id_id.rb b/db/post_migrate/20220929194453_add_index_releases_project_id_id.rb new file mode 100644 index 00000000000..179b5a636f0 --- /dev/null +++ b/db/post_migrate/20220929194453_add_index_releases_project_id_id.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexReleasesProjectIdId < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_releases_on_project_id_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :releases, %i[project_id id], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :releases, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20220929213730_schedule_delete_orphaned_operational_vulnerabilities.rb b/db/post_migrate/20220929213730_schedule_delete_orphaned_operational_vulnerabilities.rb new file mode 100644 index 00000000000..c03358b1fda --- /dev/null +++ b/db/post_migrate/20220929213730_schedule_delete_orphaned_operational_vulnerabilities.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class ScheduleDeleteOrphanedOperationalVulnerabilities < Gitlab::Database::Migration[2.0] + MIGRATION = 'DeleteOrphanedOperationalVulnerabilities' + INTERVAL = 2.minutes + BATCH_SIZE = 10_000 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :vulnerabilities, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :vulnerabilities, :id, []) + end +end diff --git a/db/post_migrate/20220930110127_remove_unique_index_build_id_to_ci_builds_metadata.rb b/db/post_migrate/20220930110127_remove_unique_index_build_id_to_ci_builds_metadata.rb new file mode 100644 index 00000000000..4c23cdc5284 --- /dev/null +++ b/db/post_migrate/20220930110127_remove_unique_index_build_id_to_ci_builds_metadata.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class RemoveUniqueIndexBuildIdToCiBuildsMetadata < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + TABLE_NAME = :ci_builds_metadata + INDEX_NAME = :index_ci_builds_metadata_on_build_id + + def up + remove_concurrent_index_by_name(TABLE_NAME, INDEX_NAME) + end + + def down + add_concurrent_index(TABLE_NAME, :build_id, unique: true, name: INDEX_NAME) + end +end diff --git a/db/post_migrate/20221003192827_add_index_resolved_on_default_branch_to_vulnerabilities_read.rb b/db/post_migrate/20221003192827_add_index_resolved_on_default_branch_to_vulnerabilities_read.rb new file mode 100644 index 00000000000..e352e324187 --- /dev/null +++ b/db/post_migrate/20221003192827_add_index_resolved_on_default_branch_to_vulnerabilities_read.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddIndexResolvedOnDefaultBranchToVulnerabilitiesRead < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'index_vuln_reads_on_resolved_on_default_branch' + COLUMNS = %i[project_id state id] + + def up + add_concurrent_index :vulnerability_reads, COLUMNS, + where: 'resolved_on_default_branch IS TRUE', + name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :vulnerability_reads, INDEX_NAME + end +end diff --git a/db/post_migrate/20221004092038_tmp_index_members_on_id_where_namespace_id_null.rb b/db/post_migrate/20221004092038_tmp_index_members_on_id_where_namespace_id_null.rb new file mode 100644 index 00000000000..68a707e99c2 --- /dev/null +++ b/db/post_migrate/20221004092038_tmp_index_members_on_id_where_namespace_id_null.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class TmpIndexMembersOnIdWhereNamespaceIdNull < Gitlab::Database::Migration[2.0] + TMP_INDEX = 'tmp_index_members_on_id_where_namespace_id_null' + + disable_ddl_transaction! + + def up + add_concurrent_index :members, :id, + name: TMP_INDEX, + where: 'member_namespace_id IS NULL' + end + + def down + remove_concurrent_index_by_name :members, name: TMP_INDEX + end +end diff --git a/db/post_migrate/20221004094814_schedule_destroy_invalid_members.rb b/db/post_migrate/20221004094814_schedule_destroy_invalid_members.rb new file mode 100644 index 00000000000..fa6a1fd5f72 --- /dev/null +++ b/db/post_migrate/20221004094814_schedule_destroy_invalid_members.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true +class ScheduleDestroyInvalidMembers < Gitlab::Database::Migration[2.0] + MIGRATION = 'DestroyInvalidMembers' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 200 + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :members, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main + ) + end + + def down + delete_batched_background_migration(MIGRATION, :members, :id, []) + end +end diff --git a/db/post_migrate/20221005103000_add_index_merge_request_id_on_scan_finding_approval_merge_request_rules.rb b/db/post_migrate/20221005103000_add_index_merge_request_id_on_scan_finding_approval_merge_request_rules.rb new file mode 100644 index 00000000000..06bf21b894a --- /dev/null +++ b/db/post_migrate/20221005103000_add_index_merge_request_id_on_scan_finding_approval_merge_request_rules.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexMergeRequestIdOnScanFindingApprovalMergeRequestRules < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'scan_finding_approval_mr_rule_index_merge_request_id' + SCAN_FINDING_REPORT_TYPE = 4 + + disable_ddl_transaction! + + def up + add_concurrent_index :approval_merge_request_rules, :merge_request_id, + where: "report_type = #{SCAN_FINDING_REPORT_TYPE}", name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :approval_merge_request_rules, INDEX_NAME + end +end diff --git a/db/post_migrate/20221006083240_prepare_partial_trigram_indexes_for_issues_attempt_2.rb b/db/post_migrate/20221006083240_prepare_partial_trigram_indexes_for_issues_attempt_2.rb new file mode 100644 index 00000000000..6ca2ba222ae --- /dev/null +++ b/db/post_migrate/20221006083240_prepare_partial_trigram_indexes_for_issues_attempt_2.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class PreparePartialTrigramIndexesForIssuesAttempt2 < Gitlab::Database::Migration[2.0] + TITLE_INDEX_NAME = 'index_issues_on_title_trigram_non_latin' + DESCRIPTION_INDEX_NAME = 'index_issues_on_description_trigram_non_latin' + + def up + prepare_async_index :issues, :title, + name: TITLE_INDEX_NAME, + using: :gin, opclass: { description: :gin_trgm_ops }, + where: "title NOT SIMILAR TO '[\\u0000-\\u02FF\\u1E00-\\u1EFF\\u2070-\\u218F]*' " \ + "OR description NOT SIMILAR TO '[\\u0000-\\u02FF\\u1E00-\\u1EFF\\u2070-\\u218F]*'" + + prepare_async_index :issues, :description, + name: DESCRIPTION_INDEX_NAME, + using: :gin, opclass: { description: :gin_trgm_ops }, + where: "title NOT SIMILAR TO '[\\u0000-\\u02FF\\u1E00-\\u1EFF\\u2070-\\u218F]*' " \ + "OR description NOT SIMILAR TO '[\\u0000-\\u02FF\\u1E00-\\u1EFF\\u2070-\\u218F]*'" + end + + def down + unprepare_async_index_by_name :issues, DESCRIPTION_INDEX_NAME + unprepare_async_index_by_name :issues, TITLE_INDEX_NAME + end +end diff --git a/db/post_migrate/20221006172302_adjust_task_note_rename_background_migration_values.rb b/db/post_migrate/20221006172302_adjust_task_note_rename_background_migration_values.rb new file mode 100644 index 00000000000..2af16fb6d3c --- /dev/null +++ b/db/post_migrate/20221006172302_adjust_task_note_rename_background_migration_values.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +class AdjustTaskNoteRenameBackgroundMigrationValues < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_main + + JOB_CLASS_NAME = 'RenameTaskSystemNoteToChecklistItem' + MIGRATION_FAILED_STATUS = 4 + MIGRATION_FINISHED_STATUS = 3 + MIGRATION_ACTIVE_STATUS = 1 + JOB_FAILED_STATUS = 2 + + OLD_BATCH_SIZE = 10_000 + NEW_BATCH_SIZE = 5_000 + + OLD_SUB_BATCH_SIZE = 100 + NEW_SUB_BATCH_SIZE = 10 + + class InlineBatchedMigration < MigrationRecord + self.table_name = :batched_background_migrations + + scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do + where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) + .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals + end + end + + class InlineBatchedJob < MigrationRecord + include EachBatch + self.table_name = :batched_background_migration_jobs + end + + def up + migration = InlineBatchedMigration.for_configuration( + JOB_CLASS_NAME, + :system_note_metadata, + :id, + [] + ).first + return if migration.blank? || migration.status == MIGRATION_FINISHED_STATUS + + InlineBatchedJob.where( + batched_background_migration_id: migration.id, + status: JOB_FAILED_STATUS + ).each_batch(of: 100) do |batch| + batch.update_all(attempts: 0, sub_batch_size: NEW_SUB_BATCH_SIZE) + end + + update_params = { batch_size: NEW_BATCH_SIZE, sub_batch_size: NEW_SUB_BATCH_SIZE } + + if migration.status == MIGRATION_FAILED_STATUS + update_params[:status] = MIGRATION_ACTIVE_STATUS + update_params[:started_at] = Time.zone.now if migration.respond_to?(:started_at) + end + + migration.update!(**update_params) + end + + def down + migration = InlineBatchedMigration.for_configuration( + JOB_CLASS_NAME, + :system_note_metadata, + :id, + [] + ).first + return if migration.blank? + + migration.update!( + batch_size: OLD_BATCH_SIZE, + sub_batch_size: OLD_SUB_BATCH_SIZE + ) + end +end diff --git a/db/post_migrate/20221010074914_migrate_security_findings_delete_queues.rb b/db/post_migrate/20221010074914_migrate_security_findings_delete_queues.rb new file mode 100644 index 00000000000..ce593e19859 --- /dev/null +++ b/db/post_migrate/20221010074914_migrate_security_findings_delete_queues.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class MigrateSecurityFindingsDeleteQueues < Gitlab::Database::Migration[2.0] + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + def up + sidekiq_queue_migrate 'security_findings_delete_by_job_id', to: 'security_scans_purge_by_job_id' + sidekiq_queue_migrate 'cronjob:security_findings_cleanup', to: 'cronjob:security_scans_purge' + end + + def down + sidekiq_queue_migrate 'security_scans_purge_by_job_id', to: 'security_findings_delete_by_job_id' + sidekiq_queue_migrate 'cronjob:security_scans_purge', to: 'cronjob:security_findings_cleanup' + end +end diff --git a/db/post_migrate/20221010121510_prepare_async_index_author_id_target_project_id_on_merge_requests.rb b/db/post_migrate/20221010121510_prepare_async_index_author_id_target_project_id_on_merge_requests.rb new file mode 100644 index 00000000000..cd9035dd9f9 --- /dev/null +++ b/db/post_migrate/20221010121510_prepare_async_index_author_id_target_project_id_on_merge_requests.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class PrepareAsyncIndexAuthorIdTargetProjectIdOnMergeRequests < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_merge_requests_on_author_id_and_target_project_id' + + disable_ddl_transaction! + + def up + prepare_async_index :merge_requests, %i[author_id target_project_id], name: INDEX_NAME + end + + def down + unprepare_async_index :merge_requests, %i[author_id target_project_id], name: INDEX_NAME + end +end diff --git a/db/post_migrate/20221010121807_add_compliance_framework_id_to_namespace_settings.rb b/db/post_migrate/20221010121807_add_compliance_framework_id_to_namespace_settings.rb new file mode 100644 index 00000000000..e07f19242d5 --- /dev/null +++ b/db/post_migrate/20221010121807_add_compliance_framework_id_to_namespace_settings.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkIdToNamespaceSettings < Gitlab::Database::Migration[2.0] + def change + add_column :namespace_settings, :default_compliance_framework_id, :bigint + end +end diff --git a/db/post_migrate/20221010122138_add_async_index_author_id_target_project_id_on_merge_requests.rb b/db/post_migrate/20221010122138_add_async_index_author_id_target_project_id_on_merge_requests.rb new file mode 100644 index 00000000000..0689c090d09 --- /dev/null +++ b/db/post_migrate/20221010122138_add_async_index_author_id_target_project_id_on_merge_requests.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddAsyncIndexAuthorIdTargetProjectIdOnMergeRequests < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_merge_requests_on_author_id_and_id' + + disable_ddl_transaction! + + def up + prepare_async_index :merge_requests, %i[author_id id], name: INDEX_NAME + end + + def down + unprepare_async_index :merge_requests, %i[author_id id], name: INDEX_NAME + end +end diff --git a/db/post_migrate/20221010122352_add_index_to_namespace_settings_on_default_compliance_framework_id.rb b/db/post_migrate/20221010122352_add_index_to_namespace_settings_on_default_compliance_framework_id.rb new file mode 100644 index 00000000000..63de7942208 --- /dev/null +++ b/db/post_migrate/20221010122352_add_index_to_namespace_settings_on_default_compliance_framework_id.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexToNamespaceSettingsOnDefaultComplianceFrameworkId < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + INDEX_NAME = 'idx_namespace_settings_on_default_compliance_framework_id' + + def up + add_concurrent_index :namespace_settings, :default_compliance_framework_id, unique: true, name: INDEX_NAME + end + + def down + remove_concurrent_index :namespace_settings, :default_compliance_framework_id, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20221010123040_add_compliance_framework_fk_to_namespace_settings.rb b/db/post_migrate/20221010123040_add_compliance_framework_fk_to_namespace_settings.rb new file mode 100644 index 00000000000..5d4966fd580 --- /dev/null +++ b/db/post_migrate/20221010123040_add_compliance_framework_fk_to_namespace_settings.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkFkToNamespaceSettings < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :namespace_settings, :compliance_management_frameworks, + column: :default_compliance_framework_id, on_delete: :nullify, reverse_lock_order: true + end + + def down + with_lock_retries do + remove_foreign_key :namespace_settings, column: :default_compliance_framework_id + end + end +end diff --git a/db/post_migrate/20221011113928_redo_remove_and_add_ci_job_variables_raw_with_new_default.rb b/db/post_migrate/20221011113928_redo_remove_and_add_ci_job_variables_raw_with_new_default.rb new file mode 100644 index 00000000000..a377b78803e --- /dev/null +++ b/db/post_migrate/20221011113928_redo_remove_and_add_ci_job_variables_raw_with_new_default.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class RedoRemoveAndAddCiJobVariablesRawWithNewDefault < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + # We are removing and adding the same column with the same parameters to refresh the table + # because we have some wrong `ci_job_variables.raw` data (`TRUE`) in the database. + def change + remove_column :ci_job_variables, :raw, :boolean, null: false, default: false + add_column :ci_job_variables, :raw, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20221012182701_remove_namespace_settings_cohort_free_user_cap_columns.rb b/db/post_migrate/20221012182701_remove_namespace_settings_cohort_free_user_cap_columns.rb new file mode 100644 index 00000000000..36408ad19ed --- /dev/null +++ b/db/post_migrate/20221012182701_remove_namespace_settings_cohort_free_user_cap_columns.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveNamespaceSettingsCohortFreeUserCapColumns < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def up + remove_column :namespace_settings, :exclude_from_free_user_cap + remove_column :namespace_settings, :include_for_free_user_cap_preview + end + + def down + add_column :namespace_settings, :exclude_from_free_user_cap, :boolean, null: false, default: false + add_column :namespace_settings, :include_for_free_user_cap_preview, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20221013154159_update_invalid_dormant_user_setting.rb b/db/post_migrate/20221013154159_update_invalid_dormant_user_setting.rb new file mode 100644 index 00000000000..1f1e47fdac1 --- /dev/null +++ b/db/post_migrate/20221013154159_update_invalid_dormant_user_setting.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class UpdateInvalidDormantUserSetting < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + # rubocop:disable Layout/LineLength + def up + execute("update application_settings set deactivate_dormant_users_period=90 where deactivate_dormant_users_period < 90") + end + # rubocop:enable Layout/LineLength + + def down + # no-op + end +end -- cgit v1.2.3