diff options
Diffstat (limited to 'db/post_migrate')
34 files changed, 885 insertions, 1 deletions
diff --git a/db/post_migrate/20200710102418_delete_user_callout_alerts_moved.rb b/db/post_migrate/20200710102418_delete_user_callout_alerts_moved.rb index e14cd7ac3ee..ac605ba494e 100644 --- a/db/post_migrate/20200710102418_delete_user_callout_alerts_moved.rb +++ b/db/post_migrate/20200710102418_delete_user_callout_alerts_moved.rb @@ -13,7 +13,7 @@ class DeleteUserCalloutAlertsMoved < ActiveRecord::Migration[6.0] BATCH_SIZE = 1_000 - # Inlined from UserCalloutEnums.feature_names + # Inlined from Enums::UserCallout.feature_names FEATURE_NAME_ALERTS_MOVED = 20 def up diff --git a/db/post_migrate/20200810191256_remove_pipeline_id_from_test_reports.rb b/db/post_migrate/20200810191256_remove_pipeline_id_from_test_reports.rb new file mode 100644 index 00000000000..4a5e6942371 --- /dev/null +++ b/db/post_migrate/20200810191256_remove_pipeline_id_from_test_reports.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemovePipelineIdFromTestReports < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + remove_column :requirements_management_test_reports, :pipeline_id + end + + def down + add_column :requirements_management_test_reports, :pipeline_id, :integer + + with_lock_retries do + # rubocop:disable Migration/AddConcurrentForeignKey + add_foreign_key :requirements_management_test_reports, :ci_pipelines, column: :pipeline_id, on_delete: :nullify + # rubocop:enable Migration/AddConcurrentForeignKey + end + end +end diff --git a/db/post_migrate/20200811130000_create_index_vulnerabilities_feedback_issue_id_not_null.rb b/db/post_migrate/20200811130000_create_index_vulnerabilities_feedback_issue_id_not_null.rb new file mode 100644 index 00000000000..118076eb254 --- /dev/null +++ b/db/post_migrate/20200811130000_create_index_vulnerabilities_feedback_issue_id_not_null.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class CreateIndexVulnerabilitiesFeedbackIssueIdNotNull < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :vulnerability_feedback, :id, where: 'issue_id IS NOT NULL', + name: "index_vulnerability_feedback_on_issue_id_not_null" + end + + def down + remove_concurrent_index_by_name :vulnerability_feedback, + :index_vulnerability_feedback_on_issue_id_not_null + end +end diff --git a/db/post_migrate/20200811130433_create_missing_vulnerabilities_issue_links.rb b/db/post_migrate/20200811130433_create_missing_vulnerabilities_issue_links.rb new file mode 100644 index 00000000000..891201eaa52 --- /dev/null +++ b/db/post_migrate/20200811130433_create_missing_vulnerabilities_issue_links.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +class CreateMissingVulnerabilitiesIssueLinks < ActiveRecord::Migration[6.0] + class VulnerabilitiesFeedback < ActiveRecord::Base + include EachBatch + self.table_name = 'vulnerability_feedback' + end + + class VulnerabilitiesIssueLink < ActiveRecord::Base + self.table_name = 'vulnerability_issue_links' + LINK_TYPE_CREATED = 2 + end + + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + VulnerabilitiesFeedback.where('issue_id IS NOT NULL').each_batch do |relation| + timestamp = Time.now + issue_links = relation + .joins("JOIN vulnerability_occurrences vo ON vo.project_id = vulnerability_feedback.project_id AND vo.report_type = vulnerability_feedback.category AND encode(vo.project_fingerprint, 'hex') = vulnerability_feedback.project_fingerprint") + .where('vo.vulnerability_id IS NOT NULL') + .pluck(:vulnerability_id, :issue_id) + .map do |v_id, i_id| + { + vulnerability_id: v_id, + issue_id: i_id, + link_type: VulnerabilitiesIssueLink::LINK_TYPE_CREATED, + created_at: timestamp, + updated_at: timestamp + } + end + + next if issue_links.empty? + + VulnerabilitiesIssueLink.insert_all( + issue_links, + returning: false + ) + end + end + + def down + end +end diff --git a/db/post_migrate/20200811211536_add_index_to_ci_job_artifacts_for_terraform_reports_id.rb b/db/post_migrate/20200811211536_add_index_to_ci_job_artifacts_for_terraform_reports_id.rb new file mode 100644 index 00000000000..6042f90cf85 --- /dev/null +++ b/db/post_migrate/20200811211536_add_index_to_ci_job_artifacts_for_terraform_reports_id.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddIndexToCiJobArtifactsForTerraformReportsId < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_ci_job_artifacts_id_for_terraform_reports' + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_job_artifacts, :id, where: 'file_type = 18', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, INDEX_NAME + end +end diff --git a/db/post_migrate/20200817100710_add_section_and_created_at_codeowner_approval_merge_request_index.rb b/db/post_migrate/20200817100710_add_section_and_created_at_codeowner_approval_merge_request_index.rb new file mode 100644 index 00000000000..33de4f1f790 --- /dev/null +++ b/db/post_migrate/20200817100710_add_section_and_created_at_codeowner_approval_merge_request_index.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AddSectionAndCreatedAtCodeownerApprovalMergeRequestIndex < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + SECTION_CREATED_AT_ON_CODEOWNER_APPROVAL_MERGE_REQUEST_RULES = "index_created_at_on_codeowner_approval_merge_request_rules" + RULE_TYPE_CODEOWNERS = 2 + CODEOWNER_SECTION_DEFAULT = 'codeowners' + + disable_ddl_transaction! + + def up + add_concurrent_index :approval_merge_request_rules, :created_at, + where: "rule_type = #{RULE_TYPE_CODEOWNERS} AND section != '#{CODEOWNER_SECTION_DEFAULT}'::text", + name: SECTION_CREATED_AT_ON_CODEOWNER_APPROVAL_MERGE_REQUEST_RULES + end + + def down + remove_concurrent_index_by_name :approval_merge_request_rules, SECTION_CREATED_AT_ON_CODEOWNER_APPROVAL_MERGE_REQUEST_RULES + end +end diff --git a/db/post_migrate/20200819082334_remove_default_from_services.rb b/db/post_migrate/20200819082334_remove_default_from_services.rb new file mode 100644 index 00000000000..2a990016c95 --- /dev/null +++ b/db/post_migrate/20200819082334_remove_default_from_services.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RemoveDefaultFromServices < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + remove_column :services, :default, :boolean + end + end + + def down + with_lock_retries do + add_column :services, :default, :boolean, default: false + end + end +end diff --git a/db/post_migrate/20200819113644_add_target_id_to_audit_events.rb b/db/post_migrate/20200819113644_add_target_id_to_audit_events.rb new file mode 100644 index 00000000000..c2d9aff928b --- /dev/null +++ b/db/post_migrate/20200819113644_add_target_id_to_audit_events.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +class AddTargetIdToAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::SchemaHelpers + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + SOURCE_TABLE_NAME = 'audit_events' + PARTITIONED_TABLE_NAME = 'audit_events_part_5fc467ac26' + TRIGGER_FUNCTION_NAME = 'table_sync_function_2be879775d' + + def up + with_lock_retries do + add_column(SOURCE_TABLE_NAME, :target_id, :bigint) + add_column(PARTITIONED_TABLE_NAME, :target_id, :bigint) + + create_trigger_function(TRIGGER_FUNCTION_NAME, replace: true) do + <<~SQL + IF (TG_OP = 'DELETE') THEN + DELETE FROM #{PARTITIONED_TABLE_NAME} where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + UPDATE #{PARTITIONED_TABLE_NAME} + SET author_id = NEW.author_id, + type = NEW.type, + entity_id = NEW.entity_id, + entity_type = NEW.entity_type, + details = NEW.details, + ip_address = NEW.ip_address, + author_name = NEW.author_name, + entity_path = NEW.entity_path, + target_details = NEW.target_details, + target_type = NEW.target_type, + target_id = NEW.target_id, + created_at = NEW.created_at + WHERE #{PARTITIONED_TABLE_NAME}.id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO #{PARTITIONED_TABLE_NAME} (id, + author_id, + type, + entity_id, + entity_type, + details, + ip_address, + author_name, + entity_path, + target_details, + target_type, + target_id, + created_at) + VALUES (NEW.id, + NEW.author_id, + NEW.type, + NEW.entity_id, + NEW.entity_type, + NEW.details, + NEW.ip_address, + NEW.author_name, + NEW.entity_path, + NEW.target_details, + NEW.target_type, + NEW.target_id, + NEW.created_at); + END IF; + RETURN NULL; + SQL + end + end + end + + def down + with_lock_retries do + remove_column SOURCE_TABLE_NAME, :target_id + + create_trigger_function(TRIGGER_FUNCTION_NAME, replace: true) do + <<~SQL + IF (TG_OP = 'DELETE') THEN + DELETE FROM #{PARTITIONED_TABLE_NAME} where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + UPDATE #{PARTITIONED_TABLE_NAME} + SET author_id = NEW.author_id, + type = NEW.type, + entity_id = NEW.entity_id, + entity_type = NEW.entity_type, + details = NEW.details, + ip_address = NEW.ip_address, + author_name = NEW.author_name, + entity_path = NEW.entity_path, + target_details = NEW.target_details, + target_type = NEW.target_type, + created_at = NEW.created_at + WHERE #{PARTITIONED_TABLE_NAME}.id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO #{PARTITIONED_TABLE_NAME} (id, + author_id, + type, + entity_id, + entity_type, + details, + ip_address, + author_name, + entity_path, + target_details, + target_type, + created_at) + VALUES (NEW.id, + NEW.author_id, + NEW.type, + NEW.entity_id, + NEW.entity_type, + NEW.details, + NEW.ip_address, + NEW.author_name, + NEW.entity_path, + NEW.target_details, + NEW.target_type, + NEW.created_at); + END IF; + RETURN NULL; + SQL + end + + remove_column PARTITIONED_TABLE_NAME, :target_id + end + end +end diff --git a/db/post_migrate/20200819202048_remove_orphaned_emails.rb b/db/post_migrate/20200819202048_remove_orphaned_emails.rb new file mode 100644 index 00000000000..82cba244ad6 --- /dev/null +++ b/db/post_migrate/20200819202048_remove_orphaned_emails.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class RemoveOrphanedEmails < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + execute <<~SQL + DELETE FROM emails + WHERE not exists ( + SELECT 1 FROM users WHERE users.id = emails.user_id + ); + SQL + + execute 'DELETE FROM emails WHERE user_id IS NULL;' + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200819202222_validate_emails_user_id_foreign_key.rb b/db/post_migrate/20200819202222_validate_emails_user_id_foreign_key.rb new file mode 100644 index 00000000000..4f7f9deb540 --- /dev/null +++ b/db/post_migrate/20200819202222_validate_emails_user_id_foreign_key.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class ValidateEmailsUserIdForeignKey < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + CONSTRAINT_NAME = 'fk_emails_user_id' + + def up + validate_foreign_key :emails, :user_id, name: CONSTRAINT_NAME + end + + def down + # no op + end +end diff --git a/db/post_migrate/20200824130028_remove_index_on_users_bio.rb b/db/post_migrate/20200824130028_remove_index_on_users_bio.rb new file mode 100644 index 00000000000..d41734eb9ce --- /dev/null +++ b/db/post_migrate/20200824130028_remove_index_on_users_bio.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class RemoveIndexOnUsersBio < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BACKGROUND_MIGRATION_CLASS = 'MigrateUsersBioToUserDetails' + INDEX_NAME = 'tmp_idx_on_user_id_where_bio_is_filled' + + disable_ddl_transaction! + + def up + Gitlab::BackgroundMigration.steal(BACKGROUND_MIGRATION_CLASS) + remove_concurrent_index_by_name(:users, INDEX_NAME) + end + + def down + add_concurrent_index :users, :id, where: "(COALESCE(bio, '') IS DISTINCT FROM '')", name: INDEX_NAME + end +end diff --git a/db/post_migrate/20200824130447_remove_users_bio_column.rb b/db/post_migrate/20200824130447_remove_users_bio_column.rb new file mode 100644 index 00000000000..cb630ec7fd5 --- /dev/null +++ b/db/post_migrate/20200824130447_remove_users_bio_column.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RemoveUsersBioColumn < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + remove_column :users, :bio + end + end + + def down + with_lock_retries do + add_column :users, :bio, :string # rubocop: disable Migration/AddColumnsToWideTables + end + end +end diff --git a/db/post_migrate/20200826053152_add_index_on_ci_pipelines_source_for_on_demand_dast.rb b/db/post_migrate/20200826053152_add_index_on_ci_pipelines_source_for_on_demand_dast.rb new file mode 100644 index 00000000000..6249adc9798 --- /dev/null +++ b/db/post_migrate/20200826053152_add_index_on_ci_pipelines_source_for_on_demand_dast.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class AddIndexOnCiPipelinesSourceForOnDemandDast < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + INDEX_NAME = 'index_ci_pipelines_for_ondemand_dast_scans' + + SOURCE_ONDEMAND_DAST_SCAN_PIPELINE = 13 + + disable_ddl_transaction! + + def up + add_concurrent_index( + :ci_pipelines, :id, + where: "source = #{SOURCE_ONDEMAND_DAST_SCAN_PIPELINE}", + name: INDEX_NAME + ) + end + + def down + remove_concurrent_index( + :ci_pipelines, :id, + where: "source = #{SOURCE_ONDEMAND_DAST_SCAN_PIPELINE}", + name: INDEX_NAME + ) + end +end diff --git a/db/post_migrate/20200826121552_remove_ci_job_artifacts_locked.rb b/db/post_migrate/20200826121552_remove_ci_job_artifacts_locked.rb new file mode 100644 index 00000000000..21c7acca7ac --- /dev/null +++ b/db/post_migrate/20200826121552_remove_ci_job_artifacts_locked.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RemoveCiJobArtifactsLocked < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + with_lock_retries do + remove_column :ci_job_artifacts, :locked + end + end + + def down + with_lock_retries do + add_column :ci_job_artifacts, :locked, :boolean + end + end +end diff --git a/db/post_migrate/20200826220745_add_compound_index_on_vulnerabilities_for_background_migration.rb b/db/post_migrate/20200826220745_add_compound_index_on_vulnerabilities_for_background_migration.rb new file mode 100644 index 00000000000..da95f708cf9 --- /dev/null +++ b/db/post_migrate/20200826220745_add_compound_index_on_vulnerabilities_for_background_migration.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddCompoundIndexOnVulnerabilitiesForBackgroundMigration < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_vulnerabilities_on_project_id_and_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :vulnerabilities, [:project_id, :id], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :vulnerabilities, INDEX_NAME + end +end diff --git a/db/post_migrate/20200826220746_schedule_populate_resolved_on_default_branch_column.rb b/db/post_migrate/20200826220746_schedule_populate_resolved_on_default_branch_column.rb new file mode 100644 index 00000000000..6faa4fc8101 --- /dev/null +++ b/db/post_migrate/20200826220746_schedule_populate_resolved_on_default_branch_column.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class SchedulePopulateResolvedOnDefaultBranchColumn < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 100 + DELAY_INTERVAL = 5.minutes.to_i + MIGRATION_CLASS = 'PopulateResolvedOnDefaultBranchColumn' + + disable_ddl_transaction! + + def up + return unless Gitlab.ee? + + EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::Vulnerability.distinct.each_batch(of: BATCH_SIZE, column: :project_id) do |batch, index| + project_ids = batch.pluck(:project_id) + migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, project_ids) + end + end + + def down + # no-op + # This migration schedules background tasks to populate + # `resolved_on_default_branch` column of `vulnerabilities` + # table so there is no rollback operation needed for this. + end +end diff --git a/db/post_migrate/20200831065320_add_not_valid_not_null_constraint_to_mr_metrics.rb b/db/post_migrate/20200831065320_add_not_valid_not_null_constraint_to_mr_metrics.rb new file mode 100644 index 00000000000..35bfabc0358 --- /dev/null +++ b/db/post_migrate/20200831065320_add_not_valid_not_null_constraint_to_mr_metrics.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNotValidNotNullConstraintToMrMetrics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_not_null_constraint :merge_request_metrics, :target_project_id, validate: false + end + + def down + remove_not_null_constraint :merge_request_metrics, :target_project_id + end +end diff --git a/db/post_migrate/20200831065322_add_tmp_index_to_target_project_id.rb b/db/post_migrate/20200831065322_add_tmp_index_to_target_project_id.rb new file mode 100644 index 00000000000..5d6d098ebfe --- /dev/null +++ b/db/post_migrate/20200831065322_add_tmp_index_to_target_project_id.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddTmpIndexToTargetProjectId < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + TMP_INDEX_NAME = 'tmp_index_on_mr_metrics_target_project_id_null' + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :merge_request_metrics, :id, where: 'target_project_id IS NULL', name: TMP_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :merge_request_metrics, name: TMP_INDEX_NAME + end +end diff --git a/db/post_migrate/20200831065705_ensure_target_project_id_is_filled.rb b/db/post_migrate/20200831065705_ensure_target_project_id_is_filled.rb new file mode 100644 index 00000000000..9b267933b04 --- /dev/null +++ b/db/post_migrate/20200831065705_ensure_target_project_id_is_filled.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class EnsureTargetProjectIdIsFilled < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + BACKGROUND_MIGRATION_CLASS = 'CopyMergeRequestTargetProjectToMergeRequestMetrics' + BATCH_SIZE = 1_000 + DOWNTIME = false + + disable_ddl_transaction! + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + end + + class MergeRequestMetrics < ActiveRecord::Base + include EachBatch + + belongs_to :merge_request + + self.table_name = 'merge_request_metrics' + end + + def up + Gitlab::BackgroundMigration.steal(BACKGROUND_MIGRATION_CLASS) + + # Do a manual update in case we lost BG jobs. The expected record count should be 0 or very low. + MergeRequestMetrics.where(target_project_id: nil).each_batch do |scope| + query_for_cte = scope.joins(:merge_request).select( + MergeRequestMetrics.arel_table[:id].as('id'), + MergeRequest.arel_table[:target_project_id].as('target_project_id') + ) + + MergeRequestMetrics.connection.execute <<-SQL + WITH target_project_id_and_metrics_id as ( + #{query_for_cte.to_sql} + ) + UPDATE #{MergeRequestMetrics.connection.quote_table_name(MergeRequestMetrics.table_name)} + SET target_project_id = target_project_id_and_metrics_id.target_project_id + FROM target_project_id_and_metrics_id + WHERE merge_request_metrics.id = target_project_id_and_metrics_id.id + SQL + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200831074356_validate_not_null_constraint_on_mr_metrics.rb b/db/post_migrate/20200831074356_validate_not_null_constraint_on_mr_metrics.rb new file mode 100644 index 00000000000..8fd54186db3 --- /dev/null +++ b/db/post_migrate/20200831074356_validate_not_null_constraint_on_mr_metrics.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class ValidateNotNullConstraintOnMrMetrics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + TMP_INDEX_NAME = 'tmp_index_on_mr_metrics_target_project_id_null' + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :merge_request_metrics, :target_project_id + + remove_concurrent_index_by_name :merge_request_metrics, name: TMP_INDEX_NAME + end + + def down + add_concurrent_index :merge_request_metrics, :id, where: 'target_project_id IS NULL', name: TMP_INDEX_NAME + end +end diff --git a/db/post_migrate/20200831224343_populate_vulnerability_historical_statistics_for_year.rb b/db/post_migrate/20200831224343_populate_vulnerability_historical_statistics_for_year.rb new file mode 100644 index 00000000000..3f4fbfbebde --- /dev/null +++ b/db/post_migrate/20200831224343_populate_vulnerability_historical_statistics_for_year.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class PopulateVulnerabilityHistoricalStatisticsForYear < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DELAY_INTERVAL = 5.minutes.to_i + BATCH_SIZE = 50 + MIGRATION = 'PopulateVulnerabilityHistoricalStatistics' + + disable_ddl_transaction! + + class Vulnerability < ActiveRecord::Base + self.table_name = 'vulnerabilities' + + include ::EachBatch + end + + def up + return unless Gitlab.ee? + + Vulnerability.select('project_id').distinct.each_batch(of: BATCH_SIZE, column: 'project_id') do |project_batch, index| + migrate_in(index * DELAY_INTERVAL, MIGRATION, [project_batch.pluck(:project_id), 365]) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200901212304_drop_code_owner_column_from_approval_merge_request_rule.rb b/db/post_migrate/20200901212304_drop_code_owner_column_from_approval_merge_request_rule.rb new file mode 100644 index 00000000000..7524ae8e15b --- /dev/null +++ b/db/post_migrate/20200901212304_drop_code_owner_column_from_approval_merge_request_rule.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class DropCodeOwnerColumnFromApprovalMergeRequestRule < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + disable_ddl_transaction! + + def up + with_lock_retries do + remove_column :approval_merge_request_rules, :code_owner + end + end + + def down + unless column_exists?(:approval_merge_request_rules, :code_owner) + with_lock_retries do + add_column :approval_merge_request_rules, :code_owner, :boolean, default: false, null: false + end + end + + add_concurrent_index( + :approval_merge_request_rules, + [:merge_request_id, :code_owner, :name], + unique: true, + where: "code_owner = true AND section IS NULL", + name: "approval_rule_name_index_for_code_owners" + ) + + add_concurrent_index( + :approval_merge_request_rules, + [:merge_request_id, :code_owner], + name: "index_approval_merge_request_rules_1" + ) + end +end diff --git a/db/post_migrate/20200903064431_add_created_at_index_to_audit_events.rb b/db/post_migrate/20200903064431_add_created_at_index_to_audit_events.rb new file mode 100644 index 00000000000..94dc2cb7adf --- /dev/null +++ b/db/post_migrate/20200903064431_add_created_at_index_to_audit_events.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AddCreatedAtIndexToAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + INDEX_NAME = 'idx_audit_events_on_entity_id_desc_author_id_created_at' + OLD_INDEX_NAME = 'index_audit_events_on_entity_id_entity_type_id_desc_author_id' + + def up + add_concurrent_index(:audit_events, [:entity_id, :entity_type, :id, :author_id, :created_at], order: { id: :desc }, name: INDEX_NAME) + remove_concurrent_index_by_name(:audit_events, OLD_INDEX_NAME) + end + + def down + add_concurrent_index(:audit_events, [:entity_id, :entity_type, :id, :author_id], order: { id: :desc }, name: OLD_INDEX_NAME) + remove_concurrent_index_by_name(:audit_events, INDEX_NAME) + end +end diff --git a/db/post_migrate/20200904174901_backfill_cleanup_for_partitioned_audit_events.rb b/db/post_migrate/20200904174901_backfill_cleanup_for_partitioned_audit_events.rb new file mode 100644 index 00000000000..1d8240012d1 --- /dev/null +++ b/db/post_migrate/20200904174901_backfill_cleanup_for_partitioned_audit_events.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class BackfillCleanupForPartitionedAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::PartitioningMigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + finalize_backfilling_partitioned_table :audit_events + end + + def down + # no op + end +end diff --git a/db/post_migrate/20200907124300_complete_namespace_settings_migration.rb b/db/post_migrate/20200907124300_complete_namespace_settings_migration.rb new file mode 100644 index 00000000000..5881869ee3c --- /dev/null +++ b/db/post_migrate/20200907124300_complete_namespace_settings_migration.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class CompleteNamespaceSettingsMigration < ActiveRecord::Migration[5.2] + DOWNTIME = false + BATCH_SIZE = 10000 + + class Namespace < ActiveRecord::Base + include EachBatch + + self.table_name = 'namespaces' + end + + def up + Gitlab::BackgroundMigration.steal('BackfillNamespaceSettings') + + ensure_data_migration + end + + def down + # no-op + end + + private + + def ensure_data_migration + Namespace.each_batch(of: BATCH_SIZE) do |query| + missing_count = query.where("NOT EXISTS (SELECT 1 FROM namespace_settings WHERE namespace_settings.namespace_id=namespaces.id)").limit(1).size + if missing_count > 0 + min, max = query.pluck("MIN(id), MAX(id)").flatten + # we expect low record count so inline execution is fine. + Gitlab::BackgroundMigration::BackfillNamespaceSettings.new.perform(min, max) + end + end + end +end diff --git a/db/post_migrate/20200908095446_update_location_fingerprint_column_for_cs.rb b/db/post_migrate/20200908095446_update_location_fingerprint_column_for_cs.rb new file mode 100644 index 00000000000..fc2b7116ed1 --- /dev/null +++ b/db/post_migrate/20200908095446_update_location_fingerprint_column_for_cs.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class UpdateLocationFingerprintColumnForCs < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + BATCH_SIZE = 1_000 + INTERVAL = 2.minutes + + # 883_152 records + def up + return unless Gitlab.ee? + + migration = Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings + migration_name = migration.to_s.demodulize + relation = migration::Finding.container_scanning + queue_background_migration_jobs_by_range_at_intervals(relation, + migration_name, + INTERVAL, + batch_size: BATCH_SIZE) + end + + def down + # no-op + # intentionally blank + end +end diff --git a/db/post_migrate/20200909194014_change_pypi_python_version_type_cleanup.rb b/db/post_migrate/20200909194014_change_pypi_python_version_type_cleanup.rb new file mode 100644 index 00000000000..c94f745e762 --- /dev/null +++ b/db/post_migrate/20200909194014_change_pypi_python_version_type_cleanup.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class ChangePypiPythonVersionTypeCleanup < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + cleanup_concurrent_column_type_change(:packages_pypi_metadata, :required_python) + end + + def down + execute('UPDATE packages_pypi_metadata SET required_python = substring(required_python from 1 for 50)') + change_column_type_concurrently :packages_pypi_metadata, :required_python, 'varchar(50)', batch_column_name: :package_id + end +end diff --git a/db/post_migrate/20200909194524_increase_pypi_version_size.rb b/db/post_migrate/20200909194524_increase_pypi_version_size.rb new file mode 100644 index 00000000000..0273d49bba9 --- /dev/null +++ b/db/post_migrate/20200909194524_increase_pypi_version_size.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class IncreasePypiVersionSize < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_text_limit :packages_pypi_metadata, :required_python, 255 + end + + def down + remove_text_limit :packages_pypi_metadata, :required_python + end +end diff --git a/db/post_migrate/20200910131217_tmp_index_for_fixing_inconsistent_vulnerability_occurrences.rb b/db/post_migrate/20200910131217_tmp_index_for_fixing_inconsistent_vulnerability_occurrences.rb new file mode 100644 index 00000000000..1985b8f1b67 --- /dev/null +++ b/db/post_migrate/20200910131217_tmp_index_for_fixing_inconsistent_vulnerability_occurrences.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class TmpIndexForFixingInconsistentVulnerabilityOccurrences < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'tmp_index_for_fixing_inconsistent_vulnerability_occurrences' + disable_ddl_transaction! + + def up + # report_type: 2 container scanning + add_concurrent_index(:vulnerability_occurrences, :id, + where: "LENGTH(location_fingerprint) = 40 AND report_type = 2", + name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(:vulnerability_occurrences, INDEX_NAME) + end +end diff --git a/db/post_migrate/20200910131218_remove_duplicated_cs_findings.rb b/db/post_migrate/20200910131218_remove_duplicated_cs_findings.rb new file mode 100644 index 00000000000..c5bfddfd265 --- /dev/null +++ b/db/post_migrate/20200910131218_remove_duplicated_cs_findings.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class RemoveDuplicatedCsFindings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + BATCH_SIZE = 1_000 + INTERVAL = 2.minutes + + # 23_893 records will be updated + # 23_893 records will be deleted + def up + return unless Gitlab.com? + + migration = Gitlab::BackgroundMigration::RemoveDuplicateCsFindings + migration_name = migration.to_s.demodulize + relation = migration::Finding.container_scanning.where("LENGTH(location_fingerprint) = 40") + queue_background_migration_jobs_by_range_at_intervals(relation, + migration_name, + INTERVAL, + batch_size: BATCH_SIZE) + end + + def down + # no-op + # intentionally blank + end +end diff --git a/db/post_migrate/20200910170908_ensure_filled_external_diff_store_on_merge_request_diffs.rb b/db/post_migrate/20200910170908_ensure_filled_external_diff_store_on_merge_request_diffs.rb new file mode 100644 index 00000000000..5070bd1c2f5 --- /dev/null +++ b/db/post_migrate/20200910170908_ensure_filled_external_diff_store_on_merge_request_diffs.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class EnsureFilledExternalDiffStoreOnMergeRequestDiffs < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + BACKGROUND_MIGRATION_CLASS = 'SetNullExternalDiffStoreToLocalValue' + BATCH_SIZE = 5_000 + LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL + DOWNTIME = false + + disable_ddl_transaction! + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + def up + Gitlab::BackgroundMigration.steal(BACKGROUND_MIGRATION_CLASS) + + # Do a manual update in case we lost BG jobs. The expected record count should be 0 or very low. + MergeRequestDiff.where(external_diff_store: nil).each_batch(of: BATCH_SIZE) do |batch, index| + batch.update_all(external_diff_store: LOCAL_STORE) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200910175553_validate_not_null_external_diff_store_on_merge_request_diffs.rb b/db/post_migrate/20200910175553_validate_not_null_external_diff_store_on_merge_request_diffs.rb new file mode 100644 index 00000000000..4dc8ac96a4c --- /dev/null +++ b/db/post_migrate/20200910175553_validate_not_null_external_diff_store_on_merge_request_diffs.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class ValidateNotNullExternalDiffStoreOnMergeRequestDiffs < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + # Remove index which was only added to fill external_diff_store + INDEX_NAME = 'index_merge_request_diffs_external_diff_store_is_null' + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :merge_request_diffs, :external_diff_store + + remove_concurrent_index_by_name :merge_request_diffs, INDEX_NAME + end + + def down + add_concurrent_index :merge_request_diffs, :id, where: 'external_diff_store IS NULL', name: INDEX_NAME + end +end diff --git a/db/post_migrate/20200915044225_schedule_migration_to_hashed_storage.rb b/db/post_migrate/20200915044225_schedule_migration_to_hashed_storage.rb new file mode 100644 index 00000000000..553d060dc58 --- /dev/null +++ b/db/post_migrate/20200915044225_schedule_migration_to_hashed_storage.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ScheduleMigrationToHashedStorage < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + MIGRATION = 'MigrateToHashedStorage' + + disable_ddl_transaction! + + def up + migrate_async(MIGRATION) + end + + def down + # NO-OP + end +end diff --git a/db/post_migrate/20200916081749_remove_cycle_analytics_total_stage_data.rb b/db/post_migrate/20200916081749_remove_cycle_analytics_total_stage_data.rb new file mode 100644 index 00000000000..94c218c0c57 --- /dev/null +++ b/db/post_migrate/20200916081749_remove_cycle_analytics_total_stage_data.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class RemoveCycleAnalyticsTotalStageData < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + execute("DELETE FROM analytics_cycle_analytics_group_stages WHERE name='production'") + execute("DELETE FROM analytics_cycle_analytics_project_stages WHERE name='production'") + end + + def down + # Migration is irreversible + end +end |