diff options
Diffstat (limited to 'db/post_migrate')
32 files changed, 964 insertions, 1 deletions
diff --git a/db/post_migrate/20200305082754_remove_duplicate_labels_from_project.rb b/db/post_migrate/20200305082754_remove_duplicate_labels_from_project.rb new file mode 100644 index 00000000000..33f8118534d --- /dev/null +++ b/db/post_migrate/20200305082754_remove_duplicate_labels_from_project.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +class RemoveDuplicateLabelsFromProject < ActiveRecord::Migration[6.0] + DOWNTIME = false + + CREATE = 1 + RENAME = 2 + + disable_ddl_transaction! + + class BackupLabel < Label + self.table_name = 'backup_labels' + end + + class Label < ApplicationRecord + self.table_name = 'labels' + end + + class Project < ApplicationRecord + include EachBatch + + self.table_name = 'projects' + end + + BATCH_SIZE = 100_000 + + def up + # Split to smaller chunks + # Loop rather than background job, every 100,000 + # there are 45,000,000 projects in total + Project.each_batch(of: BATCH_SIZE) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + transaction do + remove_full_duplicates(*range) + end + + transaction do + rename_partial_duplicates(*range) + end + end + end + + def down + Project.each_batch(of: BATCH_SIZE) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + restore_renamed_labels(*range) + restore_deleted_labels(*range) + end + end + + def remove_full_duplicates(start_id, stop_id) + # Fields that are considered duplicate: + # project_id title template description type color + + duplicate_labels = ApplicationRecord.connection.execute(<<-SQL.squish) +WITH data AS ( + SELECT labels.*, + row_number() OVER (PARTITION BY labels.project_id, labels.title, labels.template, labels.description, labels.type, labels.color ORDER BY labels.id) AS row_number, + #{CREATE} AS restore_action + FROM labels + WHERE labels.project_id BETWEEN #{start_id} AND #{stop_id} + AND NOT EXISTS (SELECT * FROM board_labels WHERE board_labels.label_id = labels.id) + AND NOT EXISTS (SELECT * FROM label_links WHERE label_links.label_id = labels.id) + AND NOT EXISTS (SELECT * FROM label_priorities WHERE label_priorities.label_id = labels.id) + AND NOT EXISTS (SELECT * FROM lists WHERE lists.label_id = labels.id) + AND NOT EXISTS (SELECT * FROM resource_label_events WHERE resource_label_events.label_id = labels.id) +) SELECT * FROM data WHERE row_number > 1; + SQL + + if duplicate_labels.any? + # create backup records + BackupLabel.insert_all!(duplicate_labels.map { |label| label.except("row_number") }) + + Label.where(id: duplicate_labels.pluck("id")).delete_all + end + end + + def rename_partial_duplicates(start_id, stop_id) + # We need to ensure that the new title (with `_duplicate#{ID}`) doesn't exceed the limit. + # Truncate the original title (if needed) to 245 characters minus the length of the ID + # then add `_duplicate#{ID}` + + soft_duplicates = ApplicationRecord.connection.execute(<<-SQL.squish) +WITH data AS ( + SELECT + *, + substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text as new_title, + #{RENAME} AS restore_action, + row_number() OVER (PARTITION BY project_id, title ORDER BY id) AS row_number + FROM labels + WHERE project_id BETWEEN #{start_id} AND #{stop_id} +) SELECT * FROM data WHERE row_number > 1; + SQL + + if soft_duplicates.any? + # create backup records + BackupLabel.insert_all!(soft_duplicates.map { |label| label.except("row_number") }) + + ApplicationRecord.connection.execute(<<-SQL.squish) +UPDATE labels SET title = substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text +WHERE labels.id IN (#{soft_duplicates.map { |dup| dup["id"] }.join(", ")}); + SQL + end + end + + def restore_renamed_labels(start_id, stop_id) + # the backup label IDs are not incremental, they are copied directly from the Labels table + ApplicationRecord.connection.execute(<<-SQL.squish) +WITH backups AS ( + SELECT id, title + FROM backup_labels + WHERE project_id BETWEEN #{start_id} AND #{stop_id} AND + restore_action = #{RENAME} +) UPDATE labels SET title = backups.title +FROM backups +WHERE labels.id = backups.id; + SQL + end + + def restore_deleted_labels(start_id, stop_id) + ActiveRecord::Base.connection.execute(<<-SQL.squish) +INSERT INTO labels +SELECT id, title, color, project_id, created_at, updated_at, template, description, description_html, type, group_id, cached_markdown_version FROM backup_labels + WHERE backup_labels.project_id BETWEEN #{start_id} AND #{stop_id} + AND backup_labels.restore_action = #{CREATE} + SQL + end +end diff --git a/db/post_migrate/20200305082858_add_uniqueness_index_to_label_title_and_project.rb b/db/post_migrate/20200305082858_add_uniqueness_index_to_label_title_and_project.rb new file mode 100644 index 00000000000..ce235ba4aea --- /dev/null +++ b/db/post_migrate/20200305082858_add_uniqueness_index_to_label_title_and_project.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddUniquenessIndexToLabelTitleAndProject < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + PROJECT_AND_TITLE = [:project_id, :title] + + def up + add_concurrent_index :labels, PROJECT_AND_TITLE, where: "labels.group_id IS NULL", unique: true, name: "index_labels_on_project_id_and_title_unique" + remove_concurrent_index :labels, PROJECT_AND_TITLE, name: "index_labels_on_project_id_and_title" + end + + def down + add_concurrent_index :labels, PROJECT_AND_TITLE, where: "labels.group_id IS NULL", unique: false, name: "index_labels_on_project_id_and_title" + remove_concurrent_index :labels, PROJECT_AND_TITLE, name: "index_labels_on_project_id_and_title_unique" + end +end diff --git a/db/post_migrate/20200519201128_migrate_vulnerability_dismissal_feedback.rb b/db/post_migrate/20200519201128_migrate_vulnerability_dismissal_feedback.rb new file mode 100644 index 00000000000..fee2f59abb5 --- /dev/null +++ b/db/post_migrate/20200519201128_migrate_vulnerability_dismissal_feedback.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class MigrateVulnerabilityDismissalFeedback < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + disable_ddl_transaction! + + MIGRATION = 'UpdateVulnerabilitiesFromDismissalFeedback' + BATCH_SIZE = 500 + DELAY_INTERVAL = 2.minutes.to_i + + class Vulnerability < ActiveRecord::Base + self.table_name = 'vulnerabilities' + self.inheritance_column = :_type_disabled + + include ::EachBatch + end + + def up + return unless Gitlab.ee? + + Vulnerability.select('project_id').group(:project_id).each_batch(of: BATCH_SIZE, column: "project_id") do |project_batch, index| + batch_delay = (index - 1) * BATCH_SIZE * DELAY_INTERVAL + + project_batch.each_with_index do |project, project_batch_index| + project_delay = project_batch_index * DELAY_INTERVAL + migrate_in(batch_delay + project_delay, MIGRATION, project[:project_id]) + end + end + end + + def down + # nothing to do + end +end diff --git a/db/post_migrate/20200526115436_dedup_mr_metrics.rb b/db/post_migrate/20200526115436_dedup_mr_metrics.rb new file mode 100644 index 00000000000..d2660504939 --- /dev/null +++ b/db/post_migrate/20200526115436_dedup_mr_metrics.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class DedupMrMetrics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + TMP_INDEX_NAME = 'tmp_unique_merge_request_metrics_by_merge_request_id' + INDEX_NAME = 'unique_merge_request_metrics_by_merge_request_id' + + disable_ddl_transaction! + + class MergeRequestMetrics < ActiveRecord::Base + self.table_name = 'merge_request_metrics' + + include EachBatch + end + + def up + last_metrics_record_id = MergeRequestMetrics.maximum(:id) || 0 + + # This index will disallow further duplicates while we're deduplicating the data. + add_concurrent_index(:merge_request_metrics, :merge_request_id, where: "id > #{Integer(last_metrics_record_id)}", unique: true, name: TMP_INDEX_NAME) + + MergeRequestMetrics.each_batch do |relation| + duplicated_merge_request_ids = MergeRequestMetrics + .where(merge_request_id: relation.select(:merge_request_id)) + .select(:merge_request_id) + .group(:merge_request_id) + .having('COUNT(merge_request_metrics.merge_request_id) > 1') + .pluck(:merge_request_id) + + duplicated_merge_request_ids.each do |merge_request_id| + deduplicate_item(merge_request_id) + end + end + + add_concurrent_index(:merge_request_metrics, :merge_request_id, unique: true, name: INDEX_NAME) + remove_concurrent_index_by_name(:merge_request_metrics, TMP_INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(:merge_request_metrics, TMP_INDEX_NAME) + remove_concurrent_index_by_name(:merge_request_metrics, INDEX_NAME) + end + + private + + def deduplicate_item(merge_request_id) + merge_request_metrics_records = MergeRequestMetrics.where(merge_request_id: merge_request_id).order(updated_at: :asc).to_a + + attributes = {} + merge_request_metrics_records.each do |merge_request_metrics_record| + params = merge_request_metrics_record.attributes.except('id') + attributes.merge!(params.compact) + end + + ActiveRecord::Base.transaction do + record_to_keep = merge_request_metrics_records.pop + records_to_delete = merge_request_metrics_records + + MergeRequestMetrics.where(id: records_to_delete.map(&:id)).delete_all + record_to_keep.update!(attributes) + end + end +end diff --git a/db/post_migrate/20200608195222_set_lock_version_not_null_constraint.rb b/db/post_migrate/20200608195222_set_lock_version_not_null_constraint.rb new file mode 100644 index 00000000000..ec72053b307 --- /dev/null +++ b/db/post_migrate/20200608195222_set_lock_version_not_null_constraint.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class SetLockVersionNotNullConstraint < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + TABLES = %i(epics merge_requests issues ci_stages ci_builds ci_pipelines).freeze + + def up + TABLES.each do |table| + add_not_null_constraint table, :lock_version, validate: false + end + end + + def down + TABLES.each do |table| + remove_not_null_constraint table, :lock_version + end + end +end diff --git a/db/post_migrate/20200608203426_set_proper_lock_version_indices.rb b/db/post_migrate/20200608203426_set_proper_lock_version_indices.rb new file mode 100644 index 00000000000..924ca73e6cc --- /dev/null +++ b/db/post_migrate/20200608203426_set_proper_lock_version_indices.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class SetProperLockVersionIndices < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + remove_concurrent_index :epics, :lock_version, where: "lock_version IS NULL" + remove_concurrent_index :merge_requests, :lock_version, where: "lock_version IS NULL" + remove_concurrent_index :issues, :lock_version, where: "lock_version IS NULL" + + add_concurrent_index :epics, :id, where: "lock_version IS NULL", name: 'index_epics_on_id' + add_concurrent_index :merge_requests, :id, where: "lock_version IS NULL", name: 'index_merge_requests_on_id' + add_concurrent_index :issues, :id, where: "lock_version IS NULL", name: 'index_issues_on_id' + end + + def down + add_concurrent_index :epics, :lock_version, where: "lock_version IS NULL" + add_concurrent_index :merge_requests, :lock_version, where: "lock_version IS NULL" + add_concurrent_index :issues, :lock_version, where: "lock_version IS NULL" + + remove_concurrent_index_by_name :epics, name: 'index_epics_on_id' + remove_concurrent_index_by_name :merge_requests, name: 'index_merge_requests_on_id' + remove_concurrent_index_by_name :issues, name: 'index_issues_on_id' + end +end diff --git a/db/post_migrate/20200608205813_set_lock_version_to_not_null.rb b/db/post_migrate/20200608205813_set_lock_version_to_not_null.rb new file mode 100644 index 00000000000..69f43a8decf --- /dev/null +++ b/db/post_migrate/20200608205813_set_lock_version_to_not_null.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class SetLockVersionToNotNull < ActiveRecord::Migration[6.0] + DOWNTIME = false + + TABLES = %w(epics merge_requests issues ci_stages ci_builds ci_pipelines).freeze + BATCH_SIZE = 10_000 + + disable_ddl_transaction! + + def declare_class(table) + Class.new(ActiveRecord::Base) do + include EachBatch + + self.table_name = table + self.inheritance_column = :_type_disabled # Disable STI + end + end + + def up + TABLES.each do |table| + declare_class(table).where(lock_version: nil).each_batch(of: BATCH_SIZE) do |batch| + batch.update_all(lock_version: 0) + end + end + end + + def down + # Nothing to do... + end +end diff --git a/db/post_migrate/20200608212030_lock_version_cleanup_for_epics.rb b/db/post_migrate/20200608212030_lock_version_cleanup_for_epics.rb new file mode 100644 index 00000000000..aafa6a83200 --- /dev/null +++ b/db/post_migrate/20200608212030_lock_version_cleanup_for_epics.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class LockVersionCleanupForEpics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :epics, :lock_version + remove_concurrent_index_by_name :epics, name: 'index_epics_on_id' + end + + def down + add_concurrent_index :epics, :id, where: "lock_version IS NULL", name: 'index_epics_on_id' + end +end diff --git a/db/post_migrate/20200608212435_lock_version_cleanup_for_merge_requests.rb b/db/post_migrate/20200608212435_lock_version_cleanup_for_merge_requests.rb new file mode 100644 index 00000000000..cb8ab86b6a3 --- /dev/null +++ b/db/post_migrate/20200608212435_lock_version_cleanup_for_merge_requests.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class LockVersionCleanupForMergeRequests < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :merge_requests, :lock_version + remove_concurrent_index_by_name :merge_requests, name: 'index_merge_requests_on_id' + end + + def down + add_concurrent_index :merge_requests, :id, where: "lock_version IS NULL", name: 'index_merge_requests_on_id' + end +end diff --git a/db/post_migrate/20200608212549_lock_version_cleanup_for_issues.rb b/db/post_migrate/20200608212549_lock_version_cleanup_for_issues.rb new file mode 100644 index 00000000000..ad3fea8b131 --- /dev/null +++ b/db/post_migrate/20200608212549_lock_version_cleanup_for_issues.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class LockVersionCleanupForIssues < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :issues, :lock_version + remove_concurrent_index_by_name :issues, name: 'index_issues_on_id' + end + + def down + add_concurrent_index :issues, :id, where: "lock_version IS NULL", name: 'index_issues_on_id' + end +end diff --git a/db/post_migrate/20200608212652_lock_version_cleanup_for_ci_stages.rb b/db/post_migrate/20200608212652_lock_version_cleanup_for_ci_stages.rb new file mode 100644 index 00000000000..12e2897123e --- /dev/null +++ b/db/post_migrate/20200608212652_lock_version_cleanup_for_ci_stages.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class LockVersionCleanupForCiStages < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :ci_stages, :lock_version + remove_concurrent_index :ci_stages, :id, where: "lock_version IS NULL", name: "tmp_index_ci_stages_lock_version" + end + + def down + add_concurrent_index :ci_stages, :id, where: "lock_version IS NULL", name: "tmp_index_ci_stages_lock_version" + end +end diff --git a/db/post_migrate/20200608212807_lock_version_cleanup_for_ci_builds.rb b/db/post_migrate/20200608212807_lock_version_cleanup_for_ci_builds.rb new file mode 100644 index 00000000000..0512869971b --- /dev/null +++ b/db/post_migrate/20200608212807_lock_version_cleanup_for_ci_builds.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class LockVersionCleanupForCiBuilds < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :ci_builds, :lock_version + remove_concurrent_index :ci_builds, :id, where: "lock_version IS NULL", name: "tmp_index_ci_builds_lock_version" + end + + def down + add_concurrent_index :ci_builds, :id, where: "lock_version IS NULL", name: "tmp_index_ci_builds_lock_version" + end +end diff --git a/db/post_migrate/20200608212824_lock_version_cleanup_for_ci_pipelines.rb b/db/post_migrate/20200608212824_lock_version_cleanup_for_ci_pipelines.rb new file mode 100644 index 00000000000..228dd72da8d --- /dev/null +++ b/db/post_migrate/20200608212824_lock_version_cleanup_for_ci_pipelines.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class LockVersionCleanupForCiPipelines < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :ci_pipelines, :lock_version + remove_concurrent_index :ci_pipelines, :id, where: "lock_version IS NULL", name: "tmp_index_ci_pipelines_lock_version" + end + + def down + add_concurrent_index :ci_pipelines, :id, where: "lock_version IS NULL", name: "tmp_index_ci_pipelines_lock_version" + end +end diff --git a/db/post_migrate/20200609002841_add_partial_index_on_locked_state_id_to_merge_requests.rb b/db/post_migrate/20200609002841_add_partial_index_on_locked_state_id_to_merge_requests.rb index 076c8fd8715..7602ad00796 100644 --- a/db/post_migrate/20200609002841_add_partial_index_on_locked_state_id_to_merge_requests.rb +++ b/db/post_migrate/20200609002841_add_partial_index_on_locked_state_id_to_merge_requests.rb @@ -14,6 +14,6 @@ class AddPartialIndexOnLockedStateIdToMergeRequests < ActiveRecord::Migration[6. end def down - remove_concurrent_index_by_name :merge_requests, name: INDEX_NAME + remove_concurrent_index_by_name :merge_requests, INDEX_NAME end end diff --git a/db/post_migrate/20200615111857_unconfirm_wrongfully_verified_emails.rb b/db/post_migrate/20200615111857_unconfirm_wrongfully_verified_emails.rb new file mode 100644 index 00000000000..41280872a94 --- /dev/null +++ b/db/post_migrate/20200615111857_unconfirm_wrongfully_verified_emails.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class UnconfirmWrongfullyVerifiedEmails < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + INTERVAL = 5.minutes.to_i + BATCH_SIZE = 500 + MIGRATION = 'WrongfullyConfirmedEmailUnconfirmer' + EMAIL_INDEX_NAME = 'tmp_index_for_email_unconfirmation_migration' + + class Email < ActiveRecord::Base + include EachBatch + end + + def up + add_concurrent_index :emails, :id, where: 'confirmed_at IS NOT NULL', name: EMAIL_INDEX_NAME + + queue_background_migration_jobs_by_range_at_intervals(Email, + MIGRATION, + INTERVAL, + batch_size: BATCH_SIZE) + end + + def down + remove_concurrent_index_by_name(:emails, EMAIL_INDEX_NAME) + end +end diff --git a/db/post_migrate/20200617001637_validate_file_store_not_null_constraint_on_lfs_objects.rb b/db/post_migrate/20200617001637_validate_file_store_not_null_constraint_on_lfs_objects.rb new file mode 100644 index 00000000000..27a30b1d696 --- /dev/null +++ b/db/post_migrate/20200617001637_validate_file_store_not_null_constraint_on_lfs_objects.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ValidateFileStoreNotNullConstraintOnLfsObjects < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_check_constraint(:lfs_objects, :check_eecfc5717d) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200617001848_validate_store_not_null_constraint_uploads.rb b/db/post_migrate/20200617001848_validate_store_not_null_constraint_uploads.rb new file mode 100644 index 00000000000..83cb6cb3e85 --- /dev/null +++ b/db/post_migrate/20200617001848_validate_store_not_null_constraint_uploads.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ValidateStoreNotNullConstraintUploads < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_check_constraint(:uploads, :check_5e9547379c) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200617002030_validate_file_store_not_null_constraint_on_ci_job_artifacts.rb b/db/post_migrate/20200617002030_validate_file_store_not_null_constraint_on_ci_job_artifacts.rb new file mode 100644 index 00000000000..8e766a508b7 --- /dev/null +++ b/db/post_migrate/20200617002030_validate_file_store_not_null_constraint_on_ci_job_artifacts.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ValidateFileStoreNotNullConstraintOnCiJobArtifacts < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_check_constraint(:ci_job_artifacts, :check_27f0f6dbab) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200618152212_update_secure_smau_index.rb b/db/post_migrate/20200618152212_update_secure_smau_index.rb new file mode 100644 index 00000000000..ba989c279be --- /dev/null +++ b/db/post_migrate/20200618152212_update_secure_smau_index.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class UpdateSecureSmauIndex < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_secure_ci_builds_on_user_id_created_at' + + disable_ddl_transaction! + + def up + add_concurrent_index( + :ci_builds, + [:user_id, :created_at], + where: "(((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('secret_detection'::character varying)::text])))", + name: INDEX_NAME + ) + end + + def down + remove_concurrent_index_by_name :ci_builds, INDEX_NAME + end +end diff --git a/db/post_migrate/20200623142159_remove_gitlab_issue_tracker_service_records.rb b/db/post_migrate/20200623142159_remove_gitlab_issue_tracker_service_records.rb new file mode 100644 index 00000000000..743499e7b76 --- /dev/null +++ b/db/post_migrate/20200623142159_remove_gitlab_issue_tracker_service_records.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class RemoveGitlabIssueTrackerServiceRecords < ActiveRecord::Migration[6.0] + DOWNTIME = false + BATCH_SIZE = 5000 + + disable_ddl_transaction! + + class Service < ActiveRecord::Base + include EachBatch + + self.table_name = 'services' + + def self.gitlab_issue_tracker_service + where(type: 'GitlabIssueTrackerService') + end + end + + def up + Service.each_batch(of: BATCH_SIZE) do |services| + services.gitlab_issue_tracker_service.delete_all + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200626060151_add_disable_overriding_approvers_per_merge_request_indices.rb b/db/post_migrate/20200626060151_add_disable_overriding_approvers_per_merge_request_indices.rb new file mode 100644 index 00000000000..6f2db4035e2 --- /dev/null +++ b/db/post_migrate/20200626060151_add_disable_overriding_approvers_per_merge_request_indices.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class AddDisableOverridingApproversPerMergeRequestIndices < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DISABLE_OVERRIDING_APPROVERS_TRUE_INDEX_NAME = "idx_projects_id_created_at_disable_overriding_approvers_true" + DISABLE_OVERRIDING_APPROVERS_FALSE_INDEX_NAME = "idx_projects_id_created_at_disable_overriding_approvers_false" + + disable_ddl_transaction! + + def up + add_concurrent_index :projects, [:id, :created_at], + where: "disable_overriding_approvers_per_merge_request = TRUE", + name: DISABLE_OVERRIDING_APPROVERS_TRUE_INDEX_NAME + + add_concurrent_index :projects, [:id, :created_at], + where: "(disable_overriding_approvers_per_merge_request = FALSE) OR (disable_overriding_approvers_per_merge_request IS NULL)", + name: DISABLE_OVERRIDING_APPROVERS_FALSE_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :projects, DISABLE_OVERRIDING_APPROVERS_TRUE_INDEX_NAME + remove_concurrent_index_by_name :projects, DISABLE_OVERRIDING_APPROVERS_FALSE_INDEX_NAME + end +end diff --git a/db/post_migrate/20200701070435_add_default_value_stream_to_groups_with_group_stages.rb b/db/post_migrate/20200701070435_add_default_value_stream_to_groups_with_group_stages.rb new file mode 100644 index 00000000000..971eb3c489f --- /dev/null +++ b/db/post_migrate/20200701070435_add_default_value_stream_to_groups_with_group_stages.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +class AddDefaultValueStreamToGroupsWithGroupStages < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Group < ActiveRecord::Base + def self.find_sti_class(typename) + if typename == 'Group' + Group + else + super + end + end + self.table_name = 'namespaces' + has_many :group_value_streams + has_many :group_stages + end + + class GroupValueStream < ActiveRecord::Base + self.table_name = 'analytics_cycle_analytics_group_value_streams' + has_many :group_stages + belongs_to :group + end + + class GroupStage < ActiveRecord::Base + self.table_name = 'analytics_cycle_analytics_group_stages' + belongs_to :group_value_stream + end + + def up + Group.where(type: 'Group').joins(:group_stages).distinct.find_each do |group| + Group.transaction do + group_value_stream = group.group_value_streams.first_or_create!(name: 'default') + group.group_stages.update_all(group_value_stream_id: group_value_stream.id) + end + end + + change_column_null :analytics_cycle_analytics_group_stages, :group_value_stream_id, false + end + + def down + change_column_null :analytics_cycle_analytics_group_stages, :group_value_stream_id, true + + GroupValueStream.where(name: 'default').includes(:group_stages).find_each do |value_stream| + GroupValueStream.transaction do + value_stream.group_stages.update_all(group_value_stream_id: nil) + value_stream.destroy! + end + end + end +end diff --git a/db/post_migrate/20200701091253_validate_foreign_key_on_cycle_analytics_group_stages.rb b/db/post_migrate/20200701091253_validate_foreign_key_on_cycle_analytics_group_stages.rb new file mode 100644 index 00000000000..0a8926ed6de --- /dev/null +++ b/db/post_migrate/20200701091253_validate_foreign_key_on_cycle_analytics_group_stages.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class ValidateForeignKeyOnCycleAnalyticsGroupStages < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + # same as in db/migrate/20200701064756_add_not_valid_foreign_key_to_cycle_analytics_group_stages.rb + CONSTRAINT_NAME = 'fk_analytics_cycle_analytics_group_stages_group_value_stream_id' + + def up + validate_foreign_key :analytics_cycle_analytics_group_stages, :group_value_stream_id, name: CONSTRAINT_NAME + end + + def down + remove_foreign_key_if_exists :analytics_cycle_analytics_group_stages, column: :group_value_stream_id, name: CONSTRAINT_NAME + add_foreign_key :analytics_cycle_analytics_group_stages, :analytics_cycle_analytics_group_value_streams, + column: :group_value_stream_id, name: CONSTRAINT_NAME, on_delete: :cascade, validate: false + end +end diff --git a/db/post_migrate/20200703064117_generate_missing_routes_for_bots.rb b/db/post_migrate/20200703064117_generate_missing_routes_for_bots.rb new file mode 100644 index 00000000000..85d62cbb6dd --- /dev/null +++ b/db/post_migrate/20200703064117_generate_missing_routes_for_bots.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +class GenerateMissingRoutesForBots < ActiveRecord::Migration[6.0] + DOWNTIME = false + + disable_ddl_transaction! + + class User < ActiveRecord::Base + self.table_name = 'users' + + USER_TYPES = { + human: nil, + support_bot: 1, + alert_bot: 2, + visual_review_bot: 3, + service_user: 4, + ghost: 5, + project_bot: 6, + migration_bot: 7 + }.with_indifferent_access.freeze + + BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot].freeze + + scope :bots, -> { where(user_type: USER_TYPES.values_at(*BOT_USER_TYPES)) } + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + + validates :path, + uniqueness: { case_sensitive: false } + end + + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + + belongs_to :owner, class_name: 'GenerateMissingRoutesForBots::User' + + scope :for_user, -> { where('type IS NULL') } + scope :for_bots, -> { for_user.joins(:owner).merge(GenerateMissingRoutesForBots::User.bots) } + + scope :without_routes, -> do + where( + 'NOT EXISTS ( + SELECT 1 + FROM routes + WHERE source_type = ? + AND source_id = namespaces.id + )', + self.source_type_for_route + ) + end + + def self.source_type_for_route + 'Namespace' + end + + def attributes_for_insert + { + source_type: self.class.source_type_for_route, + source_id: id, + name: name, + path: path + } + end + end + + def up + # Reset the column information of all the models that update the database + # to ensure the Active Record's knowledge of the table structure is current + Route.reset_column_information + + logger = Gitlab::BackgroundMigration::Logger.build + attributes_to_be_logged = %w(id path name) + + GenerateMissingRoutesForBots::Namespace.for_bots.without_routes.each do |namespace| + route = GenerateMissingRoutesForBots::Route.create(namespace.attributes_for_insert) + namespace_details = namespace.as_json.slice(*attributes_to_be_logged) + + if route.persisted? + logger.info namespace_details.merge(message: 'a new route was created for the namespace') + else + errors = route.errors.full_messages.join(',') + logger.info namespace_details.merge(message: 'route creation failed for the namespace', errors: errors) + end + end + end + + def down + # no op + end +end diff --git a/db/post_migrate/20200703125016_backfill_namespace_settings.rb b/db/post_migrate/20200703125016_backfill_namespace_settings.rb new file mode 100644 index 00000000000..a7335e2d2b8 --- /dev/null +++ b/db/post_migrate/20200703125016_backfill_namespace_settings.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class BackfillNamespaceSettings < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + MIGRATION = 'BackfillNamespaceSettings' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 10_000 + + disable_ddl_transaction! + + class Namespace < ActiveRecord::Base + include EachBatch + + self.table_name = 'namespaces' + end + + def up + say "Scheduling `#{MIGRATION}` jobs" + + queue_background_migration_jobs_by_range_at_intervals(Namespace, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + + def down + # NOOP + end +end diff --git a/db/post_migrate/20200704143633_add_index_on_user_id_and_created_at_where_source_to_ci_pipelines.rb b/db/post_migrate/20200704143633_add_index_on_user_id_and_created_at_where_source_to_ci_pipelines.rb new file mode 100644 index 00000000000..d84b2b4cad3 --- /dev/null +++ b/db/post_migrate/20200704143633_add_index_on_user_id_and_created_at_where_source_to_ci_pipelines.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexOnUserIdAndCreatedAtWhereSourceToCiPipelines < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_pipelines, [:user_id, :created_at, :config_source] + end + + def down + remove_concurrent_index :ci_pipelines, [:user_id, :created_at, :config_source] + end +end diff --git a/db/post_migrate/20200704161600_add_index_on_id_and_status_and_created_at_to_deployments.rb b/db/post_migrate/20200704161600_add_index_on_id_and_status_and_created_at_to_deployments.rb new file mode 100644 index 00000000000..3aab2fd2949 --- /dev/null +++ b/db/post_migrate/20200704161600_add_index_on_id_and_status_and_created_at_to_deployments.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddIndexOnIdAndStatusAndCreatedAtToDeployments < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :deployments, [:id, :status, :created_at] + remove_concurrent_index :deployments, [:id, :status] + end + + def down + add_concurrent_index :deployments, [:id, :status] + remove_concurrent_index :deployments, [:id, :status, :created_at] + end +end diff --git a/db/post_migrate/20200706154619_drop_ci_daily_report_results_table.rb b/db/post_migrate/20200706154619_drop_ci_daily_report_results_table.rb new file mode 100644 index 00000000000..c6ce52012d6 --- /dev/null +++ b/db/post_migrate/20200706154619_drop_ci_daily_report_results_table.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiDailyReportResultsTable < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + drop_table :ci_daily_report_results + end + + def down + create_table :ci_daily_report_results do |t| + t.date :date, null: false + t.bigint :project_id, null: false + t.bigint :last_pipeline_id, null: false + t.float :value, null: false + t.integer :param_type, limit: 8, null: false + t.string :ref_path, null: false + t.string :title, null: false + + t.index :last_pipeline_id + t.index [:project_id, :ref_path, :param_type, :date, :title], name: 'index_daily_report_results_unique_columns', unique: true + end + end +end diff --git a/db/post_migrate/20200709101408_schedule_populate_project_snippet_statistics.rb b/db/post_migrate/20200709101408_schedule_populate_project_snippet_statistics.rb new file mode 100644 index 00000000000..28527e67f4a --- /dev/null +++ b/db/post_migrate/20200709101408_schedule_populate_project_snippet_statistics.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class SchedulePopulateProjectSnippetStatistics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 500 + MIGRATION = 'PopulateProjectSnippetStatistics' + + disable_ddl_transaction! + + def up + snippets = exec_query <<~SQL + SELECT snippets.id + FROM snippets + INNER JOIN projects ON projects.id = snippets.project_id + WHERE snippets.type = 'ProjectSnippet' + ORDER BY projects.namespace_id ASC, snippets.project_id ASC, snippets.id ASC + SQL + + snippets.rows.flatten.in_groups_of(BATCH_SIZE, false).each_with_index do |snippet_ids, index| + migrate_in(index * DELAY_INTERVAL, MIGRATION, [snippet_ids]) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200710102418_delete_user_callout_alerts_moved.rb b/db/post_migrate/20200710102418_delete_user_callout_alerts_moved.rb new file mode 100644 index 00000000000..e14cd7ac3ee --- /dev/null +++ b/db/post_migrate/20200710102418_delete_user_callout_alerts_moved.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class DeleteUserCalloutAlertsMoved < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + class UserCallout < ActiveRecord::Base + include EachBatch + + self.table_name = 'user_callouts' + end + + BATCH_SIZE = 1_000 + + # Inlined from UserCalloutEnums.feature_names + FEATURE_NAME_ALERTS_MOVED = 20 + + def up + UserCallout.each_batch(of: BATCH_SIZE, column: :user_id) do |callout| + callout.where(feature_name: FEATURE_NAME_ALERTS_MOVED).delete_all + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200710102846_drop_index_ruby_objects_in_details_on_audit_events.rb b/db/post_migrate/20200710102846_drop_index_ruby_objects_in_details_on_audit_events.rb new file mode 100644 index 00000000000..6869938466a --- /dev/null +++ b/db/post_migrate/20200710102846_drop_index_ruby_objects_in_details_on_audit_events.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class DropIndexRubyObjectsInDetailsOnAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_audit_events_on_ruby_object_in_details' + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name(:audit_events, INDEX_NAME) + end + + def down + add_concurrent_index(:audit_events, :id, where: "details ~~ '%ruby/object%'", name: INDEX_NAME) + end +end diff --git a/db/post_migrate/20200713071042_confirm_project_bot_users.rb b/db/post_migrate/20200713071042_confirm_project_bot_users.rb new file mode 100644 index 00000000000..0578fc42ef2 --- /dev/null +++ b/db/post_migrate/20200713071042_confirm_project_bot_users.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ConfirmProjectBotUsers < ActiveRecord::Migration[6.0] + DOWNTIME = false + + disable_ddl_transaction! + + class User < ApplicationRecord + self.table_name = 'users' + + include ::EachBatch + + USER_TYPE_PROJECT_BOT = 6 + + scope :project_bots, -> { where(user_type: USER_TYPE_PROJECT_BOT) } + scope :unconfirmed, -> { where(confirmed_at: nil) } + end + + def up + User.reset_column_information + + User.project_bots.unconfirmed.each_batch do |relation| + relation.update_all('confirmed_at = created_at') + end + end + + def down + # no-op + end +end |