diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 21:42:06 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 21:42:06 +0300 |
commit | 6e4e1050d9dba2b7b2523fdd1768823ab85feef4 (patch) | |
tree | 78be5963ec075d80116a932011d695dd33910b4e /db/post_migrate | |
parent | 1ce776de4ae122aba3f349c02c17cebeaa8ecf07 (diff) |
Add latest changes from gitlab-org/gitlab@13-3-stable-ee
Diffstat (limited to 'db/post_migrate')
28 files changed, 918 insertions, 0 deletions
diff --git a/db/post_migrate/20200601120434_migrate_all_merge_request_user_mentions_to_db.rb b/db/post_migrate/20200601120434_migrate_all_merge_request_user_mentions_to_db.rb new file mode 100644 index 00000000000..fc7b9afe5f9 --- /dev/null +++ b/db/post_migrate/20200601120434_migrate_all_merge_request_user_mentions_to_db.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class MigrateAllMergeRequestUserMentionsToDb < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DELAY = 2.minutes.to_i + BATCH_SIZE = 100_000 + MIGRATION = 'UserMentions::CreateResourceUserMention' + + JOIN = "LEFT JOIN merge_request_user_mentions on merge_requests.id = merge_request_user_mentions.merge_request_id" + QUERY_CONDITIONS = "(description LIKE '%@%' OR title LIKE '%@%') AND merge_request_user_mentions.merge_request_id IS NULL" + + disable_ddl_transaction! + + class MergeRequest < ActiveRecord::Base + include EachBatch + end + + def up + delay = DELAY + + MergeRequest.each_batch(of: BATCH_SIZE) do |batch, _| + range = batch.pluck('MIN(merge_requests.id)', 'MAX(merge_requests.id)').first + records_count = MergeRequest.joins(JOIN).where(QUERY_CONDITIONS).where(id: range.first..range.last).count + + if records_count > 0 + migrate_in(delay, MIGRATION, ['MergeRequest', JOIN, QUERY_CONDITIONS, false, *range]) + delay += [DELAY, (records_count / 500 + 1).minutes.to_i].max + end + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200703165434_drop_temporary_table_untracked_files_for_uploads_if_exists.rb b/db/post_migrate/20200703165434_drop_temporary_table_untracked_files_for_uploads_if_exists.rb new file mode 100644 index 00000000000..1f34a05227d --- /dev/null +++ b/db/post_migrate/20200703165434_drop_temporary_table_untracked_files_for_uploads_if_exists.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class DropTemporaryTableUntrackedFilesForUploadsIfExists < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + execute 'DROP TABLE IF EXISTS untracked_files_for_uploads' + end + + def down + # no-op - this table should not exist + end +end diff --git a/db/post_migrate/20200714075739_schedule_populate_personal_snippet_statistics.rb b/db/post_migrate/20200714075739_schedule_populate_personal_snippet_statistics.rb new file mode 100644 index 00000000000..c3033da24fe --- /dev/null +++ b/db/post_migrate/20200714075739_schedule_populate_personal_snippet_statistics.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class SchedulePopulatePersonalSnippetStatistics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 500 + MIGRATION = 'PopulatePersonalSnippetStatistics' + + disable_ddl_transaction! + + def up + snippets = exec_query <<~SQL + SELECT id + FROM snippets + WHERE type = 'PersonalSnippet' + ORDER BY author_id ASC, id ASC + SQL + + snippets.rows.flatten.in_groups_of(BATCH_SIZE, false).each_with_index do |snippet_ids, index| + migrate_in(index * DELAY_INTERVAL, MIGRATION, [snippet_ids]) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200721140507_update_index_for_coverage_fuzzing_telemetry.rb b/db/post_migrate/20200721140507_update_index_for_coverage_fuzzing_telemetry.rb new file mode 100644 index 00000000000..4f884319125 --- /dev/null +++ b/db/post_migrate/20200721140507_update_index_for_coverage_fuzzing_telemetry.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class UpdateIndexForCoverageFuzzingTelemetry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + disable_ddl_transaction! + + OLD_INDEX_NAME = 'index_security_ci_builds_on_name_and_id' + NEW_INDEX_NAME = 'index_security_ci_builds_on_name_and_id_parser_features' + + OLD_CLAUSE = "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, + ('dast'::character varying)::text, + ('dependency_scanning'::character varying)::text, + ('license_management'::character varying)::text, + ('sast'::character varying)::text, + ('secret_detection'::character varying)::text, + ('license_scanning'::character varying)::text])) AND ((type)::text = 'Ci::Build'::text)" + + NEW_CLAUSE = "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, + ('dast'::character varying)::text, + ('dependency_scanning'::character varying)::text, + ('license_management'::character varying)::text, + ('sast'::character varying)::text, + ('secret_detection'::character varying)::text, + ('coverage_fuzzing'::character varying)::text, + ('license_scanning'::character varying)::text])) AND ((type)::text = 'Ci::Build'::text)" + + def up + add_concurrent_index :ci_builds, [:name, :id], name: NEW_INDEX_NAME, where: NEW_CLAUSE + remove_concurrent_index_by_name :ci_builds, OLD_INDEX_NAME + end + + def down + add_concurrent_index :ci_builds, [:name, :id], name: OLD_INDEX_NAME, where: OLD_CLAUSE + remove_concurrent_index_by_name :ci_builds, NEW_INDEX_NAME + end +end diff --git a/db/post_migrate/20200722202318_backfill_partitioned_audit_events.rb b/db/post_migrate/20200722202318_backfill_partitioned_audit_events.rb new file mode 100644 index 00000000000..538aa94e22b --- /dev/null +++ b/db/post_migrate/20200722202318_backfill_partitioned_audit_events.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class BackfillPartitionedAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::PartitioningMigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + return if ::Gitlab.com? + + enqueue_partitioning_data_migration :audit_events + end + + def down + return if ::Gitlab.com? + + cleanup_partitioning_data_migration :audit_events + end +end diff --git a/db/post_migrate/20200723040950_migrate_incident_issues_to_incident_type.rb b/db/post_migrate/20200723040950_migrate_incident_issues_to_incident_type.rb new file mode 100644 index 00000000000..0f562630381 --- /dev/null +++ b/db/post_migrate/20200723040950_migrate_incident_issues_to_incident_type.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class MigrateIncidentIssuesToIncidentType < ActiveRecord::Migration[6.0] + DOWNTIME = false + BATCH_SIZE = 100 + + disable_ddl_transaction! + + LABEL_PROPERTIES = { + title: 'incident' + }.freeze + + class Issue < ActiveRecord::Base + include EachBatch + + self.table_name = 'issues' + + scope :incident_labelled, -> do + joins("INNER JOIN label_links ON label_links.target_type = 'Issue' AND label_links.target_id = issues.id") + .joins("INNER JOIN labels ON labels.id = label_links.label_id") + .where(labels: LABEL_PROPERTIES) + end + + enum issue_type: { + issue: 0, + incident: 1 + } + + scope :incident_typed, -> { where(issue_type: :incident) } + end + + def up + incident_issues = Issue.incident_labelled + + incident_issues.each_batch(of: BATCH_SIZE) do |batch| + batch.update_all(issue_type: :incident) + end + end + + def down + incident_issues = Issue.incident_typed + + incident_issues.each_batch(of: BATCH_SIZE) do |batch| + batch.update_all(issue_type: :issue) + end + end +end diff --git a/db/post_migrate/20200723132258_schedule_copy_of_mr_target_project_id_to_mr_metrics.rb b/db/post_migrate/20200723132258_schedule_copy_of_mr_target_project_id_to_mr_metrics.rb new file mode 100644 index 00000000000..bca703121bc --- /dev/null +++ b/db/post_migrate/20200723132258_schedule_copy_of_mr_target_project_id_to_mr_metrics.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class ScheduleCopyOfMrTargetProjectIdToMrMetrics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INTERVAL = 2.minutes.to_i + BATCH_SIZE = 5_000 + MIGRATION = 'CopyMergeRequestTargetProjectToMergeRequestMetrics' + + disable_ddl_transaction! + + class MergeRequest < ActiveRecord::Base + include EachBatch + + self.table_name = 'merge_requests' + end + + def up + MergeRequest.reset_column_information + + queue_background_migration_jobs_by_range_at_intervals( + MergeRequest, + MIGRATION, + INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + # noop + end +end diff --git a/db/post_migrate/20200724100421_remove_updated_at_from_audit_events.rb b/db/post_migrate/20200724100421_remove_updated_at_from_audit_events.rb new file mode 100644 index 00000000000..0dc02491db5 --- /dev/null +++ b/db/post_migrate/20200724100421_remove_updated_at_from_audit_events.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +class RemoveUpdatedAtFromAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::SchemaHelpers + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + SOURCE_TABLE_NAME = 'audit_events' + PARTITIONED_TABLE_NAME = 'audit_events_part_5fc467ac26' + TRIGGER_FUNCTION_NAME = 'table_sync_function_2be879775d' + + def up + with_lock_retries do + remove_column SOURCE_TABLE_NAME, :updated_at + + create_trigger_function(TRIGGER_FUNCTION_NAME, replace: true) do + <<~SQL + IF (TG_OP = 'DELETE') THEN + DELETE FROM #{PARTITIONED_TABLE_NAME} where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + UPDATE #{PARTITIONED_TABLE_NAME} + SET author_id = NEW.author_id, + type = NEW.type, + entity_id = NEW.entity_id, + entity_type = NEW.entity_type, + details = NEW.details, + ip_address = NEW.ip_address, + author_name = NEW.author_name, + entity_path = NEW.entity_path, + target_details = NEW.target_details, + created_at = NEW.created_at + WHERE #{PARTITIONED_TABLE_NAME}.id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO #{PARTITIONED_TABLE_NAME} (id, + author_id, + type, + entity_id, + entity_type, + details, + ip_address, + author_name, + entity_path, + target_details, + created_at) + VALUES (NEW.id, + NEW.author_id, + NEW.type, + NEW.entity_id, + NEW.entity_type, + NEW.details, + NEW.ip_address, + NEW.author_name, + NEW.entity_path, + NEW.target_details, + NEW.created_at); + END IF; + RETURN NULL; + SQL + end + + remove_column PARTITIONED_TABLE_NAME, :updated_at + end + end + + def down + with_lock_retries do + add_column SOURCE_TABLE_NAME, :updated_at, :datetime # rubocop:disable Migration/Datetime + add_column PARTITIONED_TABLE_NAME, :updated_at, :datetime # rubocop:disable Migration/Datetime + + create_trigger_function(TRIGGER_FUNCTION_NAME, replace: true) do + <<~SQL + IF (TG_OP = 'DELETE') THEN + DELETE FROM #{PARTITIONED_TABLE_NAME} where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + UPDATE #{PARTITIONED_TABLE_NAME} + SET author_id = NEW.author_id, + type = NEW.type, + entity_id = NEW.entity_id, + entity_type = NEW.entity_type, + details = NEW.details, + updated_at = NEW.updated_at, + ip_address = NEW.ip_address, + author_name = NEW.author_name, + entity_path = NEW.entity_path, + target_details = NEW.target_details, + created_at = NEW.created_at + WHERE #{PARTITIONED_TABLE_NAME}.id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO #{PARTITIONED_TABLE_NAME} (id, + author_id, + type, + entity_id, + entity_type, + details, + updated_at, + ip_address, + author_name, + entity_path, + target_details, + created_at) + VALUES (NEW.id, + NEW.author_id, + NEW.type, + NEW.entity_id, + NEW.entity_type, + NEW.details, + NEW.updated_at, + NEW.ip_address, + NEW.author_name, + NEW.entity_path, + NEW.target_details, + NEW.created_at); + END IF; + RETURN NULL; + SQL + end + end + end +end diff --git a/db/post_migrate/20200724130639_backfill_designs_relative_position.rb b/db/post_migrate/20200724130639_backfill_designs_relative_position.rb new file mode 100644 index 00000000000..ef8b38bf90c --- /dev/null +++ b/db/post_migrate/20200724130639_backfill_designs_relative_position.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# This migration is not needed anymore and was disabled, because we're now +# also backfilling design positions immediately before moving a design. +# +# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39555 +class BackfillDesignsRelativePosition < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + # no-op + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200727100631_remove_again_gitlab_issue_tracker_service_records.rb b/db/post_migrate/20200727100631_remove_again_gitlab_issue_tracker_service_records.rb new file mode 100644 index 00000000000..b61da82aef8 --- /dev/null +++ b/db/post_migrate/20200727100631_remove_again_gitlab_issue_tracker_service_records.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class RemoveAgainGitlabIssueTrackerServiceRecords < ActiveRecord::Migration[6.0] + DOWNTIME = false + BATCH_SIZE = 5000 + + disable_ddl_transaction! + + class Service < ActiveRecord::Base + include EachBatch + + self.table_name = 'services' + + def self.gitlab_issue_tracker_service + where(type: 'GitlabIssueTrackerService') + end + end + + def up + Service.each_batch(of: BATCH_SIZE) do |services| + services.gitlab_issue_tracker_service.delete_all + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200727114147_remove_title_and_description_from_services.rb b/db/post_migrate/20200727114147_remove_title_and_description_from_services.rb new file mode 100644 index 00000000000..0c908e17438 --- /dev/null +++ b/db/post_migrate/20200727114147_remove_title_and_description_from_services.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class RemoveTitleAndDescriptionFromServices < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + remove_column :services, :title, :string + remove_column :services, :description, :string, limit: 500 + end +end diff --git a/db/post_migrate/20200730133730_remove_table_vulnerability_export_verification_status.rb b/db/post_migrate/20200730133730_remove_table_vulnerability_export_verification_status.rb new file mode 100644 index 00000000000..9aa15fa7d1a --- /dev/null +++ b/db/post_migrate/20200730133730_remove_table_vulnerability_export_verification_status.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class RemoveTableVulnerabilityExportVerificationStatus < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + return unless table_exists?(:vulnerability_export_verification_status) + + drop_table :vulnerability_export_verification_status + end + + def down + # no-op because `vulnerability_export_verification_status` table should not + # be created, see https://gitlab.com/gitlab-org/gitlab/-/issues/232977 + end +end diff --git a/db/post_migrate/20200804035230_add_partial_index_on_id_to_merge_request_diffs.rb b/db/post_migrate/20200804035230_add_partial_index_on_id_to_merge_request_diffs.rb new file mode 100644 index 00000000000..94b939eb811 --- /dev/null +++ b/db/post_migrate/20200804035230_add_partial_index_on_id_to_merge_request_diffs.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddPartialIndexOnIdToMergeRequestDiffs < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_merge_request_diffs_external_diff_store_is_null' + + disable_ddl_transaction! + + def up + add_concurrent_index :merge_request_diffs, :id, where: 'external_diff_store IS NULL', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :merge_request_diffs, INDEX_NAME + end +end diff --git a/db/post_migrate/20200804041930_add_not_null_constraint_on_external_diff_store_to_merge_request_diffs.rb b/db/post_migrate/20200804041930_add_not_null_constraint_on_external_diff_store_to_merge_request_diffs.rb new file mode 100644 index 00000000000..c33e790b37b --- /dev/null +++ b/db/post_migrate/20200804041930_add_not_null_constraint_on_external_diff_store_to_merge_request_diffs.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNotNullConstraintOnExternalDiffStoreToMergeRequestDiffs < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_not_null_constraint(:merge_request_diffs, :external_diff_store, validate: false) + end + + def down + remove_not_null_constraint(:merge_request_diffs, :external_diff_store) + end +end diff --git a/db/post_migrate/20200805152108_migrate_null_external_diff_store_to_local_value.rb b/db/post_migrate/20200805152108_migrate_null_external_diff_store_to_local_value.rb new file mode 100644 index 00000000000..e42f23d8fae --- /dev/null +++ b/db/post_migrate/20200805152108_migrate_null_external_diff_store_to_local_value.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class MigrateNullExternalDiffStoreToLocalValue < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + JOB_INTERVAL = 2.minutes + 5.seconds + BATCH_SIZE = 5_000 + MIGRATION = 'SetNullExternalDiffStoreToLocalValue' + + disable_ddl_transaction! + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + def up + # On GitLab.com, 19M of 93M rows have NULL external_diff_store. + # + # With batches of 5000 and a background migration job interval of 2m 5s, + # 3.8K jobs are scheduled over 5.5 days. + # + # The index `index_merge_request_diffs_external_diff_store_is_null` is + # expected to be used here and in the jobs. + # + # queue_background_migration_jobs_by_range_at_intervals is not used because + # it would enqueue 18.6K jobs and we have an index for getting these ranges. + MergeRequestDiff.where(external_diff_store: nil).each_batch(of: BATCH_SIZE) do |batch, index| + range = batch.pluck(Arel.sql("MIN(id)"), Arel.sql("MAX(id)")).first + delay = index * JOB_INTERVAL + + migrate_in(delay.seconds, MIGRATION, [*range]) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20200806004742_add_not_null_constraint_on_file_store_to_package_files.rb b/db/post_migrate/20200806004742_add_not_null_constraint_on_file_store_to_package_files.rb new file mode 100644 index 00000000000..7b7250fef9a --- /dev/null +++ b/db/post_migrate/20200806004742_add_not_null_constraint_on_file_store_to_package_files.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNotNullConstraintOnFileStoreToPackageFiles < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_not_null_constraint(:packages_package_files, :file_store, validate: false) + end + + def down + remove_not_null_constraint(:packages_package_files, :file_store) + end +end diff --git a/db/post_migrate/20200806172909_add_partial_index_on_id_to_package_files.rb b/db/post_migrate/20200806172909_add_partial_index_on_id_to_package_files.rb new file mode 100644 index 00000000000..7904574f890 --- /dev/null +++ b/db/post_migrate/20200806172909_add_partial_index_on_id_to_package_files.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddPartialIndexOnIdToPackageFiles < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_packages_package_files_file_store_is_null' + + disable_ddl_transaction! + + def up + add_concurrent_index :packages_package_files, :id, where: 'file_store IS NULL', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :packages_package_files, INDEX_NAME + end +end diff --git a/db/post_migrate/20200806173633_migrate_null_package_files_file_store_to_local_value.rb b/db/post_migrate/20200806173633_migrate_null_package_files_file_store_to_local_value.rb new file mode 100644 index 00000000000..088b85dd5da --- /dev/null +++ b/db/post_migrate/20200806173633_migrate_null_package_files_file_store_to_local_value.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class MigrateNullPackageFilesFileStoreToLocalValue < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + JOB_INTERVAL = 2.minutes + 5.seconds + BATCH_SIZE = 5_000 + MIGRATION = 'SetNullPackageFilesFileStoreToLocalValue' + + disable_ddl_transaction! + + class PackageFile < ActiveRecord::Base + self.table_name = 'packages_package_files' + + include ::EachBatch + end + + def up + # On GitLab.com, there are 2M package files. None have NULL file_store + # because they are all object stored. This is a no-op for GitLab.com. + # + # If a customer had 2M package files with NULL file_store, with batches of + # 5000 and a background migration job interval of 2m 5s, then 400 jobs would + # be scheduled over 14 hours. + # + # The index `index_packages_package_files_file_store_is_null` is + # expected to be used here and in the jobs. + # + # queue_background_migration_jobs_by_range_at_intervals is not used because + # it would enqueue 18.6K jobs and we have an index for getting these ranges. + PackageFile.where(file_store: nil).each_batch(of: BATCH_SIZE) do |batch, index| + range = batch.pluck(Arel.sql("MIN(id)"), Arel.sql("MAX(id)")).first + delay = index * JOB_INTERVAL + + migrate_in(delay.seconds, MIGRATION, [*range]) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20200807110237_add_migration_index_to_vulnerabilities_occurrences.rb b/db/post_migrate/20200807110237_add_migration_index_to_vulnerabilities_occurrences.rb new file mode 100644 index 00000000000..e806a337f8f --- /dev/null +++ b/db/post_migrate/20200807110237_add_migration_index_to_vulnerabilities_occurrences.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddMigrationIndexToVulnerabilitiesOccurrences < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :vulnerability_occurrences, + "project_id, report_type, encode(project_fingerprint, 'hex'::text)", + name: 'index_vulnerability_occurrences_for_issue_links_migration' + end + + def down + remove_concurrent_index_by_name :vulnerability_occurrences, + :index_vulnerability_occurrences_for_issue_links_migration + end +end diff --git a/db/post_migrate/20200807152315_backfill_merge_request_diffs_files_counts.rb b/db/post_migrate/20200807152315_backfill_merge_request_diffs_files_counts.rb new file mode 100644 index 00000000000..bc7e4712d19 --- /dev/null +++ b/db/post_migrate/20200807152315_backfill_merge_request_diffs_files_counts.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class BackfillMergeRequestDiffsFilesCounts < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + # There are ~72 million records on GitLab.com at time of writing, so go fast + BATCH_SIZE = 10_000 + DELAY_INTERVAL = 2.minutes.to_i + MIGRATION = 'SetMergeRequestDiffFilesCount' + + disable_ddl_transaction! + + class MergeRequestDiff < ActiveRecord::Base + include EachBatch + + self.table_name = 'merge_request_diffs' + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + MergeRequestDiff, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20200808221641_add_index_for_license_compliance_artifacts.rb b/db/post_migrate/20200808221641_add_index_for_license_compliance_artifacts.rb new file mode 100644 index 00000000000..fce4ee095bd --- /dev/null +++ b/db/post_migrate/20200808221641_add_index_for_license_compliance_artifacts.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddIndexForLicenseComplianceArtifacts < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_ci_job_artifacts_on_license_compliance_file_types' + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_job_artifacts, [:job_id, :file_type], where: 'file_type = 10 OR file_type = 101', name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_job_artifacts, name: INDEX_NAME + end +end diff --git a/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb b/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb new file mode 100644 index 00000000000..0a5dfd72392 --- /dev/null +++ b/db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class MigrateLicenseManagementArtifactsToLicenseScanning < ActiveRecord::Migration[6.0] + DOWNTIME = false + LICENSE_MANAGEMENT_FILE_TYPE = 10 + LICENSE_SCANNING_FILE_TYPE = 101 + + disable_ddl_transaction! + + class JobArtifact < ActiveRecord::Base + include EachBatch + + self.table_name = 'ci_job_artifacts' + end + + # We're updating file_type of ci artifacts from license_management to license_scanning + # But before that we need to delete "rogue" artifacts for CI builds that have associated with them + # both license_scanning and license_management artifacts. It's an edge case and usually, we don't have + # such builds in the database. + def up + return unless Gitlab.ee? + + JobArtifact + .where("file_type = 10 OR file_type = 101") + .each_batch(column: :job_id, of: 1000) do |relation| + min, max = relation.pluck('MIN(job_id)', 'MAX(job_id)').flatten + + ActiveRecord::Base.connection.execute <<~SQL + WITH ci_job_artifacts_with_row_number as ( + SELECT job_id, id, ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY id ASC) as row_number + FROM ci_job_artifacts + WHERE (file_type = #{LICENSE_SCANNING_FILE_TYPE} OR file_type = #{LICENSE_MANAGEMENT_FILE_TYPE}) + AND job_id >= #{Integer(min)} AND job_id < #{Integer(max)} + ) + DELETE FROM ci_job_artifacts + WHERE ci_job_artifacts.id IN (SELECT id from ci_job_artifacts_with_row_number WHERE ci_job_artifacts_with_row_number.row_number > 1) + SQL + end + + JobArtifact.where(file_type: LICENSE_MANAGEMENT_FILE_TYPE).each_batch(column: :job_id, of: 1000) do |relation| + relation.update_all(file_type: LICENSE_SCANNING_FILE_TYPE) + end + end + + def down + # no-op + # we're deleting duplicating artifacts and updating file_type for license_management artifacts + end +end diff --git a/db/post_migrate/20200810100921_add_target_type_to_audit_event.rb b/db/post_migrate/20200810100921_add_target_type_to_audit_event.rb new file mode 100644 index 00000000000..8dde5945f0d --- /dev/null +++ b/db/post_migrate/20200810100921_add_target_type_to_audit_event.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +class AddTargetTypeToAuditEvent < ActiveRecord::Migration[6.0] + include Gitlab::Database::SchemaHelpers + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + SOURCE_TABLE_NAME = 'audit_events' + PARTITIONED_TABLE_NAME = 'audit_events_part_5fc467ac26' + TRIGGER_FUNCTION_NAME = 'table_sync_function_2be879775d' + + def up + with_lock_retries do + # rubocop:disable Migration/AddLimitToTextColumns + add_column('audit_events', :target_type, :text) + add_column('audit_events_part_5fc467ac26', :target_type, :text) + # rubocop:enable Migration/AddLimitToTextColumns + + create_trigger_function(TRIGGER_FUNCTION_NAME, replace: true) do + <<~SQL + IF (TG_OP = 'DELETE') THEN + DELETE FROM #{PARTITIONED_TABLE_NAME} where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + UPDATE #{PARTITIONED_TABLE_NAME} + SET author_id = NEW.author_id, + type = NEW.type, + entity_id = NEW.entity_id, + entity_type = NEW.entity_type, + details = NEW.details, + ip_address = NEW.ip_address, + author_name = NEW.author_name, + entity_path = NEW.entity_path, + target_details = NEW.target_details, + target_type = NEW.target_type, + created_at = NEW.created_at + WHERE #{PARTITIONED_TABLE_NAME}.id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO #{PARTITIONED_TABLE_NAME} (id, + author_id, + type, + entity_id, + entity_type, + details, + ip_address, + author_name, + entity_path, + target_details, + target_type, + created_at) + VALUES (NEW.id, + NEW.author_id, + NEW.type, + NEW.entity_id, + NEW.entity_type, + NEW.details, + NEW.ip_address, + NEW.author_name, + NEW.entity_path, + NEW.target_details, + NEW.target_type, + NEW.created_at); + END IF; + RETURN NULL; + SQL + end + end + end + + def down + with_lock_retries do + remove_column SOURCE_TABLE_NAME, :target_type + + create_trigger_function(TRIGGER_FUNCTION_NAME, replace: true) do + <<~SQL + IF (TG_OP = 'DELETE') THEN + DELETE FROM #{PARTITIONED_TABLE_NAME} where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + UPDATE #{PARTITIONED_TABLE_NAME} + SET author_id = NEW.author_id, + type = NEW.type, + entity_id = NEW.entity_id, + entity_type = NEW.entity_type, + details = NEW.details, + ip_address = NEW.ip_address, + author_name = NEW.author_name, + entity_path = NEW.entity_path, + target_details = NEW.target_details, + created_at = NEW.created_at + WHERE #{PARTITIONED_TABLE_NAME}.id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO #{PARTITIONED_TABLE_NAME} (id, + author_id, + type, + entity_id, + entity_type, + details, + ip_address, + author_name, + entity_path, + target_details, + created_at) + VALUES (NEW.id, + NEW.author_id, + NEW.type, + NEW.entity_id, + NEW.entity_type, + NEW.details, + NEW.ip_address, + NEW.author_name, + NEW.entity_path, + NEW.target_details, + NEW.created_at); + END IF; + RETURN NULL; + SQL + end + + remove_column PARTITIONED_TABLE_NAME, :target_type + end + end +end diff --git a/db/post_migrate/20200810101029_add_text_limit_to_audit_event_target_type.rb b/db/post_migrate/20200810101029_add_text_limit_to_audit_event_target_type.rb new file mode 100644 index 00000000000..2a5ea9cd245 --- /dev/null +++ b/db/post_migrate/20200810101029_add_text_limit_to_audit_event_target_type.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddTextLimitToAuditEventTargetType < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + SOURCE_TABLE_NAME = 'audit_events' + PARTITIONED_TABLE_NAME = 'audit_events_part_5fc467ac26' + + disable_ddl_transaction! + + def up + add_text_limit(SOURCE_TABLE_NAME, :target_type, 255) + add_text_limit(PARTITIONED_TABLE_NAME, :target_type, 255) + end + + def down + remove_text_limit(SOURCE_TABLE_NAME, :target_type) + remove_text_limit(PARTITIONED_TABLE_NAME, :target_type) + end +end diff --git a/db/post_migrate/20200810160355_add_not_null_constraint_to_ci_pipeline_artifact_file.rb b/db/post_migrate/20200810160355_add_not_null_constraint_to_ci_pipeline_artifact_file.rb new file mode 100644 index 00000000000..0c233d44313 --- /dev/null +++ b/db/post_migrate/20200810160355_add_not_null_constraint_to_ci_pipeline_artifact_file.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddNotNullConstraintToCiPipelineArtifactFile < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_not_null_constraint :ci_pipeline_artifacts, :file, validate: true + end + + def down + remove_not_null_constraint :ci_pipeline_artifacts, :file + end +end diff --git a/db/post_migrate/20200813153434_import_latest_common_metrics.rb b/db/post_migrate/20200813153434_import_latest_common_metrics.rb new file mode 100644 index 00000000000..203e495674f --- /dev/null +++ b/db/post_migrate/20200813153434_import_latest_common_metrics.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class ImportLatestCommonMetrics < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + # The common_metrics queries were updated to work with K8s versions that + # use the pod/container label names as well as K8s versions that use the + # older pod_name/container_name convention. + ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute + end + + def down + # no-op + # The import cannot be reversed since we do not know the state that the + # common metrics in the PrometheusMetric table were in before the import. + + # To manually revert this migration. + # 1. Go back to the previous version of the config/prometheus/common_metrics.yml file. (git checkout 74447f11349617ed8b273196d6a5781d9a67a613) + # 2. Execute `rails runner '::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute'` + end +end diff --git a/db/post_migrate/20200817070126_update_index_secure_for_coverage_fuzzing_telemetry.rb b/db/post_migrate/20200817070126_update_index_secure_for_coverage_fuzzing_telemetry.rb new file mode 100644 index 00000000000..d9aa7314115 --- /dev/null +++ b/db/post_migrate/20200817070126_update_index_secure_for_coverage_fuzzing_telemetry.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class UpdateIndexSecureForCoverageFuzzingTelemetry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + disable_ddl_transaction! + + OLD_SECURE_INDEX_NAME = 'index_secure_ci_builds_on_user_id_created_at' + NEW_SECURE_INDEX_NAME = 'index_secure_ci_builds_on_user_id_created_at_parser_features' + + def up + add_concurrent_index(:ci_builds, + [:user_id, :created_at], + where: "(((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('coverage_fuzzing'::character varying)::text, ('secret_detection'::character varying)::text])))", + name: NEW_SECURE_INDEX_NAME) + remove_concurrent_index_by_name :ci_builds, OLD_SECURE_INDEX_NAME + end + + def down + add_concurrent_index(:ci_builds, + [:user_id, :created_at], + where: "(((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('secret_detection'::character varying)::text])))", + name: OLD_SECURE_INDEX_NAME) + remove_concurrent_index_by_name :ci_builds, NEW_SECURE_INDEX_NAME + end +end diff --git a/db/post_migrate/20200821224343_schedule_populate_vulnerability_historical_statistics.rb b/db/post_migrate/20200821224343_schedule_populate_vulnerability_historical_statistics.rb new file mode 100644 index 00000000000..f3a57ab78ef --- /dev/null +++ b/db/post_migrate/20200821224343_schedule_populate_vulnerability_historical_statistics.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class SchedulePopulateVulnerabilityHistoricalStatistics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DELAY_INTERVAL = 2.minutes.to_i + BATCH_SIZE = 50 + MIGRATION = 'PopulateVulnerabilityHistoricalStatistics' + + disable_ddl_transaction! + + class Vulnerability < ActiveRecord::Base + self.table_name = 'vulnerabilities' + + include ::EachBatch + end + + def up + return unless Gitlab.ee? + + Vulnerability.select('project_id').distinct.each_batch(of: BATCH_SIZE, column: 'project_id') do |project_batch, index| + migrate_in(index * DELAY_INTERVAL, MIGRATION, [project_batch.pluck(:project_id)]) + end + end + + def down + # no-op + end +end |