diff options
Diffstat (limited to 'db/post_migrate')
12 files changed, 341 insertions, 0 deletions
diff --git a/db/post_migrate/20201026185514_ensure_u2f_registrations_migrated.rb b/db/post_migrate/20201026185514_ensure_u2f_registrations_migrated.rb new file mode 100644 index 00000000000..121b9fee623 --- /dev/null +++ b/db/post_migrate/20201026185514_ensure_u2f_registrations_migrated.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class EnsureU2fRegistrationsMigrated < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + BACKGROUND_MIGRATION_CLASS = 'MigrateU2fWebauthn' + BATCH_SIZE = 100 + DOWNTIME = false + + disable_ddl_transaction! + + class U2fRegistration < ActiveRecord::Base + include EachBatch + + self.table_name = 'u2f_registrations' + end + + def up + Gitlab::BackgroundMigration.steal(BACKGROUND_MIGRATION_CLASS) + + # Do a manual update in case we lost BG jobs. The expected record count should be 0 or very low. + U2fRegistration + .joins("LEFT JOIN webauthn_registrations ON webauthn_registrations.u2f_registration_id = u2f_registrations.id") + .where("webauthn_registrations.u2f_registration_id IS NULL") + .each_batch(of: BATCH_SIZE) do |batch, index| + batch.each do |record| + Gitlab::BackgroundMigration::MigrateU2fWebauthn.new.perform(record.id, record.id) + rescue => e + Gitlab::ErrorTracking.track_exception(e, u2f_registration_id: record.id) + end + end + end + + def down + # no-op (we can't "unsteal" migrations) + end +end diff --git a/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb b/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb new file mode 100644 index 00000000000..2cb7c9c5250 --- /dev/null +++ b/db/post_migrate/20201030121314_schedule_update_existing_users_that_require_two_factor_auth.rb @@ -0,0 +1,35 @@ +# # frozen_string_literal: true + +class ScheduleUpdateExistingUsersThatRequireTwoFactorAuth < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'UpdateExistingUsersThatRequireTwoFactorAuth' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + INDEX_NAME = 'index_users_on_require_two_factor_authentication_from_group' + + disable_ddl_transaction! + + class User < ActiveRecord::Base + include EachBatch + + self.table_name = 'users' + end + + def up + add_concurrent_index :users, + :require_two_factor_authentication_from_group, + where: 'require_two_factor_authentication_from_group = TRUE', + name: INDEX_NAME + + relation = User.where(require_two_factor_authentication_from_group: true) + + queue_background_migration_jobs_by_range_at_intervals( + relation, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + end + + def down + remove_concurrent_index_by_name :users, INDEX_NAME + end +end diff --git a/db/post_migrate/20201112145311_add_index_on_sha_for_initial_deployments.rb b/db/post_migrate/20201112145311_add_index_on_sha_for_initial_deployments.rb new file mode 100644 index 00000000000..15debddb9cc --- /dev/null +++ b/db/post_migrate/20201112145311_add_index_on_sha_for_initial_deployments.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddIndexOnShaForInitialDeployments < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + NEW_INDEX_NAME = 'index_deployments_on_environment_status_sha' + OLD_INDEX_NAME = 'index_deployments_on_environment_id_and_status' + + disable_ddl_transaction! + + def up + add_concurrent_index :deployments, %i[environment_id status sha], name: NEW_INDEX_NAME + remove_concurrent_index_by_name :deployments, OLD_INDEX_NAME + end + + def down + add_concurrent_index :deployments, %i[environment_id status], name: OLD_INDEX_NAME + remove_concurrent_index_by_name :services, NEW_INDEX_NAME + end +end diff --git a/db/post_migrate/20201113105000_update_index_secure_for_api_fuzzing_telemetry.rb b/db/post_migrate/20201113105000_update_index_secure_for_api_fuzzing_telemetry.rb new file mode 100644 index 00000000000..9e8313f79f8 --- /dev/null +++ b/db/post_migrate/20201113105000_update_index_secure_for_api_fuzzing_telemetry.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class UpdateIndexSecureForApiFuzzingTelemetry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + disable_ddl_transaction! + + OLD_SECURE_INDEX_NAME = 'index_secure_ci_builds_on_user_id_created_at_parser_features' + NEW_SECURE_INDEX_NAME = 'index_secure_ci_builds_on_user_id_name_created_at' + + def up + add_concurrent_index(:ci_builds, + [:user_id, :name, :created_at], + where: "(((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('coverage_fuzzing'::character varying)::text, ('apifuzzer_fuzz'::character varying)::text, ('apifuzzer_fuzz_dnd'::character varying)::text, ('secret_detection'::character varying)::text])))", + name: NEW_SECURE_INDEX_NAME) + remove_concurrent_index_by_name :ci_builds, OLD_SECURE_INDEX_NAME + end + + def down + add_concurrent_index(:ci_builds, + [:user_id, :created_at], + where: "(((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('secret_detection'::character varying)::text])))", + name: OLD_SECURE_INDEX_NAME) + remove_concurrent_index_by_name :ci_builds, NEW_SECURE_INDEX_NAME + end +end diff --git a/db/post_migrate/20201119092319_schedule_repopulate_historical_vulnerability_statistics.rb b/db/post_migrate/20201119092319_schedule_repopulate_historical_vulnerability_statistics.rb new file mode 100644 index 00000000000..598cc4d93d0 --- /dev/null +++ b/db/post_migrate/20201119092319_schedule_repopulate_historical_vulnerability_statistics.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class ScheduleRepopulateHistoricalVulnerabilityStatistics < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 50 + DELAY_INTERVAL = 5.minutes + MIGRATION_CLASS = 'PopulateVulnerabilityHistoricalStatistics' + DAY_COUNT = 365 + + disable_ddl_transaction! + + class ProjectSetting < ActiveRecord::Base + include EachBatch + + self.table_name = 'project_settings' + + scope :has_vulnerabilities, -> { where('has_vulnerabilities IS TRUE') } + end + + def up + ProjectSetting.has_vulnerabilities.each_batch(of: BATCH_SIZE) do |batch, index| + migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, [batch.pluck(:project_id), DAY_COUNT]) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201120071303_drop_feature_filter_type_from_user_preferences.rb b/db/post_migrate/20201120071303_drop_feature_filter_type_from_user_preferences.rb new file mode 100644 index 00000000000..b00ea0aba76 --- /dev/null +++ b/db/post_migrate/20201120071303_drop_feature_filter_type_from_user_preferences.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class DropFeatureFilterTypeFromUserPreferences < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + with_lock_retries do + remove_column :user_preferences, :feature_filter_type + end + end + + def down + with_lock_retries do + add_column :user_preferences, :feature_filter_type, :bigint + end + end +end diff --git a/db/post_migrate/20201120140210_add_runner_id_and_id_desc_index_to_ci_builds.rb b/db/post_migrate/20201120140210_add_runner_id_and_id_desc_index_to_ci_builds.rb new file mode 100644 index 00000000000..5eda0e25dbe --- /dev/null +++ b/db/post_migrate/20201120140210_add_runner_id_and_id_desc_index_to_ci_builds.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddRunnerIdAndIdDescIndexToCiBuilds < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + NEW_INDEX = 'index_ci_builds_on_runner_id_and_id_desc' + OLD_INDEX = 'index_ci_builds_on_runner_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_builds, %i[runner_id id], name: NEW_INDEX, order: { id: :desc } + remove_concurrent_index_by_name :ci_builds, OLD_INDEX + end + + def down + add_concurrent_index :ci_builds, %i[runner_id], name: OLD_INDEX + remove_concurrent_index_by_name :ci_builds, NEW_INDEX + end +end diff --git a/db/post_migrate/20201124122817_populate_remaining_missing_dismissal_information_for_vulnerabilities.rb b/db/post_migrate/20201124122817_populate_remaining_missing_dismissal_information_for_vulnerabilities.rb new file mode 100644 index 00000000000..9dc41d17231 --- /dev/null +++ b/db/post_migrate/20201124122817_populate_remaining_missing_dismissal_information_for_vulnerabilities.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class PopulateRemainingMissingDismissalInformationForVulnerabilities < ActiveRecord::Migration[6.0] + DOWNTIME = false + + disable_ddl_transaction! + + def up + Gitlab::BackgroundMigration.steal('PopulateMissingVulnerabilityDismissalInformation') + + ::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Vulnerability.broken.each_batch(of: 100) do |batch, index| + vulnerability_ids = batch.pluck(:id) + + ::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation.new.perform(*vulnerability_ids) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201124185639_remove_unused_indexes.rb b/db/post_migrate/20201124185639_remove_unused_indexes.rb new file mode 100644 index 00000000000..c4b0d8a84cc --- /dev/null +++ b/db/post_migrate/20201124185639_remove_unused_indexes.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class RemoveUnusedIndexes < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :packages_package_files, "packages_packages_verification_failure_partial" + remove_concurrent_index_by_name :packages_package_files, "packages_packages_verification_checksum_partial" + remove_concurrent_index_by_name :snippet_repositories, 'snippet_repositories_verification_failure_partial' + remove_concurrent_index_by_name :snippet_repositories, 'snippet_repositories_verification_checksum_partial' + remove_concurrent_index_by_name :terraform_state_versions, 'terraform_state_versions_verification_failure_partial' + remove_concurrent_index_by_name :terraform_state_versions, 'terraform_state_versions_verification_checksum_partial' + end + + def down + add_concurrent_index :packages_package_files, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "packages_packages_verification_failure_partial" + add_concurrent_index :packages_package_files, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "packages_packages_verification_checksum_partial" + add_concurrent_index :snippet_repositories, :verification_failure, where: "(verification_failure IS NOT NULL)", name: 'snippet_repositories_verification_failure_partial' + add_concurrent_index :snippet_repositories, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: 'snippet_repositories_verification_checksum_partial' + add_concurrent_index :terraform_state_versions, :verification_failure, where: "(verification_failure IS NOT NULL)", name: 'terraform_state_versions_verification_failure_partial' + add_concurrent_index :terraform_state_versions, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: 'terraform_state_versions_verification_checksum_partial' + end +end diff --git a/db/post_migrate/20201130103926_schedule_populate_dismissed_state_for_vulnerabilities.rb b/db/post_migrate/20201130103926_schedule_populate_dismissed_state_for_vulnerabilities.rb new file mode 100644 index 00000000000..5e8da532251 --- /dev/null +++ b/db/post_migrate/20201130103926_schedule_populate_dismissed_state_for_vulnerabilities.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +class SchedulePopulateDismissedStateForVulnerabilities < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + TMP_INDEX_NAME = 'tmp_index_on_vulnerabilities_non_dismissed' + + DOWNTIME = false + BATCH_SIZE = 1_000 + VULNERABILITY_BATCH_SIZE = 5_000 + DELAY_INTERVAL = 3.minutes.to_i + MIGRATION_CLASS = 'PopulateDismissedStateForVulnerabilities' + + VULNERABILITY_JOIN_CONDITION = 'JOIN "vulnerability_occurrences" ON "vulnerability_occurrences"."vulnerability_id" = "vulnerabilities"."id"' + FEEDBACK_WHERE_CONDITION = <<~SQL + EXISTS (SELECT 1 FROM vulnerability_feedback + WHERE "vulnerability_occurrences"."project_id" = "vulnerability_feedback"."project_id" + AND "vulnerability_occurrences"."report_type" = "vulnerability_feedback"."category" + AND ENCODE("vulnerability_occurrences"."project_fingerprint", 'hex') = "vulnerability_feedback"."project_fingerprint" + AND "vulnerability_feedback"."feedback_type" = 0 + ) + SQL + + class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'vulnerabilities' + end + + disable_ddl_transaction! + + def up + add_concurrent_index(:vulnerabilities, :id, where: 'state <> 2', name: TMP_INDEX_NAME) + + batch = [] + index = 1 + + Vulnerability.where('state <> 2').each_batch(of: VULNERABILITY_BATCH_SIZE) do |relation| + ids = relation + .joins(VULNERABILITY_JOIN_CONDITION) + .where(FEEDBACK_WHERE_CONDITION) + .pluck('vulnerabilities.id') + + ids.each do |id| + batch << id + + if batch.size == BATCH_SIZE + migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, batch) + index += 1 + + batch.clear + end + end + end + + migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, batch) unless batch.empty? + end + + def down + remove_concurrent_index_by_name(:vulnerabilities, TMP_INDEX_NAME) + end +end diff --git a/db/post_migrate/20201203123201_remove_orphan_service_hooks.rb b/db/post_migrate/20201203123201_remove_orphan_service_hooks.rb new file mode 100644 index 00000000000..c430e2205c2 --- /dev/null +++ b/db/post_migrate/20201203123201_remove_orphan_service_hooks.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class RemoveOrphanServiceHooks < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + class WebHook < ActiveRecord::Base + include EachBatch + + self.table_name = 'web_hooks' + + def self.service_hooks + where(type: 'ServiceHook') + end + end + + class Service < ActiveRecord::Base + self.table_name = 'services' + end + + def up + WebHook.service_hooks.where.not(service_id: Service.select(:id)).where.not(service_id: nil).each_batch do |relation| + relation.delete_all + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20201207151651_truncate_security_findings_table_2.rb b/db/post_migrate/20201207151651_truncate_security_findings_table_2.rb new file mode 100644 index 00000000000..2ac6941be6d --- /dev/null +++ b/db/post_migrate/20201207151651_truncate_security_findings_table_2.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require_relative Rails.root.join('db', 'post_migrate', '20201102152945_truncate_security_findings_table.rb') + +# This is the second time we are truncating this table +# so the migration class name has choosen like this for this reason. +class TruncateSecurityFindingsTable2 < TruncateSecurityFindingsTable; end |