diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-02-20 16:49:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-02-20 16:49:51 +0300 |
commit | 71786ddc8e28fbd3cb3fcc4b3ff15e5962a1c82e (patch) | |
tree | 6a2d93ef3fb2d353bb7739e4b57e6541f51cdd71 /lib/gitlab/background_migration | |
parent | a7253423e3403b8c08f8a161e5937e1488f5f407 (diff) |
Add latest changes from gitlab-org/gitlab@15-9-stable-eev15.9.0-rc42
Diffstat (limited to 'lib/gitlab/background_migration')
11 files changed, 294 insertions, 121 deletions
diff --git a/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb b/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb new file mode 100644 index 00000000000..46e2d5cb930 --- /dev/null +++ b/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Iterates through the namespaces table and attempts to set the + # opposite of the value of the column "emails_disabled" to a new + # column in namespace_settings called emails_enabled + class AddNamespacesEmailsEnabledColumnData < BatchedMigrationJob + feature_category :database + operation_name :add_namespaces_emails_enabled_column_data + + # Targeted table + class NamespaceSetting < ApplicationRecord + self.table_name = 'namespace_settings' + end + + def perform + each_sub_batch do |sub_batch| + plucked_list = sub_batch.where('NOT emails_disabled IS NULL').pluck(:id, :emails_disabled) + next if plucked_list.empty? + + ApplicationRecord.connection.execute <<~SQL + UPDATE namespace_settings + SET emails_enabled= NOT subquery.emails_enabled + FROM (SELECT * FROM (#{Arel::Nodes::ValuesList.new(plucked_list).to_sql}) AS updates(namespace_id, emails_enabled)) AS subquery + WHERE namespace_settings.namespace_id=subquery.namespace_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb b/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb new file mode 100644 index 00000000000..a0ce5d22597 --- /dev/null +++ b/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Iterates through the Projects table and attempts to set the + # opposite of the value of the column "emails_disabled" to a new + # column in project_settings called emails_enabled + class AddProjectsEmailsEnabledColumnData < BatchedMigrationJob + feature_category :database + operation_name :add_projects_emails_enabled_column_data + + # Targeted table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch do |sub_batch| + plucked_list = sub_batch.where('NOT emails_disabled IS NULL').pluck(:id, :emails_disabled) + next if plucked_list.empty? + + ApplicationRecord.connection.execute <<~SQL + UPDATE project_settings + SET emails_enabled=NOT subquery.emails_enabled + FROM (SELECT * FROM (#{Arel::Nodes::ValuesList.new(plucked_list).to_sql}) AS updates(project_id, emails_enabled)) AS subquery + WHERE project_settings.project_id=subquery.project_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb deleted file mode 100644 index 669e5338dd1..00000000000 --- a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2, -# it's enough at the moment to simply notice if the url is from `atlassian.net` -module Gitlab - module BackgroundMigration - # Backfill the deployment_type in jira_tracker_data table - class BackfillJiraTrackerDeploymentType2 - # Migration only version of jira_tracker_data table - class JiraTrackerDataTemp < ApplicationRecord - self.table_name = 'jira_tracker_data' - - def self.encryption_options - { - key: Settings.attr_encrypted_db_key_base_32, - encode: true, - mode: :per_attribute_iv, - algorithm: 'aes-256-gcm' - } - end - - attr_encrypted :url, encryption_options - attr_encrypted :api_url, encryption_options - - enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment - end - - # Migration only version of services table - class JiraServiceTemp < ApplicationRecord - self.table_name = 'services' - self.inheritance_column = :_type_disabled - end - - def perform(start_id, stop_id) - @server_ids = [] - @cloud_ids = [] - - JiraTrackerDataTemp - .where(id: start_id..stop_id, deployment_type: 0) - .each do |jira_tracker_data| - collect_deployment_type(jira_tracker_data) - end - - unless cloud_ids.empty? - JiraTrackerDataTemp.where(id: cloud_ids) - .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud]) - end - - unless server_ids.empty? - JiraTrackerDataTemp.where(id: server_ids) - .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server]) - end - - mark_jobs_as_succeeded(start_id, stop_id) - end - - private - - attr_reader :server_ids, :cloud_ids - - def client_url(jira_tracker_data) - jira_tracker_data.api_url.presence || jira_tracker_data.url.presence - end - - def server_type(url) - url.downcase.include?('.atlassian.net') ? :cloud : :server - end - - def collect_deployment_type(jira_tracker_data) - url = client_url(jira_tracker_data) - return unless url - - case server_type(url) - when :cloud - cloud_ids << jira_tracker_data.id - else - server_ids << jira_tracker_data.id - end - end - - def mark_jobs_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name.demodulize, arguments) - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb new file mode 100644 index 00000000000..b6e22e481fa --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migration to make sure that all the prevously saved tokens have their encrypted values in the db. + class EncryptCiTriggerToken < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :continuous_integration + scope_to ->(relation) { relation.where(encrypted_token: nil) } + operation_name :update + # Class that is imitating Ci::Trigger + class CiTrigger < ::Ci::ApplicationRecord + ALGORITHM = 'aes-256-gcm' + + self.table_name = 'ci_triggers' + + attr_encrypted :encrypted_token_tmp, + attribute: :encrypted_token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_vi: false + + before_save :copy_token_to_encrypted_token + + def copy_token_to_encrypted_token + self.encrypted_token_tmp = token + end + end + + def perform + each_sub_batch do |sub_batch| + sub_batch.each do |trigger| + Gitlab::BackgroundMigration::EncryptCiTriggerToken::CiTrigger.find(trigger.id).save! + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb index 3843356af69..c9582da2a51 100644 --- a/lib/gitlab/background_migration/encrypt_integration_properties.rb +++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb @@ -31,7 +31,7 @@ module Gitlab def encrypt_properties data = ::Gitlab::Json.parse(properties) iv = generate_iv(ALGORITHM) - ep = self.class.encrypt(:encrypted_properties_tmp, data, { iv: iv }) + ep = self.class.attr_encrypt(:encrypted_properties_tmp, data, { iv: iv }) [ep, iv] end diff --git a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb new file mode 100644 index 00000000000..4b6bb12c91b --- /dev/null +++ b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A migration that will: + # * get all project statistics where packages_size is non zero + # * check the coherence with the related package files + # * fix non coherent packages_size values + class FixIncoherentPackagesSizeOnProjectStatistics < BatchedMigrationJob + MIGRATOR = 'FixIncoherentPackagesSizeOnProjectStatistics' + + feature_category :package_registry + + operation_name :fix_incorrect_packages_size + + def perform + each_sub_batch do |sub_batch| + fix_packages_size(sub_batch) + end + end + + private + + def fix_packages_size(project_statistics) + statistics_table = FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics.arel_table + from = [ + statistics_table, + FixIncoherentPackagesSizeOnProjectStatistics::PackageFile.sum_query.arel.lateral.as('size_sum') + ] + size_sum_table = ::Arel::Table.new(:size_sum) + + project_statistics.select(:id, :project_id, :packages_size, size_sum_table[:total]) + .from(from) + .where.not(statistics_table[:packages_size].eq(size_sum_table[:total])) + .each do |stat| + increment = stat[:total].to_i - stat[:packages_size] - buffered_update(stat) + next if increment == 0 + + ::Gitlab::BackgroundMigration::Logger.info( + migrator: MIGRATOR, + project_id: stat[:project_id], + old_size: stat[:packages_size], + new_size: stat[:total].to_i + ) + + stat.becomes(FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics) # rubocop:disable Cop/AvoidBecomes + .increment(increment) + end + end + + def buffered_update(stat) + key = "project:{#{stat[:project_id]}}:counters:ProjectStatistics:#{stat[:id]}:packages_size" + + Gitlab::Redis::SharedState.with do |redis| + redis.get(key).to_i + end + end + + # rubocop:disable Style/Documentation + class ProjectStatistics < ::ApplicationRecord + self.table_name = 'project_statistics' + + def increment(amount) + FixIncoherentPackagesSizeOnProjectStatistics::BufferedCounter.new(self).increment(amount) + end + end + + class Package < ::ApplicationRecord + self.table_name = 'packages_packages' + + has_many :package_files, + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' # rubocop:disable Layout/LineLength + end + + class PackageFile < ::ApplicationRecord + self.table_name = 'packages_package_files' + + belongs_to :package, + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' # rubocop:disable Layout/LineLength + + def self.sum_query + packages = FixIncoherentPackagesSizeOnProjectStatistics::Package.arel_table + stats = FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics.arel_table + + joins(:package) + .where(packages[:project_id].eq(stats[:project_id])) + .where.not(size: nil) + .select('SUM(packages_package_files.size) as total') + end + end + + class BufferedCounter + WORKER_DELAY = 10.minutes + + def initialize(stat) + @stat = stat + end + + def key + "project:{#{@stat.project_id}}:counters:ProjectStatistics:#{@stat.id}:packages_size" + end + + def increment(amount) + Gitlab::Redis::SharedState.with do |redis| + redis.incrby(key, amount) + end + + FlushCounterIncrementsWorker.perform_in( + WORKER_DELAY, + 'ProjectStatistics', + @stat.id, + :packages_size + ) + end + end + # rubocop:enable Style/Documentation + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb index 914ababa5c2..db3f98bc2ba 100644 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -68,7 +68,7 @@ module Gitlab def valid_json?(metadata) Oj.load(metadata) true - rescue Oj::ParseError, EncodingError, JSON::ParserError, Encoding::UndefinedConversionError + rescue Oj::ParseError, EncodingError, JSON::ParserError, JSON::GeneratorError, Encoding::UndefinedConversionError false end diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb new file mode 100644 index 00000000000..592ef3220ff --- /dev/null +++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify `projects.creator_id` column of projects who creator + # does not exist in `users` table anymore. + class NullifyCreatorIdColumnOfOrphanedProjects < BatchedMigrationJob + scope_to ->(relation) do + relation.where.not(creator_id: nil) + .joins('LEFT OUTER JOIN users ON users.id = projects.creator_id') + .where(users: { id: nil }) + end + + operation_name :update_all + feature_category :projects + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(creator_id: nil) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/rebalance_partition_id.rb b/lib/gitlab/background_migration/rebalance_partition_id.rb new file mode 100644 index 00000000000..7000ae5a856 --- /dev/null +++ b/lib/gitlab/background_migration/rebalance_partition_id.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This rebalances partition_id to fix invalid records in production + class RebalancePartitionId < BatchedMigrationJob + INVALID_PARTITION_ID = 101 + VALID_PARTITION_ID = 100 + + scope_to ->(relation) { relation.where(partition_id: INVALID_PARTITION_ID) } + operation_name :update_all + feature_category :continuous_integration + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(partition_id: VALID_PARTITION_ID) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/sanitize_confidential_todos.rb b/lib/gitlab/background_migration/sanitize_confidential_todos.rb index 2df0b8a4d93..8215e92cbeb 100644 --- a/lib/gitlab/background_migration/sanitize_confidential_todos.rb +++ b/lib/gitlab/background_migration/sanitize_confidential_todos.rb @@ -10,43 +10,14 @@ module Gitlab # to extract all related logic. # Details in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908#note_952459215 class SanitizeConfidentialTodos < BatchedMigrationJob - scope_to ->(relation) { relation.where(confidential: true) } - operation_name :delete_invalid_todos feature_category :database def perform - each_sub_batch do |sub_batch| - delete_ids = invalid_todo_ids(sub_batch) - - Todo.where(id: delete_ids).delete_all if delete_ids.present? - end - end - - private - - def invalid_todo_ids(notes_batch) - todos = Todo.where(note_id: notes_batch.select(:id)).includes(:note, :user) - - todos.each_with_object([]) do |todo, ids| - ids << todo.id if invalid_todo?(todo) - end - end - - def invalid_todo?(todo) - return false unless todo.note - return false if Ability.allowed?(todo.user, :read_todo, todo) - - logger.info( - message: "#{self.class.name} deleting invalid todo", - attributes: todo.attributes - ) - - true - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build + # no-op: this BG migration is left here only for compatibility reasons, + # but it's not scheduled from any migration anymore. + # It was a temporary migration which used not-isolated code. + # https://gitlab.com/gitlab-org/gitlab/-/issues/382557 end end end diff --git a/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb b/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb new file mode 100644 index 00000000000..24080a06c46 --- /dev/null +++ b/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class ThirdRecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +# we just want to re-enqueue the previous BackfillEpicCacheCounts migration, +# because it's a EE-only migation and it's a module, we just prepend new +# RecountEpicCacheCounts with existing batched migration module (which is same in both cases) +Gitlab::BackgroundMigration::ThirdRecountEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength |