diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-18 12:45:46 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-18 12:45:46 +0300 |
commit | a7b3560714b4d9cc4ab32dffcd1f74a284b93580 (patch) | |
tree | 7452bd5c3545c2fa67a28aa013835fb4fa071baf /lib/gitlab/background_migration | |
parent | ee9173579ae56a3dbfe5afe9f9410c65bb327ca7 (diff) |
Add latest changes from gitlab-org/gitlab@14-8-stable-eev14.8.0-rc42
Diffstat (limited to 'lib/gitlab/background_migration')
33 files changed, 427 insertions, 664 deletions
diff --git a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb b/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb new file mode 100644 index 00000000000..63112b52584 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Ensure queuing entries are present even if admins skip upgrades. + class BackfillCiQueuingTables + class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + end + + class Project < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'projects' + + belongs_to :namespace + has_one :ci_cd_settings, class_name: 'Gitlab::BackgroundMigration::BackfillCiQueuingTables::ProjectCiCdSetting' + + def group_runners_enabled? + return false unless ci_cd_settings + + ci_cd_settings.group_runners_enabled? + end + end + + class ProjectCiCdSetting < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'project_ci_cd_settings' + end + + class Taggings < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'taggings' + end + + module Ci + class Build < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + + belongs_to :project + + scope :pending, -> do + where(status: :pending, type: 'Ci::Build', runner_id: nil) + end + + def self.each_batch(of: 1000, column: :id, order: { runner_id: :asc, id: :asc }, order_hint: nil) + start = except(:select).select(column).reorder(order) + start = start.take + return unless start + + start_id = start[column] + arel_table = self.arel_table + + 1.step do |index| + start_cond = arel_table[column].gteq(start_id) + stop = except(:select).select(column).where(start_cond).reorder(order) + stop = stop.offset(of).limit(1).take + relation = where(start_cond) + + if stop + stop_id = stop[column] + start_id = stop_id + stop_cond = arel_table[column].lt(stop_id) + relation = relation.where(stop_cond) + end + + # Any ORDER BYs are useless for this relation and can lead to less + # efficient UPDATE queries, hence we get rid of it. + relation = relation.except(:order) + + # Using unscoped is necessary to prevent leaking the current scope used by + # ActiveRecord to chain `each_batch` method. + unscoped { yield relation, index } + + break unless stop + end + end + + def tags_ids + BackfillCiQueuingTables::Taggings + .where(taggable_id: id, taggable_type: 'CommitStatus') + .pluck(:tag_id) + end + end + + class PendingBuild < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'ci_pending_builds' + + class << self + def upsert_from_build!(build) + entry = self.new(args_from_build(build)) + + self.upsert( + entry.attributes.compact, + returning: %w[build_id], + unique_by: :build_id) + end + + def args_from_build(build) + project = build.project + + { + build_id: build.id, + project_id: build.project_id, + protected: build.protected?, + namespace_id: project.namespace_id, + tag_ids: build.tags_ids, + instance_runners_enabled: project.shared_runners_enabled?, + namespace_traversal_ids: namespace_traversal_ids(project) + } + end + + def namespace_traversal_ids(project) + if project.group_runners_enabled? + project.namespace.traversal_ids + else + [] + end + end + end + end + end + + BATCH_SIZE = 100 + + def perform(start_id, end_id) + scope = BackfillCiQueuingTables::Ci::Build.pending.where(id: start_id..end_id) + pending_builds_query = BackfillCiQueuingTables::Ci::PendingBuild + .where('ci_builds.id = ci_pending_builds.build_id') + .select(1) + + scope.each_batch(of: BATCH_SIZE) do |builds| + builds = builds.where('NOT EXISTS (?)', pending_builds_query) + builds = builds.includes(:project, project: [:namespace, :ci_cd_settings]) + + builds.each do |build| + BackfillCiQueuingTables::Ci::PendingBuild.upsert_from_build!(build) + end + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_designs_relative_position.rb b/lib/gitlab/background_migration/backfill_designs_relative_position.rb deleted file mode 100644 index efbb1b950ad..00000000000 --- a/lib/gitlab/background_migration/backfill_designs_relative_position.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration is not needed anymore and was disabled, because we're now - # also backfilling design positions immediately before moving a design. - # - # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39555 - class BackfillDesignsRelativePosition - def perform(issue_ids) - # no-op - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_integrations_type_new.rb b/lib/gitlab/background_migration/backfill_integrations_type_new.rb index d1a939af58e..a234cebfce5 100644 --- a/lib/gitlab/background_migration/backfill_integrations_type_new.rb +++ b/lib/gitlab/background_migration/backfill_integrations_type_new.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Database::DynamicModelHelpers def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = define_batchable_model(batch_table) + parent_batch_relation = define_batchable_model(batch_table, connection: connection) .where(batch_column => start_id..stop_id) parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb b/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb deleted file mode 100644 index 6dc92672929..00000000000 --- a/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will fill the project_repositories table for projects that - # are on legacy storage and an entry is is missing in this table. - class BackfillLegacyProjectRepositories < BackfillProjectRepositories - private - - def projects - Project.with_parent.on_legacy_storage - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb new file mode 100644 index 00000000000..fe3edd3322b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `routes.namespace_id` column, by copying source_id value + # (for groups and user namespaces source_id == namespace_id) + class BackfillNamespaceIdForNamespaceRoute + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:update_all) do + sub_batch.update_all('namespace_id=source_id') + end + + pause_ms = [0, pause_ms].max + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + .joins('inner join namespaces on routes.source_id = namespaces.id') + .where(source_key_column => start_id..stop_id) + .where(namespace_id: nil) + .where(source_type: 'Namespace') + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb deleted file mode 100644 index 030dfd2d99b..00000000000 --- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Update existent project update_at column after their repository storage was moved - class BackfillProjectUpdatedAtAfterRepositoryStorageMove - def perform(*project_ids) - updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) - - Project.connection.execute <<-SQL - WITH repository_storage_cte as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - #{updated_repository_storages.to_sql} - ) - UPDATE projects - SET updated_at = (repository_storage_cte.updated_at + interval '1 second') - FROM repository_storage_cte - WHERE projects.id = repository_storage_cte.project_id AND projects.updated_at <= repository_storage_cte.updated_at - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_user_namespace.rb b/lib/gitlab/background_migration/backfill_user_namespace.rb index f55eaa3b14e..ab569e236fb 100644 --- a/lib/gitlab/background_migration/backfill_user_namespace.rb +++ b/lib/gitlab/background_migration/backfill_user_namespace.rb @@ -29,7 +29,7 @@ module Gitlab end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table) + define_batchable_model(source_table, connection: connection) .where(source_key_column => start_id..stop_id) .where(type: nil) end diff --git a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb deleted file mode 100644 index 41f7f7f2f24..00000000000 --- a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class BackfillVersionDataFromGitaly - def perform(issue_id) - end - end - end -end - -Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_mod_with('Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly') diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb new file mode 100644 index 00000000000..f352c527b54 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Batching class to use for back-filling project namespaces for a single group. + # Batches over the projects table and id column combination, scoped to a given group returning the MIN() and MAX() + # values for the next batch as an array. + # + # If no more batches exist in the table, returns nil. + class BackfillProjectNamespacePerGroupBatchingStrategy < PrimaryKeyBatchingStrategy + # Finds and returns the next batch in the table. + # + # table_name - The table to batch over + # column_name - The column to batch over + # batch_min_value - The minimum value which the next batch will start at + # batch_size - The size of the next batch + # job_arguments - The migration job arguments + def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:) + next_batch_bounds = nil + model_class = ::Gitlab::BackgroundMigration::ProjectNamespaces::Models::Project + quoted_column_name = model_class.connection.quote_column_name(column_name) + projects_table = model_class.arel_table + hierarchy_cte_sql = Arel::Nodes::SqlLiteral.new(::Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces.hierarchy_cte(job_arguments.first)) + relation = model_class.where(projects_table[:namespace_id].in(hierarchy_cte_sql)).where("#{quoted_column_name} >= ?", batch_min_value) + + relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop + next_batch_bounds = batch.pluck(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")).first + + break + end + + next_batch_bounds + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index 80693728e86..09700438d47 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -17,8 +17,9 @@ module Gitlab # column_name - The column to batch over # batch_min_value - The minimum value which the next batch will start at # batch_size - The size of the next batch - def next_batch(table_name, column_name, batch_min_value:, batch_size:) - model_class = define_batchable_model(table_name) + # job_arguments - The migration job arguments + def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:) + model_class = define_batchable_model(table_name, connection: ActiveRecord::Base.connection) quoted_column_name = model_class.connection.quote_column_name(column_name) relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb deleted file mode 100644 index 7b334b9c1d0..00000000000 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class CalculateWikiSizes - def perform(start_id, stop_id) - ::ProjectStatistics.where(wiki_size: nil) - .where(id: start_id..stop_id) - .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| - statistics.refresh!(only: [:wiki_size]) - rescue StandardError => e - Gitlab::AppLogger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" - end - end - end - end -end diff --git a/lib/gitlab/background_migration/cleanup_optimistic_locking_nulls.rb b/lib/gitlab/background_migration/cleanup_optimistic_locking_nulls.rb deleted file mode 100644 index bf69ef352cc..00000000000 --- a/lib/gitlab/background_migration/cleanup_optimistic_locking_nulls.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class CleanupOptimisticLockingNulls - QUERY_ITEM_SIZE = 1_000 - - # table - The name of the table the migration is performed for. - # start_id - The ID of the object to start at - # stop_id - The ID of the object to end at - def perform(start_id, stop_id, table) - model = define_model_for(table) - - # After analysis done, a batch size of 1,000 items per query was found to be - # the most optimal. Discussion in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18418#note_282285336 - (start_id..stop_id).each_slice(QUERY_ITEM_SIZE).each do |range| - model - .where(lock_version: nil) - .where("ID BETWEEN ? AND ?", range.first, range.last) - .update_all(lock_version: 0) - end - end - - def define_model_for(table) - Class.new(ActiveRecord::Base) do - self.table_name = table - end - end - end - end -end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 529b8cdf8d4..137b4d4bc4e 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -13,7 +13,7 @@ module Gitlab # - We skip the NULL checks as they may result in not using an index scan # - The table that is migrated does _not_ need `id` as the primary key # We use the provided primary_key column to perform the update. - class CopyColumnUsingBackgroundMigrationJob + class CopyColumnUsingBackgroundMigrationJob < BaseJob include Gitlab::Database::DynamicModelHelpers # start_id - The start ID of the range of rows to update. @@ -52,12 +52,8 @@ module Gitlab private - def connection - ActiveRecord::Base.connection - end - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table).where(source_key_column => start_id..stop_id) + define_batchable_model(source_table, connection: connection).where(source_key_column => start_id..stop_id) end def column_assignment_clauses(copy_from, copy_to) diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb index 9ac4111ff0f..5d41a46c8cd 100644 --- a/lib/gitlab/background_migration/delete_orphaned_deployments.rb +++ b/lib/gitlab/background_migration/delete_orphaned_deployments.rb @@ -15,7 +15,7 @@ module Gitlab end def orphaned_deployments - define_batchable_model('deployments') + define_batchable_model('deployments', connection: ActiveRecord::Base.connection) .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb deleted file mode 100644 index 7b5c32e3d6d..00000000000 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # save validity time pages domain - class FillValidTimeForPagesDomainCertificate - # define PagesDomain with only needed code - class PagesDomain < ActiveRecord::Base - self.table_name = 'pages_domains' - - def x509 - return unless certificate.present? - - @x509 ||= OpenSSL::X509::Certificate.new(certificate) - rescue OpenSSL::X509::CertificateError - nil - end - end - - def perform(start_id, stop_id) - PagesDomain.where(id: start_id..stop_id).find_each do |domain| - # for some reason activerecord doesn't append timezone, iso8601 forces this - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before&.iso8601, - certificate_valid_not_after: domain.x509&.not_after&.iso8601 - ) - rescue StandardError => e - Gitlab::AppLogger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" - end - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb b/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb new file mode 100644 index 00000000000..2c09b8c0b24 --- /dev/null +++ b/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class FixIncorrectMaxSeatsUsed + def perform(batch = nil) + end + end + end +end + +Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed.prepend_mod_with('Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed') diff --git a/lib/gitlab/background_migration/fix_pages_access_level.rb b/lib/gitlab/background_migration/fix_pages_access_level.rb deleted file mode 100644 index 8e46021bd93..00000000000 --- a/lib/gitlab/background_migration/fix_pages_access_level.rb +++ /dev/null @@ -1,128 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # corrects stored pages access level on db depending on project visibility - class FixPagesAccessLevel - # Copy routable here to avoid relying on application logic - module Routable - def build_full_path - if parent && path - parent.build_full_path + '/' + path - else - path - end - end - end - - # Namespace - class Namespace < ActiveRecord::Base - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - - include Routable - - belongs_to :parent, class_name: "Namespace" - end - - # Project - class Project < ActiveRecord::Base - self.table_name = 'projects' - self.inheritance_column = :_type_disabled - - include Routable - - belongs_to :namespace - alias_method :parent, :namespace - alias_attribute :parent_id, :namespace_id - - PRIVATE = 0 - INTERNAL = 10 - PUBLIC = 20 - - def pages_deployed? - Dir.exist?(public_pages_path) - end - - def public_pages_path - File.join(pages_path, 'public') - end - - def pages_path - # TODO: when we migrate Pages to work with new storage types, change here to use disk_path - File.join(Settings.pages.path, build_full_path) - end - end - - # ProjectFeature - class ProjectFeature < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'project_features' - - belongs_to :project - - PRIVATE = 10 - ENABLED = 20 - PUBLIC = 30 - end - - def perform(start_id, stop_id) - fix_public_access_level(start_id, stop_id) - - make_internal_projects_public(start_id, stop_id) - - fix_private_access_level(start_id, stop_id) - end - - private - - def access_control_is_enabled - @access_control_is_enabled = Gitlab.config.pages.access_control - end - - # Public projects are allowed to have only enabled pages_access_level - # which is equivalent to public - def fix_public_access_level(start_id, stop_id) - project_features(start_id, stop_id, ProjectFeature::PUBLIC, Project::PUBLIC).each_batch do |features| - features.update_all(pages_access_level: ProjectFeature::ENABLED) - end - end - - # If access control is disabled and project has pages deployed - # project will become unavailable when access control will become enabled - # we make these projects public to avoid negative surprise to user - def make_internal_projects_public(start_id, stop_id) - return if access_control_is_enabled - - project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::INTERNAL).find_each do |project_feature| - next unless project_feature.project.pages_deployed? - - project_feature.update(pages_access_level: ProjectFeature::PUBLIC) - end - end - - # Private projects are not allowed to have enabled access level, only `private` and `public` - # If access control is enabled, these projects currently behave as if they have `private` pages_access_level - # if access control is disabled, these projects currently behave as if they have `public` pages_access_level - # so we preserve this behaviour for projects with pages already deployed - # for project without pages we always set `private` access_level - def fix_private_access_level(start_id, stop_id) - project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::PRIVATE).find_each do |project_feature| - if access_control_is_enabled - project_feature.update!(pages_access_level: ProjectFeature::PRIVATE) - else - fixed_access_level = project_feature.project.pages_deployed? ? ProjectFeature::PUBLIC : ProjectFeature::PRIVATE - project_feature.update!(pages_access_level: fixed_access_level) - end - end - end - - def project_features(start_id, stop_id, pages_access_level, project_visibility_level) - ProjectFeature.where(id: start_id..stop_id).joins(:project) - .where(pages_access_level: pages_access_level) - .where(projects: { visibility_level: project_visibility_level }) - end - end - end -end diff --git a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb deleted file mode 100644 index 160e6d2fe8b..00000000000 --- a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class GenerateGitlabSubscriptions - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_mod_with('Gitlab::BackgroundMigration::GenerateGitlabSubscriptions') diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index 5dc77f935e3..b7d47c389df 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -10,28 +10,32 @@ module Gitlab # Also provides a database connection to the correct tracking database. class JobCoordinator # rubocop:disable Metrics/ClassLength class << self + def for_tracking_database(tracking_database) + worker_class = worker_for_tracking_database[tracking_database] + + if worker_class.nil? + raise ArgumentError, "tracking_database must be one of [#{worker_for_tracking_database.keys.join(', ')}]" + end + + new(worker_class) + end + + private + def worker_classes @worker_classes ||= [ - BackgroundMigrationWorker + ::BackgroundMigrationWorker, + ::BackgroundMigration::CiDatabaseWorker ].freeze end def worker_for_tracking_database @worker_for_tracking_database ||= worker_classes + .select { |worker_class| Gitlab::Database.has_config?(worker_class.tracking_database) } .index_by(&:tracking_database) .with_indifferent_access .freeze end - - def for_tracking_database(tracking_database) - worker_class = worker_for_tracking_database[tracking_database] - - if worker_class.nil? - raise ArgumentError, "tracking_database must be one of [#{worker_for_tracking_database.keys.join(', ')}]" - end - - new(worker_class) - end end attr_reader :worker_class @@ -146,7 +150,7 @@ module Gitlab def connection @connection ||= Gitlab::Database .database_base_models - .fetch(worker_class.tracking_database, Gitlab::Database::PRIMARY_DATABASE_NAME) + .fetch(worker_class.tracking_database) .connection end end diff --git a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb deleted file mode 100644 index d85f980d3f1..00000000000 --- a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module BackgroundMigration - # EE-specific migration - class MigrateDevopsSegmentsToGroups - def perform - # no-op for CE - end - end - end -end - -Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_mod_with('Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') diff --git a/lib/gitlab/background_migration/migrate_job_artifact_registry_to_ssf.rb b/lib/gitlab/background_migration/migrate_job_artifact_registry_to_ssf.rb new file mode 100644 index 00000000000..bcc02c1dbf2 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_job_artifact_registry_to_ssf.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MigrateJobArtifactRegistryToSsf + def perform(*job_artifact_ids) + end + end + end +end + +Gitlab::BackgroundMigration::MigrateJobArtifactRegistryToSsf.prepend_mod_with('Gitlab::BackgroundMigration::MigrateJobArtifactRegistryToSsf') diff --git a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb deleted file mode 100644 index 909bf10341a..00000000000 --- a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class populates the `finding_uuid` attribute for - # the existing `vulnerability_feedback` records. - class PopulateFindingUuidForVulnerabilityFeedback - REPORT_TYPES = { - sast: 0, - dependency_scanning: 1, - container_scanning: 2, - dast: 3, - secret_detection: 4, - coverage_fuzzing: 5, - api_fuzzing: 6 - }.freeze - - class VulnerabilityFeedback < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'vulnerability_feedback' - - enum category: REPORT_TYPES - - scope :in_range, -> (start, stop) { where(id: start..stop) } - scope :without_uuid, -> { where(finding_uuid: nil) } - - def self.load_vulnerability_findings - all.to_a.tap { |collection| collection.each(&:vulnerability_finding) } - end - - def set_finding_uuid - return unless vulnerability_finding.present? && vulnerability_finding.primary_identifier.present? - - update_column(:finding_uuid, calculated_uuid) - rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - end - - def vulnerability_finding - BatchLoader.for(finding_key).batch do |finding_keys, loader| - project_ids = finding_keys.map { |key| key[:project_id] } - categories = finding_keys.map { |key| key[:category] } - fingerprints = finding_keys.map { |key| key[:project_fingerprint] } - - findings = Finding.with_primary_identifier.where( - project_id: project_ids.uniq, - report_type: categories.uniq, - project_fingerprint: fingerprints.uniq - ).to_a - - finding_keys.each do |finding_key| - loader.call( - finding_key, - findings.find { |f| finding_key == f.finding_key } - ) - end - end - end - - private - - def calculated_uuid - ::Security::VulnerabilityUUID.generate( - report_type: category, - primary_identifier_fingerprint: vulnerability_finding.primary_identifier.fingerprint, - location_fingerprint: vulnerability_finding.location_fingerprint, - project_id: project_id - ) - end - - def finding_key - { - project_id: project_id, - category: category, - project_fingerprint: project_fingerprint - } - end - end - - class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation - include ShaAttribute - - self.table_name = 'vulnerability_occurrences' - - sha_attribute :project_fingerprint - sha_attribute :location_fingerprint - - belongs_to :primary_identifier, class_name: 'Gitlab::BackgroundMigration::PopulateFindingUuidForVulnerabilityFeedback::Identifier' - - enum report_type: REPORT_TYPES - - scope :with_primary_identifier, -> { includes(:primary_identifier) } - - def finding_key - { - project_id: project_id, - category: report_type, - project_fingerprint: project_fingerprint - } - end - end - - class Identifier < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'vulnerability_identifiers' - end - - def perform(*range) - feedback = VulnerabilityFeedback.without_uuid.in_range(*range).load_vulnerability_findings - feedback.each(&:set_finding_uuid) - - log_info(feedback.count) - end - - def log_info(feedback_count) - ::Gitlab::BackgroundMigration::Logger.info( - migrator: self.class.name, - message: '`finding_uuid` attributes has been set', - count: feedback_count - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb deleted file mode 100644 index 2b959b81f45..00000000000 --- a/lib/gitlab/background_migration/populate_issue_email_participants.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class to migrate service_desk_reply_to email addresses to issue_email_participants - class PopulateIssueEmailParticipants - # rubocop:disable Style/Documentation - class TmpIssue < ActiveRecord::Base - self.table_name = 'issues' - end - - def perform(start_id, stop_id) - issues = TmpIssue.select(:id, :service_desk_reply_to, :created_at).where(id: (start_id..stop_id)).where.not(service_desk_reply_to: nil) - - rows = issues.map do |issue| - { - issue_id: issue.id, - email: issue.service_desk_reply_to, - created_at: issue.created_at, - updated_at: issue.created_at - } - end - - ApplicationRecord.legacy_bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb new file mode 100644 index 00000000000..769ca4be7f3 --- /dev/null +++ b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to populates the non private projects counter of topics + class PopulateTopicsNonPrivateProjectsCount + SUB_BATCH_SIZE = 100 + + # Temporary AR model for topics + class Topic < ActiveRecord::Base + include EachBatch + + self.table_name = 'topics' + end + + def perform(start_id, stop_id) + Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| + ActiveRecord::Base.connection.execute(<<~SQL) + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) + UPDATE topics + SET non_private_projects_count = ( + SELECT COUNT(*) + FROM project_topics + INNER JOIN projects + ON project_topics.project_id = projects.id + WHERE project_topics.topic_id = batched_relation.id + AND projects.visibility_level > 0 + ) + FROM batched_relation + WHERE topics.id = batched_relation.id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_vulnerability_reads.rb b/lib/gitlab/background_migration/populate_vulnerability_reads.rb new file mode 100644 index 00000000000..7b6d4c1ff81 --- /dev/null +++ b/lib/gitlab/background_migration/populate_vulnerability_reads.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class PopulateVulnerabilityReads + include Gitlab::Database::DynamicModelHelpers + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + vulnerability_model.where(id: start_id..end_id).each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + connection.execute(insert_query(first, last)) + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + def vulnerability_model + define_batchable_model('vulnerabilities', connection: connection) + end + + def connection + ActiveRecord::Base.connection + end + + def insert_query(start_id, end_id) + <<~SQL + INSERT INTO vulnerability_reads ( + vulnerability_id, + project_id, + scanner_id, + report_type, + severity, + state, + has_issues, + resolved_on_default_branch, + uuid, + location_image + ) + SELECT + vulnerabilities.id, + vulnerabilities.project_id, + vulnerability_scanners.id, + vulnerabilities.report_type, + vulnerabilities.severity, + vulnerabilities.state, + CASE + WHEN + vulnerability_issue_links.vulnerability_id IS NOT NULL + THEN + true + ELSE + false + END + has_issues, + vulnerabilities.resolved_on_default_branch, + vulnerability_occurrences.uuid::uuid, + vulnerability_occurrences.location ->> 'image' + FROM + vulnerabilities + INNER JOIN vulnerability_occurrences ON vulnerability_occurrences.vulnerability_id = vulnerabilities.id + INNER JOIN vulnerability_scanners ON vulnerability_scanners.id = vulnerability_occurrences.scanner_id + LEFT JOIN vulnerability_issue_links ON vulnerability_issue_links.vulnerability_id = vulnerabilities.id + WHERE vulnerabilities.id BETWEEN #{start_id} AND #{end_id} + ON CONFLICT(vulnerability_id) DO NOTHING; + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + # rubocop:enable Style/Documentation + end +end diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb index 8e94c16369e..ba3f7c47047 100644 --- a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -5,19 +5,15 @@ module Gitlab module ProjectNamespaces # Back-fill project namespaces for projects that do not yet have a namespace. # - # TODO: remove this comment when an actuall backfill migration is added. - # - # This is first being added without an actual migration as we need to initially test - # if backfilling project namespaces affects performance in any significant way. # rubocop: disable Metrics/ClassLength class BackfillProjectNamespaces - BATCH_SIZE = 100 - DELETE_BATCH_SIZE = 10 + SUB_BATCH_SIZE = 25 PROJECT_NAMESPACE_STI_NAME = 'Project' IsolatedModels = ::Gitlab::BackgroundMigration::ProjectNamespaces::Models - def perform(start_id, end_id, namespace_id, migration_type = 'up') + def perform(start_id, end_id, migration_table_name, migration_column_name, sub_batch_size, pause_ms, namespace_id, migration_type = 'up') + @sub_batch_size = sub_batch_size || SUB_BATCH_SIZE load_project_ids(start_id, end_id, namespace_id) case migration_type @@ -34,10 +30,13 @@ module Gitlab private - attr_accessor :project_ids + attr_accessor :project_ids, :sub_batch_size def backfill_project_namespaces(namespace_id) - project_ids.each_slice(BATCH_SIZE) do |project_ids| + project_ids.each_slice(sub_batch_size) do |project_ids| + ActiveRecord::Base.connection.execute("select gin_clean_pending_list('index_namespaces_on_name_trigram')") + ActiveRecord::Base.connection.execute("select gin_clean_pending_list('index_namespaces_on_path_trigram')") + # We need to lock these project records for the period when we create project namespaces # and link them to projects so that if a project is modified in the time between creating # project namespaces `batch_insert_namespaces` and linking them to projects `batch_update_projects` @@ -45,18 +44,17 @@ module Gitlab # # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72527#note_730679469 Project.transaction do - Project.where(id: project_ids).select(:id).lock!('FOR UPDATE') + Project.where(id: project_ids).select(:id).lock!('FOR UPDATE').load batch_insert_namespaces(project_ids) batch_update_projects(project_ids) + batch_update_project_namespaces_traversal_ids(project_ids) end - - batch_update_project_namespaces_traversal_ids(project_ids) end end def cleanup_backfilled_project_namespaces(namespace_id) - project_ids.each_slice(BATCH_SIZE) do |project_ids| + project_ids.each_slice(sub_batch_size) do |project_ids| # IMPORTANT: first nullify project_namespace_id in projects table to avoid removing projects when records # from namespaces are deleted due to FK/triggers nullify_project_namespaces_in_projects(project_ids) @@ -109,7 +107,10 @@ module Gitlab end def delete_project_namespace_records(project_ids) - project_ids.each_slice(DELETE_BATCH_SIZE) do |p_ids| + # keep the deletes a 10x smaller batch as deletes seem to be much more expensive + delete_batch_size = (sub_batch_size / 10).to_i + 1 + + project_ids.each_slice(delete_batch_size) do |p_ids| IsolatedModels::Namespace.where(type: PROJECT_NAMESPACE_STI_NAME).where(tmp_project_id: p_ids).delete_all end end @@ -117,7 +118,7 @@ module Gitlab def load_project_ids(start_id, end_id, namespace_id) projects = IsolatedModels::Project.arel_table relation = IsolatedModels::Project.where(projects[:id].between(start_id..end_id)) - relation = relation.where(projects[:namespace_id].in(Arel::Nodes::SqlLiteral.new(hierarchy_cte(namespace_id)))) if namespace_id + relation = relation.where(projects[:namespace_id].in(Arel::Nodes::SqlLiteral.new(self.class.hierarchy_cte(namespace_id)))) if namespace_id @project_ids = relation.pluck(:id) end @@ -126,7 +127,7 @@ module Gitlab ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillProjectNamespaces', arguments) end - def hierarchy_cte(root_namespace_id) + def self.hierarchy_cte(root_namespace_id) <<-SQL WITH RECURSIVE "base_and_descendants" AS ( ( diff --git a/lib/gitlab/background_migration/recalculate_project_authorizations.rb b/lib/gitlab/background_migration/recalculate_project_authorizations.rb deleted file mode 100644 index 6a250a96c94..00000000000 --- a/lib/gitlab/background_migration/recalculate_project_authorizations.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class RecalculateProjectAuthorizations - def perform(user_ids) - # no-op - end - end - end -end diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb index 31fb5e97c5d..323f109449b 100644 --- a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb +++ b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb @@ -10,7 +10,9 @@ module Gitlab include Gitlab::Database::DynamicModelHelpers def perform(start_id, stop_id) - define_batchable_model('vulnerability_finding_links').where(id: start_id..stop_id).delete_all + define_batchable_model('vulnerability_finding_links', connection: ActiveRecord::Base.connection) + .where(id: start_id..stop_id) + .delete_all end end end diff --git a/lib/gitlab/background_migration/sync_blocking_issues_count.rb b/lib/gitlab/background_migration/sync_blocking_issues_count.rb deleted file mode 100644 index 49a632952fb..00000000000 --- a/lib/gitlab/background_migration/sync_blocking_issues_count.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class SyncBlockingIssuesCount - def perform(start_id, end_id) - end - end - end -end - -Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_mod_with('Gitlab::BackgroundMigration::SyncBlockingIssuesCount') diff --git a/lib/gitlab/background_migration/sync_issues_state_id.rb b/lib/gitlab/background_migration/sync_issues_state_id.rb deleted file mode 100644 index 2a0751928b8..00000000000 --- a/lib/gitlab/background_migration/sync_issues_state_id.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class SyncIssuesStateId - def perform(start_id, end_id) - ActiveRecord::Base.connection.execute <<~SQL - UPDATE issues - SET state_id = - CASE state - WHEN 'opened' THEN 1 - WHEN 'closed' THEN 2 - END - WHERE state_id IS NULL - AND id BETWEEN #{start_id} AND #{end_id} - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/sync_merge_requests_state_id.rb b/lib/gitlab/background_migration/sync_merge_requests_state_id.rb deleted file mode 100644 index 6707e178d8b..00000000000 --- a/lib/gitlab/background_migration/sync_merge_requests_state_id.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class SyncMergeRequestsStateId - def perform(start_id, end_id) - ActiveRecord::Base.connection.execute <<~SQL - UPDATE merge_requests - SET state_id = - CASE state - WHEN 'opened' THEN 1 - WHEN 'closed' THEN 2 - WHEN 'merged' THEN 3 - WHEN 'locked' THEN 4 - END - WHERE state_id IS NULL - AND id BETWEEN #{start_id} AND #{end_id} - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb index c95ef9f5515..f54bb8256d0 100644 --- a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb +++ b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb @@ -9,7 +9,9 @@ module Gitlab BATCH_SIZE = 100 def perform(start_id, stop_id) - define_batchable_model('timelogs').where(spent_at: nil, id: start_id..stop_id).each_batch(of: 100) do |subbatch| + define_batchable_model('timelogs', connection: connection) + .where(spent_at: nil, id: start_id..stop_id) + .each_batch(of: 100) do |subbatch| batch_start, batch_end = subbatch.pluck('min(id), max(id)').first update_timelogs(batch_start, batch_end) @@ -25,9 +27,12 @@ module Gitlab SQL end - def execute(sql) + def connection @connection ||= ::ActiveRecord::Base.connection - @connection.execute(sql) + end + + def execute(sql) + connection.execute(sql) end end end diff --git a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb deleted file mode 100644 index 665ad7abcbb..00000000000 --- a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class WrongfullyConfirmedEmailUnconfirmer - class UserModel < ActiveRecord::Base - alias_method :reset, :reload - - self.table_name = 'users' - - scope :active, -> { where(state: 'active', user_type: nil) } # only humans, skip bots - - devise :confirmable - end - - class EmailModel < ActiveRecord::Base - alias_method :reset, :reload - - self.table_name = 'emails' - - belongs_to :user - - devise :confirmable - - def self.wrongfully_confirmed_emails(start_id, stop_id) - joins(:user) - .merge(UserModel.active) - .where(id: (start_id..stop_id)) - .where.not('emails.confirmed_at' => nil) - .where('emails.confirmed_at = users.confirmed_at') - .where('emails.email <> users.email') - .where('NOT EXISTS (SELECT 1 FROM user_synced_attributes_metadata WHERE user_id=users.id AND email_synced IS true)') - end - end - - def perform(start_id, stop_id) - email_records = EmailModel - .wrongfully_confirmed_emails(start_id, stop_id) - .to_a - - user_ids = email_records.map(&:user_id).uniq - - ActiveRecord::Base.transaction do - update_email_records(start_id, stop_id) - update_user_records(user_ids) - end - - # Refind the records with the "real" Email model so devise will notice that the user / email is unconfirmed - unconfirmed_email_records = ::Email.where(id: email_records.map(&:id)) - ActiveRecord::Associations::Preloader.new.preload(unconfirmed_email_records, [:user]) - - send_emails(unconfirmed_email_records) - end - - private - - def update_email_records(start_id, stop_id) - EmailModel.connection.execute <<-SQL - WITH md5_strings as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - #{email_query_for_update(start_id, stop_id).to_sql} - ) - UPDATE #{EmailModel.connection.quote_table_name(EmailModel.table_name)} - SET confirmed_at = NULL, - confirmation_token = md5_strings.md5_string, - confirmation_sent_at = NOW() - FROM md5_strings - WHERE id = md5_strings.email_id - SQL - end - - def update_user_records(user_ids) - UserModel - .where(id: user_ids) - .update_all("confirmed_at = NULL, confirmation_sent_at = NOW(), unconfirmed_email = NULL, confirmation_token=md5(users.id::varchar || users.created_at || users.encrypted_password || '#{Integer(Time.now.to_i)}')") - end - - def email_query_for_update(start_id, stop_id) - EmailModel - .wrongfully_confirmed_emails(start_id, stop_id) - .select('emails.id as email_id', "md5(emails.id::varchar || emails.created_at || users.encrypted_password || '#{Integer(Time.now.to_i)}') as md5_string") - end - - def send_emails(email_records) - user_records = email_records.map(&:user).uniq - - user_records.each do |user| - Gitlab::BackgroundMigration::Mailers::UnconfirmMailer.unconfirm_notification_email(user).deliver_later - DeviseMailer.confirmation_instructions(user, user.confirmation_token).deliver_later(wait: 1.minute) - end - - email_records.each do |email| - DeviseMailer.confirmation_instructions(email, email.confirmation_token).deliver_later(wait: 1.minute) - end - end - end - end -end |