Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-06-15 09:10:13 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-06-15 09:10:13 +0300
commitb4610c51192adc832d25192038a781d2aa1ebb98 (patch)
tree860bb889ee850e13a1f947489d768801556f9213 /lib
parenta7b6d465ae75b497ac34cb43361edd0c8ce45fbd (diff)
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r--lib/api/validations/validators/absence.rb2
-rw-r--r--lib/api/validations/validators/array_none_any.rb2
-rw-r--r--lib/api/validations/validators/bulk_imports.rb6
-rw-r--r--lib/api/validations/validators/check_assignees_count.rb2
-rw-r--r--lib/api/validations/validators/email_or_email_list.rb2
-rw-r--r--lib/api/validations/validators/file_path.rb2
-rw-r--r--lib/api/validations/validators/git_ref.rb2
-rw-r--r--lib/api/validations/validators/git_sha.rb2
-rw-r--r--lib/api/validations/validators/integer_or_custom_value.rb2
-rw-r--r--lib/api/validations/validators/limit.rb2
-rw-r--r--lib/api/validations/validators/project_portable.rb2
-rw-r--r--lib/api/validations/validators/untrusted_regexp.rb2
-rw-r--r--lib/banzai/filter/references/user_reference_filter.rb2
-rw-r--r--lib/gitlab/background_migration/backfill_ci_queuing_tables.rb153
-rw-r--r--lib/gitlab/background_migration/backfill_group_features.rb35
-rw-r--r--lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb38
-rw-r--r--lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb48
-rw-r--r--lib/gitlab/background_migration/encrypt_integration_properties.rb84
-rw-r--r--lib/gitlab/background_migration/encrypt_static_object_token.rb70
-rw-r--r--lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb82
-rw-r--r--lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb13
-rw-r--r--lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb124
-rw-r--r--lib/gitlab/background_migration/merge_topics_with_same_name.rb76
-rw-r--r--lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb45
-rw-r--r--lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb27
-rw-r--r--lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb44
-rw-r--r--lib/gitlab/background_migration/populate_container_repository_migration_plan.rb51
-rw-r--r--lib/gitlab/background_migration/populate_namespace_statistics.rb47
-rw-r--r--lib/gitlab/background_migration/populate_test_reports_issue_id.rb14
-rw-r--r--lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb36
-rw-r--r--lib/gitlab/background_migration/populate_vulnerability_reads.rb84
-rw-r--r--lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb218
-rw-r--r--lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb13
-rw-r--r--lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb53
-rw-r--r--lib/gitlab/background_migration/remove_vulnerability_finding_links.rb19
-rw-r--r--lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb40
-rw-r--r--lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb40
-rw-r--r--lib/gitlab/background_migration/update_timelogs_null_spent_at.rb39
38 files changed, 15 insertions, 1508 deletions
diff --git a/lib/api/validations/validators/absence.rb b/lib/api/validations/validators/absence.rb
index 7858ce7140b..49bf7f4a1c2 100644
--- a/lib/api/validations/validators/absence.rb
+++ b/lib/api/validations/validators/absence.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class Absence < Grape::Validations::Base
+ class Absence < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
return if params.respond_to?(:key?) && !params.key?(attr_name)
diff --git a/lib/api/validations/validators/array_none_any.rb b/lib/api/validations/validators/array_none_any.rb
index 8c064eefbf2..cc86bd1a535 100644
--- a/lib/api/validations/validators/array_none_any.rb
+++ b/lib/api/validations/validators/array_none_any.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class ArrayNoneAny < Grape::Validations::Base
+ class ArrayNoneAny < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
value = params[attr_name]
diff --git a/lib/api/validations/validators/bulk_imports.rb b/lib/api/validations/validators/bulk_imports.rb
index bff3424a0ac..f8ad5ed6d14 100644
--- a/lib/api/validations/validators/bulk_imports.rb
+++ b/lib/api/validations/validators/bulk_imports.rb
@@ -4,7 +4,7 @@ module API
module Validations
module Validators
module BulkImports
- class DestinationSlugPath < Grape::Validations::Base
+ class DestinationSlugPath < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
if Feature.disabled?(:restrict_special_characters_in_namespace_path)
return if params[attr_name] =~ Gitlab::Regex.group_path_regex
@@ -29,7 +29,7 @@ module API
end
end
- class DestinationNamespacePath < Grape::Validations::Base
+ class DestinationNamespacePath < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
return if params[attr_name].blank?
@@ -42,7 +42,7 @@ module API
end
end
- class SourceFullPath < Grape::Validations::Base
+ class SourceFullPath < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
return if params[attr_name] =~ Gitlab::Regex.bulk_import_source_full_path_regex
diff --git a/lib/api/validations/validators/check_assignees_count.rb b/lib/api/validations/validators/check_assignees_count.rb
index 15f48c09a4f..7d90f030f1a 100644
--- a/lib/api/validations/validators/check_assignees_count.rb
+++ b/lib/api/validations/validators/check_assignees_count.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class CheckAssigneesCount < Grape::Validations::Base
+ class CheckAssigneesCount < Grape::Validations::Validators::Base
def self.coerce
lambda do |value|
case value
diff --git a/lib/api/validations/validators/email_or_email_list.rb b/lib/api/validations/validators/email_or_email_list.rb
index 715a29c613d..8aa7c3bc32c 100644
--- a/lib/api/validations/validators/email_or_email_list.rb
+++ b/lib/api/validations/validators/email_or_email_list.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class EmailOrEmailList < Grape::Validations::Base
+ class EmailOrEmailList < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
value = params[attr_name]
diff --git a/lib/api/validations/validators/file_path.rb b/lib/api/validations/validators/file_path.rb
index 2440d8890e2..7b32e1d71f8 100644
--- a/lib/api/validations/validators/file_path.rb
+++ b/lib/api/validations/validators/file_path.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class FilePath < Grape::Validations::Base
+ class FilePath < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
options = @option.is_a?(Hash) ? @option : {}
path_allowlist = options.fetch(:allowlist, [])
diff --git a/lib/api/validations/validators/git_ref.rb b/lib/api/validations/validators/git_ref.rb
index dcb1db6ca33..711c272ab4e 100644
--- a/lib/api/validations/validators/git_ref.rb
+++ b/lib/api/validations/validators/git_ref.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class GitRef < Grape::Validations::Base
+ class GitRef < Grape::Validations::Validators::Base
# There are few checks that a Git reference should pass through to be valid reference.
# The link contains some rules that have been added to this validator.
# https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
diff --git a/lib/api/validations/validators/git_sha.rb b/lib/api/validations/validators/git_sha.rb
index 665d1878b4c..830137a0197 100644
--- a/lib/api/validations/validators/git_sha.rb
+++ b/lib/api/validations/validators/git_sha.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class GitSha < Grape::Validations::Base
+ class GitSha < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
sha = params[attr_name]
diff --git a/lib/api/validations/validators/integer_or_custom_value.rb b/lib/api/validations/validators/integer_or_custom_value.rb
index ad7848583fc..ff2df858a36 100644
--- a/lib/api/validations/validators/integer_or_custom_value.rb
+++ b/lib/api/validations/validators/integer_or_custom_value.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class IntegerOrCustomValue < Grape::Validations::Base
+ class IntegerOrCustomValue < Grape::Validations::Validators::Base
def initialize(attrs, options, required, scope, **opts)
@custom_values = extract_custom_values(options)
super
diff --git a/lib/api/validations/validators/limit.rb b/lib/api/validations/validators/limit.rb
index 7e11f1d77cc..781bebac716 100644
--- a/lib/api/validations/validators/limit.rb
+++ b/lib/api/validations/validators/limit.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class Limit < Grape::Validations::Base
+ class Limit < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
value = params[attr_name]
diff --git a/lib/api/validations/validators/project_portable.rb b/lib/api/validations/validators/project_portable.rb
index 3a7ea5ea71e..50ab32b6b7a 100644
--- a/lib/api/validations/validators/project_portable.rb
+++ b/lib/api/validations/validators/project_portable.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class ProjectPortable < Grape::Validations::Base
+ class ProjectPortable < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
portable = params[attr_name]
diff --git a/lib/api/validations/validators/untrusted_regexp.rb b/lib/api/validations/validators/untrusted_regexp.rb
index 3ddea2bd9de..c5560be2e16 100644
--- a/lib/api/validations/validators/untrusted_regexp.rb
+++ b/lib/api/validations/validators/untrusted_regexp.rb
@@ -3,7 +3,7 @@
module API
module Validations
module Validators
- class UntrustedRegexp < Grape::Validations::Base
+ class UntrustedRegexp < Grape::Validations::Validators::Base
def validate_param!(attr_name, params)
value = params[attr_name]
return unless value
diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb
index 5983036a8e5..d6b6fdb7149 100644
--- a/lib/banzai/filter/references/user_reference_filter.rb
+++ b/lib/banzai/filter/references/user_reference_filter.rb
@@ -45,7 +45,7 @@ module Banzai
# have `gfm` and `gfm-project_member` class names attached for styling.
def object_link_filter(text, pattern, link_content: nil, link_reference: false)
references_in(text, pattern) do |match, username|
- if username == 'all' && !skip_project_check?
+ if Feature.disabled?(:disable_all_mention) && username == 'all' && !skip_project_check?
link_to_all(link_content: link_content)
else
cached_call(:banzai_url_for_object, match, path: [User, username.downcase]) do
diff --git a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb b/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb
deleted file mode 100644
index 63112b52584..00000000000
--- a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb
+++ /dev/null
@@ -1,153 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Ensure queuing entries are present even if admins skip upgrades.
- class BackfillCiQueuingTables
- class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation
- self.table_name = 'namespaces'
- self.inheritance_column = :_type_disabled
- end
-
- class Project < ActiveRecord::Base # rubocop:disable Style/Documentation
- self.table_name = 'projects'
-
- belongs_to :namespace
- has_one :ci_cd_settings, class_name: 'Gitlab::BackgroundMigration::BackfillCiQueuingTables::ProjectCiCdSetting'
-
- def group_runners_enabled?
- return false unless ci_cd_settings
-
- ci_cd_settings.group_runners_enabled?
- end
- end
-
- class ProjectCiCdSetting < ActiveRecord::Base # rubocop:disable Style/Documentation
- self.table_name = 'project_ci_cd_settings'
- end
-
- class Taggings < ActiveRecord::Base # rubocop:disable Style/Documentation
- self.table_name = 'taggings'
- end
-
- module Ci
- class Build < ActiveRecord::Base # rubocop:disable Style/Documentation
- include EachBatch
-
- self.table_name = 'ci_builds'
- self.inheritance_column = :_type_disabled
-
- belongs_to :project
-
- scope :pending, -> do
- where(status: :pending, type: 'Ci::Build', runner_id: nil)
- end
-
- def self.each_batch(of: 1000, column: :id, order: { runner_id: :asc, id: :asc }, order_hint: nil)
- start = except(:select).select(column).reorder(order)
- start = start.take
- return unless start
-
- start_id = start[column]
- arel_table = self.arel_table
-
- 1.step do |index|
- start_cond = arel_table[column].gteq(start_id)
- stop = except(:select).select(column).where(start_cond).reorder(order)
- stop = stop.offset(of).limit(1).take
- relation = where(start_cond)
-
- if stop
- stop_id = stop[column]
- start_id = stop_id
- stop_cond = arel_table[column].lt(stop_id)
- relation = relation.where(stop_cond)
- end
-
- # Any ORDER BYs are useless for this relation and can lead to less
- # efficient UPDATE queries, hence we get rid of it.
- relation = relation.except(:order)
-
- # Using unscoped is necessary to prevent leaking the current scope used by
- # ActiveRecord to chain `each_batch` method.
- unscoped { yield relation, index }
-
- break unless stop
- end
- end
-
- def tags_ids
- BackfillCiQueuingTables::Taggings
- .where(taggable_id: id, taggable_type: 'CommitStatus')
- .pluck(:tag_id)
- end
- end
-
- class PendingBuild < ActiveRecord::Base # rubocop:disable Style/Documentation
- self.table_name = 'ci_pending_builds'
-
- class << self
- def upsert_from_build!(build)
- entry = self.new(args_from_build(build))
-
- self.upsert(
- entry.attributes.compact,
- returning: %w[build_id],
- unique_by: :build_id)
- end
-
- def args_from_build(build)
- project = build.project
-
- {
- build_id: build.id,
- project_id: build.project_id,
- protected: build.protected?,
- namespace_id: project.namespace_id,
- tag_ids: build.tags_ids,
- instance_runners_enabled: project.shared_runners_enabled?,
- namespace_traversal_ids: namespace_traversal_ids(project)
- }
- end
-
- def namespace_traversal_ids(project)
- if project.group_runners_enabled?
- project.namespace.traversal_ids
- else
- []
- end
- end
- end
- end
- end
-
- BATCH_SIZE = 100
-
- def perform(start_id, end_id)
- scope = BackfillCiQueuingTables::Ci::Build.pending.where(id: start_id..end_id)
- pending_builds_query = BackfillCiQueuingTables::Ci::PendingBuild
- .where('ci_builds.id = ci_pending_builds.build_id')
- .select(1)
-
- scope.each_batch(of: BATCH_SIZE) do |builds|
- builds = builds.where('NOT EXISTS (?)', pending_builds_query)
- builds = builds.includes(:project, project: [:namespace, :ci_cd_settings])
-
- builds.each do |build|
- BackfillCiQueuingTables::Ci::PendingBuild.upsert_from_build!(build)
- end
- end
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb
deleted file mode 100644
index c45dcad5b2d..00000000000
--- a/lib/gitlab/background_migration/backfill_group_features.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Backfill group_features for an array of groups
- class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob
- job_arguments :batch_size
- operation_name :upsert_group_features
- feature_category :database
-
- def perform
- each_sub_batch(
- batching_arguments: { order_hint: :type },
- batching_scope: ->(relation) { relation.where(type: 'Group') }
- ) do |sub_batch|
- upsert_group_features(sub_batch)
- end
- end
-
- private
-
- def upsert_group_features(relation)
- connection.execute(
- <<~SQL
- INSERT INTO group_features (group_id, created_at, updated_at)
- SELECT namespaces.id as group_id, now(), now()
- FROM namespaces
- WHERE namespaces.type = 'Group' AND namespaces.id IN(#{relation.select(:id).limit(batch_size).to_sql})
- ON CONFLICT (group_id) DO NOTHING;
- SQL
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb
deleted file mode 100644
index 0585924cb7b..00000000000
--- a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Backfills the `routes.namespace_id` column, by copying source_id value
- # (for groups and user namespaces source_id == namespace_id)
- class BackfillNamespaceIdForNamespaceRoute
- include Gitlab::Database::DynamicModelHelpers
-
- def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
- parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
-
- parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
- batch_metrics.time_operation(:update_all) do
- sub_batch.update_all('namespace_id=source_id')
- end
-
- pause_ms = [0, pause_ms].max
- sleep(pause_ms * 0.001)
- end
- end
-
- def batch_metrics
- @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
- end
-
- private
-
- def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
- define_batchable_model(source_table, connection: ApplicationRecord.connection)
- .joins('inner join namespaces on routes.source_id = namespaces.id')
- .where(source_key_column => start_id..stop_id)
- .where(namespace_id: nil)
- .where(source_type: 'Namespace')
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb b/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb
deleted file mode 100644
index b703faf6a6c..00000000000
--- a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Cleanup draft column data inserted by a faulty regex
- #
- class CleanupDraftDataFromFaultyRegex
- # Migration only version of MergeRequest table
- ##
- class MergeRequest < ActiveRecord::Base
- LEAKY_REGEXP_STR = "^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP"
- CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)"
-
- include EachBatch
-
- self.table_name = 'merge_requests'
-
- def self.eligible
- where(state_id: 1)
- .where(draft: true)
- .where("title ~* ?", LEAKY_REGEXP_STR)
- .where("title !~* ?", CORRECTED_REGEXP_STR)
- end
- end
-
- def perform(start_id, end_id)
- eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id)
-
- return if eligible_mrs.empty?
-
- eligible_mrs.each_slice(10) do |slice|
- MergeRequest.where(id: slice).update_all(draft: false)
- end
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- 'CleanupDraftDataFromFaultyRegex',
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb
deleted file mode 100644
index 28c28ae48eb..00000000000
--- a/lib/gitlab/background_migration/encrypt_integration_properties.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Migrates the integration.properties column from plaintext to encrypted text.
- class EncryptIntegrationProperties
- # The Integration model, with just the relevant bits.
- class Integration < ActiveRecord::Base
- include EachBatch
-
- ALGORITHM = 'aes-256-gcm'
-
- self.table_name = 'integrations'
- self.inheritance_column = :_type_disabled
-
- scope :with_properties, -> { where.not(properties: nil) }
- scope :not_already_encrypted, -> { where(encrypted_properties: nil) }
- scope :for_batch, ->(range) { where(id: range) }
-
- attr_encrypted :encrypted_properties_tmp,
- attribute: :encrypted_properties,
- mode: :per_attribute_iv,
- key: ::Settings.attr_encrypted_db_key_base_32,
- algorithm: ALGORITHM,
- marshal: true,
- marshaler: ::Gitlab::Json,
- encode: false,
- encode_iv: false
-
- # See 'Integration#reencrypt_properties'
- def encrypt_properties
- data = ::Gitlab::Json.parse(properties)
- iv = generate_iv(ALGORITHM)
- ep = self.class.attr_encrypt(:encrypted_properties_tmp, data, { iv: iv })
-
- [ep, iv]
- end
- end
-
- def perform(start_id, stop_id)
- batch_query = Integration.with_properties.not_already_encrypted.for_batch(start_id..stop_id)
- encrypt_batch(batch_query)
- mark_job_as_succeeded(start_id, stop_id)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
-
- # represent binary string as a PSQL binary literal:
- # https://www.postgresql.org/docs/9.4/datatype-binary.html
- def bytea(value)
- "'\\x#{value.unpack1('H*')}'::bytea"
- end
-
- def encrypt_batch(batch_query)
- values = batch_query.select(:id, :properties).map do |record|
- encrypted_properties, encrypted_properties_iv = record.encrypt_properties
- "(#{record.id}, #{bytea(encrypted_properties)}, #{bytea(encrypted_properties_iv)})"
- end
-
- return if values.empty?
-
- Integration.connection.execute(<<~SQL.squish)
- WITH cte(cte_id, cte_encrypted_properties, cte_encrypted_properties_iv)
- AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
- SELECT *
- FROM (VALUES #{values.join(',')}) AS t (id, encrypted_properties, encrypted_properties_iv)
- )
- UPDATE #{Integration.table_name}
- SET encrypted_properties = cte_encrypted_properties
- , encrypted_properties_iv = cte_encrypted_properties_iv
- FROM cte
- WHERE cte_id = id
- SQL
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb
deleted file mode 100644
index 961dea028c9..00000000000
--- a/lib/gitlab/background_migration/encrypt_static_object_token.rb
+++ /dev/null
@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Populates "static_object_token_encrypted" field with encrypted versions
- # of values from "static_object_token" field
- class EncryptStaticObjectToken
- # rubocop:disable Style/Documentation
- class User < ActiveRecord::Base
- include ::EachBatch
- self.table_name = 'users'
- scope :with_static_object_token, -> { where.not(static_object_token: nil) }
- scope :without_static_object_token_encrypted, -> { where(static_object_token_encrypted: nil) }
- end
- # rubocop:enable Style/Documentation
-
- BATCH_SIZE = 100
-
- def perform(start_id, end_id)
- ranged_query = User
- .where(id: start_id..end_id)
- .with_static_object_token
- .without_static_object_token_encrypted
-
- ranged_query.each_batch(of: BATCH_SIZE) do |sub_batch|
- first, last = sub_batch.pick(Arel.sql('min(id), max(id)'))
-
- batch_query = User.unscoped
- .where(id: first..last)
- .with_static_object_token
- .without_static_object_token_encrypted
-
- user_tokens = batch_query.pluck(:id, :static_object_token)
-
- user_encrypted_tokens = user_tokens.map do |(id, plaintext_token)|
- next if plaintext_token.blank?
-
- [id, Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token)]
- end
-
- encrypted_tokens_sql = user_encrypted_tokens.compact.map { |(id, token)| "(#{id}, '#{token}')" }.join(',')
-
- next unless user_encrypted_tokens.present?
-
- User.connection.execute(<<~SQL)
- WITH cte(cte_id, cte_token) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
- SELECT *
- FROM (VALUES #{encrypted_tokens_sql}) AS t (id, token)
- )
- UPDATE #{User.table_name}
- SET static_object_token_encrypted = cte_token
- FROM cte
- WHERE cte_id = id
- SQL
- end
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb
deleted file mode 100644
index 3772430d0b7..00000000000
--- a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb
+++ /dev/null
@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Fix project name duplicates and backfill missing project namespace ids
- class FixDuplicateProjectNameAndPath
- SUB_BATCH_SIZE = 10
- # isolated project active record
- class Project < ActiveRecord::Base
- include ::EachBatch
-
- self.table_name = 'projects'
-
- scope :without_project_namespace, -> { where(project_namespace_id: nil) }
- scope :id_in, ->(ids) { where(id: ids) }
- end
-
- def perform(start_id, end_id)
- @project_ids = fetch_project_ids(start_id, end_id)
- backfill_project_namespaces_service = init_backfill_service(project_ids)
- backfill_project_namespaces_service.cleanup_gin_index('projects')
-
- project_ids.each_slice(SUB_BATCH_SIZE) do |ids|
- ApplicationRecord.connection.execute(update_projects_name_and_path_sql(ids))
- end
-
- backfill_project_namespaces_service.backfill_project_namespaces
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- private
-
- attr_accessor :project_ids
-
- def fetch_project_ids(start_id, end_id)
- Project.without_project_namespace.where(id: start_id..end_id)
- end
-
- def init_backfill_service(project_ids)
- service = Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces.new
- service.project_ids = project_ids
- service.sub_batch_size = SUB_BATCH_SIZE
-
- service
- end
-
- def update_projects_name_and_path_sql(project_ids)
- <<~SQL
- WITH cte (project_id, path_from_route ) AS (
- #{path_from_route_sql(project_ids).to_sql}
- )
- UPDATE
- projects
- SET
- name = concat(projects.name, '-', id),
- path = CASE
- WHEN projects.path <> cte.path_from_route THEN path_from_route
- ELSE projects.path
- END
- FROM
- cte
- WHERE
- projects.id = cte.project_id;
- SQL
- end
-
- def path_from_route_sql(project_ids)
- Project.without_project_namespace.id_in(project_ids)
- .joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'")
- .select("projects.id, SUBSTRING(routes.path FROM '[^/]+(?=/$|$)') AS path_from_route")
- end
-
- def mark_job_as_succeeded(*arguments)
- ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- 'FixDuplicateProjectNameAndPath',
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb b/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb
deleted file mode 100644
index 2c09b8c0b24..00000000000
--- a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # rubocop: disable Style/Documentation
- class FixIncorrectMaxSeatsUsed
- def perform(batch = nil)
- end
- end
- end
-end
-
-Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed.prepend_mod_with('Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed')
diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb
deleted file mode 100644
index db3f98bc2ba..00000000000
--- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb
+++ /dev/null
@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-require 'parser/ruby27'
-
-module Gitlab
- module BackgroundMigration
- # This migration fixes raw_metadata entries which have incorrectly been passed a Ruby Hash instead of JSON data.
- class FixVulnerabilityOccurrencesWithHashesAsRawMetadata
- CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7
- GENERIC_REPORT_TYPE = 99
-
- # Type error is used to handle unexpected types when parsing stringified hashes.
- class TypeError < ::StandardError
- attr_reader :message, :type
-
- def initialize(message, type)
- @message = message
- @type = type
- end
- end
-
- # Migration model namespace isolated from application code.
- class Finding < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'vulnerability_occurrences'
-
- scope :by_api_report_types, -> { where(report_type: [CLUSTER_IMAGE_SCANNING_REPORT_TYPE, GENERIC_REPORT_TYPE]) }
- end
-
- def perform(start_id, end_id)
- Finding.by_api_report_types.where(id: start_id..end_id).each do |finding|
- next if valid_json?(finding.raw_metadata)
-
- metadata = hash_from_s(finding.raw_metadata)
-
- finding.update(raw_metadata: metadata.to_json) if metadata
- end
- mark_job_as_succeeded(start_id, end_id)
- end
-
- def hash_from_s(str_hash)
- ast = Parser::Ruby27.parse(str_hash)
-
- unless ast.type == :hash
- ::Gitlab::AppLogger.error(message: "expected raw_metadata to be a hash", type: ast.type)
- return
- end
-
- parse_hash(ast)
- rescue Parser::SyntaxError => e
- ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message)
- nil
- rescue TypeError => e
- ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message, type: e.type)
- nil
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- 'FixVulnerabilityOccurrencesWithHashesAsRawMetadata',
- arguments
- )
- end
-
- def valid_json?(metadata)
- Oj.load(metadata)
- true
- rescue Oj::ParseError, EncodingError, JSON::ParserError, JSON::GeneratorError, Encoding::UndefinedConversionError
- false
- end
-
- def parse_hash(hash)
- out = {}
- hash.children.each do |node|
- unless node.type == :pair
- raise TypeError.new("expected child of hash to be a `pair`", node.type)
- end
-
- key, value = node.children
-
- key = parse_key(key)
- value = parse_value(value)
-
- out[key] = value
- end
-
- out
- end
-
- def parse_key(key)
- case key.type
- when :sym, :str, :int
- key.children.first
- else
- raise TypeError.new("expected key to be either symbol, string, or integer", key.type)
- end
- end
-
- def parse_value(value)
- case value.type
- when :sym, :str, :int
- value.children.first
- # rubocop:disable Lint/BooleanSymbol
- when :true
- true
- when :false
- false
- # rubocop:enable Lint/BooleanSymbol
- when :nil
- nil
- when :array
- value.children.map { |c| parse_value(c) }
- when :hash
- parse_hash(value)
- else
- raise TypeError.new("value of a pair was an unexpected type", value.type)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/merge_topics_with_same_name.rb b/lib/gitlab/background_migration/merge_topics_with_same_name.rb
deleted file mode 100644
index 07231098a5f..00000000000
--- a/lib/gitlab/background_migration/merge_topics_with_same_name.rb
+++ /dev/null
@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # The class to merge project topics with the same case insensitive name
- class MergeTopicsWithSameName
- # Temporary AR model for topics
- class Topic < ActiveRecord::Base
- self.table_name = 'topics'
- end
-
- # Temporary AR model for project topic assignment
- class ProjectTopic < ActiveRecord::Base
- self.table_name = 'project_topics'
- end
-
- def perform(topic_names)
- topic_names.each do |topic_name|
- topics = Topic.where('LOWER(name) = ?', topic_name)
- .order(total_projects_count: :desc, non_private_projects_count: :desc, id: :asc)
- .to_a
- topic_to_keep = topics.shift
- merge_topics(topic_to_keep, topics) if topics.any?
- end
- end
-
- private
-
- def merge_topics(topic_to_keep, topics_to_remove)
- description = topic_to_keep.description
-
- topics_to_remove.each do |topic|
- description ||= topic.description if topic.description.present?
- process_avatar(topic_to_keep, topic) if topic.avatar.present?
-
- ProjectTopic.transaction do
- ProjectTopic.where(topic_id: topic.id)
- .where.not(project_id: ProjectTopic.where(topic_id: topic_to_keep).select(:project_id))
- .update_all(topic_id: topic_to_keep.id)
- ProjectTopic.where(topic_id: topic.id).delete_all
- end
- end
-
- Topic.where(id: topics_to_remove).delete_all
-
- topic_to_keep.update(
- description: description,
- total_projects_count: total_projects_count(topic_to_keep.id),
- non_private_projects_count: non_private_projects_count(topic_to_keep.id)
- )
- end
-
- # We intentionally use application code here because we need to copy/remove avatar files
- def process_avatar(topic_to_keep, topic_to_remove)
- topic_to_remove = ::Projects::Topic.find(topic_to_remove.id)
- topic_to_keep = ::Projects::Topic.find(topic_to_keep.id)
- unless topic_to_keep.avatar.present?
- topic_to_keep.avatar = topic_to_remove.avatar
- topic_to_keep.save!
- end
-
- topic_to_remove.remove_avatar!
- topic_to_remove.save!
- end
-
- def total_projects_count(topic_id)
- ProjectTopic.where(topic_id: topic_id).count
- end
-
- def non_private_projects_count(topic_id)
- ProjectTopic.joins('INNER JOIN projects ON project_topics.project_id = projects.id')
- .where(project_topics: { topic_id: topic_id }).where('projects.visibility_level in (10, 20)').count
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb b/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb
deleted file mode 100644
index 49eff6e2771..00000000000
--- a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Migrates personal namespace project `maintainer` memberships (for the associated user only) to OWNER
- # Does not create any missing records, simply migrates existing ones
- class MigratePersonalNamespaceProjectMaintainerToOwner
- include Gitlab::Database::DynamicModelHelpers
-
- def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
- parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
-
- parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
- batch_metrics.time_operation(:update_all) do
- sub_batch.update_all('access_level = 50')
- end
-
- pause_ms = 0 if pause_ms < 0
- sleep(pause_ms * 0.001)
- end
- end
-
- def batch_metrics
- @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
- end
-
- private
-
- def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
- # members of projects within their own personal namespace
-
- # rubocop: disable CodeReuse/ActiveRecord
- define_batchable_model(:members, connection: ApplicationRecord.connection)
- .where(source_key_column => start_id..stop_id)
- .joins("INNER JOIN projects ON members.source_id = projects.id")
- .joins("INNER JOIN namespaces ON projects.namespace_id = namespaces.id")
- .where(type: 'ProjectMember')
- .where("namespaces.type = 'User'")
- .where('members.access_level < 50')
- .where('namespaces.owner_id = members.user_id')
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
- end
-end
diff --git a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb
deleted file mode 100644
index d7d24960a41..00000000000
--- a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb
+++ /dev/null
@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # The class to migrate category of integrations to third_party_wiki for confluence and shimo
- class MigrateShimoConfluenceIntegrationCategory
- include Gitlab::Database::DynamicModelHelpers
-
- def perform(start_id, end_id)
- define_batchable_model('integrations', connection: ApplicationRecord.connection)
- .where(id: start_id..end_id, type_new: %w[Integrations::Confluence Integrations::Shimo])
- .update_all(category: 'third_party_wiki')
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb
deleted file mode 100644
index 13b66b2e02e..00000000000
--- a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # A job to nullify orphan runner_id on ci_builds table
- class NullifyOrphanRunnerIdOnCiBuilds
- include Gitlab::Database::DynamicModelHelpers
-
- def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
- pause_ms = 0 if pause_ms < 0
-
- batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
- batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
- batch_metrics.time_operation(:update_all) do
- filtered_sub_batch(sub_batch).update_all(runner_id: nil)
- end
-
- sleep(pause_ms * 0.001)
- end
- end
-
- def batch_metrics
- @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
- end
-
- private
-
- def connection
- ::Ci::ApplicationRecord.connection
- end
-
- def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
- define_batchable_model(source_table, connection: connection)
- .where(source_key_column => start_id..stop_id)
- end
-
- def filtered_sub_batch(sub_batch)
- sub_batch
- .joins('LEFT OUTER JOIN ci_runners ON ci_runners.id = ci_builds.runner_id')
- .where('ci_builds.runner_id IS NOT NULL AND ci_runners.id IS NULL')
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb
deleted file mode 100644
index a9611e9814c..00000000000
--- a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # The class to populates the migration_plan column of container_repositories
- # with the current plan of the namespaces that owns the container_repository
- #
- # The plan can be NULL, in which case no UPDATE
- # will be executed.
- class PopulateContainerRepositoryMigrationPlan
- def perform(start_id, end_id)
- (start_id..end_id).each do |id|
- execute(<<~SQL)
- WITH selected_plan AS (
- SELECT "plans"."name"
- FROM "container_repositories"
- INNER JOIN "projects" ON "projects"."id" = "container_repositories"."project_id"
- INNER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id"
- INNER JOIN "gitlab_subscriptions" ON "gitlab_subscriptions"."namespace_id" = "namespaces"."traversal_ids"[1]
- INNER JOIN "plans" ON "plans"."id" = "gitlab_subscriptions"."hosted_plan_id"
- WHERE "container_repositories"."id" = #{id}
- )
- UPDATE container_repositories
- SET migration_plan = selected_plan.name
- FROM selected_plan
- WHERE container_repositories.id = #{id};
- SQL
- end
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- private
-
- def connection
- @connection ||= ApplicationRecord.connection
- end
-
- def execute(sql)
- connection.execute(sql)
- end
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb
deleted file mode 100644
index 97927ef48c2..00000000000
--- a/lib/gitlab/background_migration/populate_namespace_statistics.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # This class creates/updates those namespace statistics
- # that haven't been created nor initialized.
- # It also updates the related namespace statistics
- class PopulateNamespaceStatistics
- def perform(group_ids, statistics)
- # Updating group statistics might involve calling Gitaly.
- # For example, when calculating `wiki_size`, we will need
- # to perform the request to check if the repo exists and
- # also the repository size.
- #
- # The `allow_n_plus_1_calls` method is only intended for
- # dev and test. It won't be raised in prod.
- ::Gitlab::GitalyClient.allow_n_plus_1_calls do
- relation(group_ids).each do |group|
- upsert_namespace_statistics(group, statistics)
- end
- end
- end
-
- private
-
- def upsert_namespace_statistics(group, statistics)
- response = ::Groups::UpdateStatisticsService.new(group, statistics: statistics).execute
-
- error_message("#{response.message} group: #{group.id}") if response.error?
- end
-
- def logger
- @logger ||= ::Gitlab::BackgroundMigration::Logger.build
- end
-
- def error_message(message)
- logger.error(message: "Namespace Statistics Migration: #{message}")
- end
-
- def relation(group_ids)
- Group.includes(:namespace_statistics).where(id: group_ids)
- end
- end
- end
-end
-
-Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateNamespaceStatistics')
diff --git a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb b/lib/gitlab/background_migration/populate_test_reports_issue_id.rb
deleted file mode 100644
index 301efd0c943..00000000000
--- a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-# rubocop: disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class PopulateTestReportsIssueId
- def perform(start_id, stop_id)
- # NO OP
- end
- end
- end
-end
-
-Gitlab::BackgroundMigration::PopulateTestReportsIssueId.prepend_mod
diff --git a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb
deleted file mode 100644
index 1f2b55004e4..00000000000
--- a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # The class to populates the non private projects counter of topics
- class PopulateTopicsNonPrivateProjectsCount
- SUB_BATCH_SIZE = 100
-
- # Temporary AR model for topics
- class Topic < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'topics'
- end
-
- def perform(start_id, stop_id)
- Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch|
- ApplicationRecord.connection.execute(<<~SQL)
- WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql})
- UPDATE topics
- SET non_private_projects_count = (
- SELECT COUNT(*)
- FROM project_topics
- INNER JOIN projects
- ON project_topics.project_id = projects.id
- WHERE project_topics.topic_id = batched_relation.id
- AND projects.visibility_level > 0
- )
- FROM batched_relation
- WHERE topics.id = batched_relation.id
- SQL
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/populate_vulnerability_reads.rb b/lib/gitlab/background_migration/populate_vulnerability_reads.rb
deleted file mode 100644
index 656c62d9ee5..00000000000
--- a/lib/gitlab/background_migration/populate_vulnerability_reads.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # rubocop:disable Style/Documentation
- class PopulateVulnerabilityReads
- include Gitlab::Database::DynamicModelHelpers
-
- PAUSE_SECONDS = 0.1
-
- def perform(start_id, end_id, sub_batch_size)
- vulnerability_model.where(id: start_id..end_id).each_batch(of: sub_batch_size) do |sub_batch|
- first, last = sub_batch.pick(Arel.sql('min(id), max(id)'))
- connection.execute(insert_query(first, last))
-
- sleep PAUSE_SECONDS
- end
-
- mark_job_as_succeeded(start_id, end_id, sub_batch_size)
- end
-
- private
-
- def vulnerability_model
- define_batchable_model('vulnerabilities', connection: connection)
- end
-
- def connection
- ApplicationRecord.connection
- end
-
- def insert_query(start_id, end_id)
- <<~SQL
- INSERT INTO vulnerability_reads (
- vulnerability_id,
- project_id,
- scanner_id,
- report_type,
- severity,
- state,
- has_issues,
- resolved_on_default_branch,
- uuid,
- location_image
- )
- SELECT
- vulnerabilities.id,
- vulnerabilities.project_id,
- vulnerability_scanners.id,
- vulnerabilities.report_type,
- vulnerabilities.severity,
- vulnerabilities.state,
- CASE
- WHEN
- vulnerability_issue_links.vulnerability_id IS NOT NULL
- THEN
- true
- ELSE
- false
- END
- has_issues,
- vulnerabilities.resolved_on_default_branch,
- vulnerability_occurrences.uuid::uuid,
- vulnerability_occurrences.location ->> 'image'
- FROM
- vulnerabilities
- INNER JOIN vulnerability_occurrences ON vulnerability_occurrences.vulnerability_id = vulnerabilities.id
- INNER JOIN vulnerability_scanners ON vulnerability_scanners.id = vulnerability_occurrences.scanner_id
- LEFT JOIN vulnerability_issue_links ON vulnerability_issue_links.vulnerability_id = vulnerabilities.id
- WHERE vulnerabilities.id BETWEEN #{start_id} AND #{end_id}
- ON CONFLICT(vulnerability_id) DO NOTHING;
- SQL
- end
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- # rubocop:enable Style/Documentation
- end
-end
diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
deleted file mode 100644
index 9a42d035285..00000000000
--- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
+++ /dev/null
@@ -1,218 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop: disable Style/Documentation
-class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # rubocop:disable Metrics/ClassLength
- # rubocop: disable Gitlab/NamespacedClass
- class VulnerabilitiesIdentifier < ActiveRecord::Base
- self.table_name = "vulnerability_identifiers"
- has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id'
- end
-
- class VulnerabilitiesFinding < ActiveRecord::Base
- include EachBatch
- include ShaAttribute
-
- self.table_name = "vulnerability_occurrences"
-
- has_many :signatures, foreign_key: 'finding_id', class_name: 'VulnerabilityFindingSignature', inverse_of: :finding
- belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id'
-
- REPORT_TYPES = {
- sast: 0,
- dependency_scanning: 1,
- container_scanning: 2,
- dast: 3,
- secret_detection: 4,
- coverage_fuzzing: 5,
- api_fuzzing: 6,
- cluster_image_scanning: 7,
- generic: 99
- }.with_indifferent_access.freeze
- enum report_type: REPORT_TYPES
-
- sha_attribute :fingerprint
- sha_attribute :location_fingerprint
- end
-
- class VulnerabilityFindingSignature < ActiveRecord::Base
- include ShaAttribute
-
- self.table_name = 'vulnerability_finding_signatures'
- belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'VulnerabilitiesFinding'
-
- sha_attribute :signature_sha
- end
-
- class VulnerabilitiesFindingPipeline < ActiveRecord::Base
- include EachBatch
- self.table_name = "vulnerability_occurrence_pipelines"
- end
-
- class Vulnerability < ActiveRecord::Base
- include EachBatch
- self.table_name = "vulnerabilities"
- end
-
- class CalculateFindingUUID
- FINDING_NAMESPACES_IDS = {
- development: "a143e9e2-41b3-47bc-9a19-081d089229f4",
- test: "a143e9e2-41b3-47bc-9a19-081d089229f4",
- staging: "a6930898-a1b2-4365-ab18-12aa474d9b26",
- production: "58dc0f06-936c-43b3-93bb-71693f1b6570"
- }.freeze
-
- NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze
- PACK_PATTERN = "NnnnnN"
-
- def self.call(value)
- Digest::UUID.uuid_v5(namespace_id, value)
- end
-
- def self.namespace_id
- namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym)
- # Digest::UUID is broken when using an UUID in namespace_id
- # https://github.com/rails/rails/issues/37681#issue-520718028
- namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN)
- end
- end
- # rubocop: enable Gitlab/NamespacedClass
-
- # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength
- def perform(start_id, end_id)
- log_info('Migration started', start_id: start_id, end_id: end_id)
-
- VulnerabilitiesFinding
- .joins(:primary_identifier)
- .includes(:signatures)
- .select(:id, :report_type, :primary_identifier_id, :fingerprint, :location_fingerprint, :project_id, :created_at, :vulnerability_id, :uuid)
- .where(id: start_id..end_id)
- .each_batch(of: 50) do |relation|
- duplicates = find_duplicates(relation)
- remove_findings(ids: duplicates) if duplicates.present?
-
- to_update = relation.reject { |finding| duplicates.include?(finding.id) }
-
- begin
- known_uuids = Set.new
- to_be_deleted = []
-
- mappings = to_update.each_with_object({}) do |finding, hash|
- uuid = calculate_uuid_v5_for_finding(finding)
-
- if known_uuids.add?(uuid)
- hash[finding] = { uuid: uuid }
- else
- to_be_deleted << finding.id
- end
- end
-
- # It is technically still possible to have duplicate uuids
- # if the data integrity is broken somehow and the primary identifiers of
- # the findings are pointing to different projects with the same fingerprint values.
- if to_be_deleted.present?
- log_info('Conflicting UUIDs found within the batch', finding_ids: to_be_deleted)
-
- remove_findings(ids: to_be_deleted)
- end
-
- ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) if mappings.present?
-
- log_info('Recalculation is done', finding_ids: mappings.keys.pluck(:id))
- rescue ActiveRecord::RecordNotUnique => error
- log_info('RecordNotUnique error received')
-
- match_data = /\(uuid\)=\((?<uuid>\S{36})\)/.match(error.message)
-
- # This exception returns the **correct** UUIDv5 which probably comes from a later record
- # and it's the one we can drop in the easiest way before retrying the UPDATE query
- if match_data
- uuid = match_data[:uuid]
- log_info('Conflicting UUID found', uuid: uuid)
-
- id = VulnerabilitiesFinding.find_by(uuid: uuid)&.id
- remove_findings(ids: id) if id
- retry
- else
- log_error('Couldnt find conflicting uuid')
-
- Gitlab::ErrorTracking.track_and_raise_exception(error)
- end
- end
- end
-
- mark_job_as_succeeded(start_id, end_id)
- rescue StandardError => error
- log_error('An exception happened')
-
- Gitlab::ErrorTracking.track_and_raise_exception(error)
- end
- # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength
-
- private
-
- def find_duplicates(relation)
- to_exclude = []
- relation.flat_map do |record|
- # Assuming we're scanning id 31 and the duplicate is id 40
- # first we'd process 31 and add 40 to the list of ids to remove
- # then we would process record 40 and add 31 to the list of removals
- # so we would drop both records
- to_exclude << record.id
-
- VulnerabilitiesFinding.where(
- report_type: record.report_type,
- location_fingerprint: record.location_fingerprint,
- primary_identifier_id: record.primary_identifier_id,
- project_id: record.project_id
- ).where.not(id: to_exclude).pluck(:id)
- end
- end
-
- def remove_findings(ids:)
- ids = Array(ids)
- log_info('Removing Findings and associated records', ids: ids)
-
- vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).uniq.compact
-
- VulnerabilitiesFindingPipeline.where(occurrence_id: ids).each_batch { |batch| batch.delete_all }
- Vulnerability.where(id: vulnerability_ids).each_batch { |batch| batch.delete_all }
- VulnerabilitiesFinding.where(id: ids).delete_all
- end
-
- def calculate_uuid_v5_for_finding(vulnerability_finding)
- return unless vulnerability_finding
-
- signatures = vulnerability_finding.signatures.sort_by { |signature| signature.algorithm_type_before_type_cast }
- location_fingerprint = signatures.last&.signature_sha || vulnerability_finding.location_fingerprint
-
- uuid_v5_name_components = {
- report_type: vulnerability_finding.report_type,
- primary_identifier_fingerprint: vulnerability_finding.fingerprint,
- location_fingerprint: location_fingerprint,
- project_id: vulnerability_finding.project_id
- }
-
- name = uuid_v5_name_components.values.join('-')
-
- CalculateFindingUUID.call(name)
- end
-
- def log_info(message, **extra)
- logger.info(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra)
- end
-
- def log_error(message, **extra)
- logger.error(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra)
- end
-
- def logger
- @logger ||= Gitlab::BackgroundMigration::Logger.build
- end
-
- def mark_job_as_succeeded(*arguments)
- Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- 'RecalculateVulnerabilitiesOccurrencesUuid',
- arguments
- )
- end
-end
diff --git a/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb b/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb
deleted file mode 100644
index 20200a1d508..00000000000
--- a/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # rubocop: disable Style/Documentation
- class RecalculateVulnerabilityFindingSignaturesForFindings
- def perform(start_id, stop_id)
- end
- end
- end
-end
-
-Gitlab::BackgroundMigration::RecalculateVulnerabilityFindingSignaturesForFindings.prepend_mod
diff --git a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb
deleted file mode 100644
index d47aa76f24b..00000000000
--- a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb
+++ /dev/null
@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Removing expire_at timestamps that shouldn't have
- # been written to traces on gitlab.com.
- class RemoveAllTraceExpirationDates
- include Gitlab::Database::MigrationHelpers
-
- BATCH_SIZE = 1_000
-
- # Stubbed class to connect to the CI database
- # connects_to has to be called in abstract classes.
- class MultiDbAdaptableClass < ActiveRecord::Base
- self.abstract_class = true
-
- if Gitlab::Database.has_config?(:ci)
- connects_to database: { writing: :ci, reading: :ci }
- end
- end
-
- # Stubbed class to access the ci_job_artifacts table
- class JobArtifact < MultiDbAdaptableClass
- include EachBatch
-
- self.table_name = 'ci_job_artifacts'
-
- TARGET_TIMESTAMPS = [
- Date.new(2021, 04, 22).midnight.utc,
- Date.new(2021, 05, 22).midnight.utc,
- Date.new(2021, 06, 22).midnight.utc,
- Date.new(2022, 01, 22).midnight.utc,
- Date.new(2022, 02, 22).midnight.utc,
- Date.new(2022, 03, 22).midnight.utc,
- Date.new(2022, 04, 22).midnight.utc
- ].freeze
-
- scope :traces, -> { where(file_type: 3) }
- scope :between, -> (start_id, end_id) { where(id: start_id..end_id) }
- scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) }
- end
-
- def perform(start_id, end_id)
- return unless Gitlab.com?
-
- JobArtifact.traces
- .between(start_id, end_id)
- .in_targeted_timestamps
- .each_batch(of: BATCH_SIZE) { |batch| batch.update_all(expire_at: nil) }
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb
deleted file mode 100644
index 4acef9029f9..00000000000
--- a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Remove vulnerability finding link records
- # The records will be repopulated from the `raw_metadata`
- # column of `vulnerability_occurrences` once the unique
- # index is in place.
- class RemoveVulnerabilityFindingLinks
- include Gitlab::Database::DynamicModelHelpers
-
- def perform(start_id, stop_id)
- define_batchable_model('vulnerability_finding_links', connection: ApplicationRecord.connection)
- .where(id: start_id..stop_id)
- .delete_all
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb
deleted file mode 100644
index 190e2fc22fb..00000000000
--- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # A job to nullify duplicate runners_token_encrypted values in projects table in batches
- class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects
- class Project < ActiveRecord::Base # rubocop:disable Style/Documentation
- include EachBatch
-
- self.table_name = 'projects'
-
- scope :base_query, -> { where.not(runners_token_encrypted: nil) }
- end
-
- def perform(start_id, end_id)
- # Reset duplicate runner tokens that would prevent creating an unique index.
- batch_records = Project.base_query.where(id: start_id..end_id)
-
- duplicate_tokens = Project.base_query
- .where(runners_token_encrypted: batch_records.select(:runners_token_encrypted).distinct)
- .group(:runners_token_encrypted)
- .having('COUNT(*) > 1')
- .pluck(:runners_token_encrypted)
-
- batch_records.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any?
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb
deleted file mode 100644
index b58eefa0ab3..00000000000
--- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # A job to nullify duplicate ci_runners_token values in projects table in batches
- class ResetDuplicateCiRunnersTokenValuesOnProjects
- class Project < ActiveRecord::Base # rubocop:disable Style/Documentation
- include EachBatch
-
- self.table_name = 'projects'
-
- scope :base_query, -> { where.not(runners_token: nil) }
- end
-
- def perform(start_id, end_id)
- # Reset duplicate runner tokens that would prevent creating an unique index.
- batch_records = Project.base_query.where(id: start_id..end_id)
-
- duplicate_tokens = Project.base_query
- .where(runners_token: batch_records.select(:runners_token).distinct)
- .group(:runners_token)
- .having('COUNT(*) > 1')
- .pluck(:runners_token)
-
- batch_records.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any?
-
- mark_job_as_succeeded(start_id, end_id)
- end
-
- private
-
- def mark_job_as_succeeded(*arguments)
- ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
- self.class.name.demodulize,
- arguments
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb
deleted file mode 100644
index b61f2ee7f4c..00000000000
--- a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Class to populate spent_at for timelogs
- class UpdateTimelogsNullSpentAt
- include Gitlab::Database::DynamicModelHelpers
-
- BATCH_SIZE = 100
-
- def perform(start_id, stop_id)
- define_batchable_model('timelogs', connection: connection)
- .where(spent_at: nil, id: start_id..stop_id)
- .each_batch(of: 100) do |subbatch|
- batch_start, batch_end = subbatch.pick('min(id), max(id)')
-
- update_timelogs(batch_start, batch_end)
- end
- end
-
- def update_timelogs(batch_start, batch_stop)
- execute(<<~SQL)
- UPDATE timelogs
- SET spent_at = created_at
- WHERE spent_at IS NULL
- AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop};
- SQL
- end
-
- def connection
- @connection ||= ApplicationRecord.connection
- end
-
- def execute(sql)
- connection.execute(sql)
- end
- end
- end
-end