diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-15 09:10:13 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-15 09:10:13 +0300 |
commit | b4610c51192adc832d25192038a781d2aa1ebb98 (patch) | |
tree | 860bb889ee850e13a1f947489d768801556f9213 /lib | |
parent | a7b6d465ae75b497ac34cb43361edd0c8ce45fbd (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
38 files changed, 15 insertions, 1508 deletions
diff --git a/lib/api/validations/validators/absence.rb b/lib/api/validations/validators/absence.rb index 7858ce7140b..49bf7f4a1c2 100644 --- a/lib/api/validations/validators/absence.rb +++ b/lib/api/validations/validators/absence.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class Absence < Grape::Validations::Base + class Absence < Grape::Validations::Validators::Base def validate_param!(attr_name, params) return if params.respond_to?(:key?) && !params.key?(attr_name) diff --git a/lib/api/validations/validators/array_none_any.rb b/lib/api/validations/validators/array_none_any.rb index 8c064eefbf2..cc86bd1a535 100644 --- a/lib/api/validations/validators/array_none_any.rb +++ b/lib/api/validations/validators/array_none_any.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class ArrayNoneAny < Grape::Validations::Base + class ArrayNoneAny < Grape::Validations::Validators::Base def validate_param!(attr_name, params) value = params[attr_name] diff --git a/lib/api/validations/validators/bulk_imports.rb b/lib/api/validations/validators/bulk_imports.rb index bff3424a0ac..f8ad5ed6d14 100644 --- a/lib/api/validations/validators/bulk_imports.rb +++ b/lib/api/validations/validators/bulk_imports.rb @@ -4,7 +4,7 @@ module API module Validations module Validators module BulkImports - class DestinationSlugPath < Grape::Validations::Base + class DestinationSlugPath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) if Feature.disabled?(:restrict_special_characters_in_namespace_path) return if params[attr_name] =~ Gitlab::Regex.group_path_regex @@ -29,7 +29,7 @@ module API end end - class DestinationNamespacePath < Grape::Validations::Base + class DestinationNamespacePath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) return if params[attr_name].blank? @@ -42,7 +42,7 @@ module API end end - class SourceFullPath < Grape::Validations::Base + class SourceFullPath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) return if params[attr_name] =~ Gitlab::Regex.bulk_import_source_full_path_regex diff --git a/lib/api/validations/validators/check_assignees_count.rb b/lib/api/validations/validators/check_assignees_count.rb index 15f48c09a4f..7d90f030f1a 100644 --- a/lib/api/validations/validators/check_assignees_count.rb +++ b/lib/api/validations/validators/check_assignees_count.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class CheckAssigneesCount < Grape::Validations::Base + class CheckAssigneesCount < Grape::Validations::Validators::Base def self.coerce lambda do |value| case value diff --git a/lib/api/validations/validators/email_or_email_list.rb b/lib/api/validations/validators/email_or_email_list.rb index 715a29c613d..8aa7c3bc32c 100644 --- a/lib/api/validations/validators/email_or_email_list.rb +++ b/lib/api/validations/validators/email_or_email_list.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class EmailOrEmailList < Grape::Validations::Base + class EmailOrEmailList < Grape::Validations::Validators::Base def validate_param!(attr_name, params) value = params[attr_name] diff --git a/lib/api/validations/validators/file_path.rb b/lib/api/validations/validators/file_path.rb index 2440d8890e2..7b32e1d71f8 100644 --- a/lib/api/validations/validators/file_path.rb +++ b/lib/api/validations/validators/file_path.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class FilePath < Grape::Validations::Base + class FilePath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) options = @option.is_a?(Hash) ? @option : {} path_allowlist = options.fetch(:allowlist, []) diff --git a/lib/api/validations/validators/git_ref.rb b/lib/api/validations/validators/git_ref.rb index dcb1db6ca33..711c272ab4e 100644 --- a/lib/api/validations/validators/git_ref.rb +++ b/lib/api/validations/validators/git_ref.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class GitRef < Grape::Validations::Base + class GitRef < Grape::Validations::Validators::Base # There are few checks that a Git reference should pass through to be valid reference. # The link contains some rules that have been added to this validator. # https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html diff --git a/lib/api/validations/validators/git_sha.rb b/lib/api/validations/validators/git_sha.rb index 665d1878b4c..830137a0197 100644 --- a/lib/api/validations/validators/git_sha.rb +++ b/lib/api/validations/validators/git_sha.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class GitSha < Grape::Validations::Base + class GitSha < Grape::Validations::Validators::Base def validate_param!(attr_name, params) sha = params[attr_name] diff --git a/lib/api/validations/validators/integer_or_custom_value.rb b/lib/api/validations/validators/integer_or_custom_value.rb index ad7848583fc..ff2df858a36 100644 --- a/lib/api/validations/validators/integer_or_custom_value.rb +++ b/lib/api/validations/validators/integer_or_custom_value.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class IntegerOrCustomValue < Grape::Validations::Base + class IntegerOrCustomValue < Grape::Validations::Validators::Base def initialize(attrs, options, required, scope, **opts) @custom_values = extract_custom_values(options) super diff --git a/lib/api/validations/validators/limit.rb b/lib/api/validations/validators/limit.rb index 7e11f1d77cc..781bebac716 100644 --- a/lib/api/validations/validators/limit.rb +++ b/lib/api/validations/validators/limit.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class Limit < Grape::Validations::Base + class Limit < Grape::Validations::Validators::Base def validate_param!(attr_name, params) value = params[attr_name] diff --git a/lib/api/validations/validators/project_portable.rb b/lib/api/validations/validators/project_portable.rb index 3a7ea5ea71e..50ab32b6b7a 100644 --- a/lib/api/validations/validators/project_portable.rb +++ b/lib/api/validations/validators/project_portable.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class ProjectPortable < Grape::Validations::Base + class ProjectPortable < Grape::Validations::Validators::Base def validate_param!(attr_name, params) portable = params[attr_name] diff --git a/lib/api/validations/validators/untrusted_regexp.rb b/lib/api/validations/validators/untrusted_regexp.rb index 3ddea2bd9de..c5560be2e16 100644 --- a/lib/api/validations/validators/untrusted_regexp.rb +++ b/lib/api/validations/validators/untrusted_regexp.rb @@ -3,7 +3,7 @@ module API module Validations module Validators - class UntrustedRegexp < Grape::Validations::Base + class UntrustedRegexp < Grape::Validations::Validators::Base def validate_param!(attr_name, params) value = params[attr_name] return unless value diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb index 5983036a8e5..d6b6fdb7149 100644 --- a/lib/banzai/filter/references/user_reference_filter.rb +++ b/lib/banzai/filter/references/user_reference_filter.rb @@ -45,7 +45,7 @@ module Banzai # have `gfm` and `gfm-project_member` class names attached for styling. def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, username| - if username == 'all' && !skip_project_check? + if Feature.disabled?(:disable_all_mention) && username == 'all' && !skip_project_check? link_to_all(link_content: link_content) else cached_call(:banzai_url_for_object, match, path: [User, username.downcase]) do diff --git a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb b/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb deleted file mode 100644 index 63112b52584..00000000000 --- a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb +++ /dev/null @@ -1,153 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Ensure queuing entries are present even if admins skip upgrades. - class BackfillCiQueuingTables - class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - end - - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'projects' - - belongs_to :namespace - has_one :ci_cd_settings, class_name: 'Gitlab::BackgroundMigration::BackfillCiQueuingTables::ProjectCiCdSetting' - - def group_runners_enabled? - return false unless ci_cd_settings - - ci_cd_settings.group_runners_enabled? - end - end - - class ProjectCiCdSetting < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'project_ci_cd_settings' - end - - class Taggings < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'taggings' - end - - module Ci - class Build < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - - belongs_to :project - - scope :pending, -> do - where(status: :pending, type: 'Ci::Build', runner_id: nil) - end - - def self.each_batch(of: 1000, column: :id, order: { runner_id: :asc, id: :asc }, order_hint: nil) - start = except(:select).select(column).reorder(order) - start = start.take - return unless start - - start_id = start[column] - arel_table = self.arel_table - - 1.step do |index| - start_cond = arel_table[column].gteq(start_id) - stop = except(:select).select(column).where(start_cond).reorder(order) - stop = stop.offset(of).limit(1).take - relation = where(start_cond) - - if stop - stop_id = stop[column] - start_id = stop_id - stop_cond = arel_table[column].lt(stop_id) - relation = relation.where(stop_cond) - end - - # Any ORDER BYs are useless for this relation and can lead to less - # efficient UPDATE queries, hence we get rid of it. - relation = relation.except(:order) - - # Using unscoped is necessary to prevent leaking the current scope used by - # ActiveRecord to chain `each_batch` method. - unscoped { yield relation, index } - - break unless stop - end - end - - def tags_ids - BackfillCiQueuingTables::Taggings - .where(taggable_id: id, taggable_type: 'CommitStatus') - .pluck(:tag_id) - end - end - - class PendingBuild < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'ci_pending_builds' - - class << self - def upsert_from_build!(build) - entry = self.new(args_from_build(build)) - - self.upsert( - entry.attributes.compact, - returning: %w[build_id], - unique_by: :build_id) - end - - def args_from_build(build) - project = build.project - - { - build_id: build.id, - project_id: build.project_id, - protected: build.protected?, - namespace_id: project.namespace_id, - tag_ids: build.tags_ids, - instance_runners_enabled: project.shared_runners_enabled?, - namespace_traversal_ids: namespace_traversal_ids(project) - } - end - - def namespace_traversal_ids(project) - if project.group_runners_enabled? - project.namespace.traversal_ids - else - [] - end - end - end - end - end - - BATCH_SIZE = 100 - - def perform(start_id, end_id) - scope = BackfillCiQueuingTables::Ci::Build.pending.where(id: start_id..end_id) - pending_builds_query = BackfillCiQueuingTables::Ci::PendingBuild - .where('ci_builds.id = ci_pending_builds.build_id') - .select(1) - - scope.each_batch(of: BATCH_SIZE) do |builds| - builds = builds.where('NOT EXISTS (?)', pending_builds_query) - builds = builds.includes(:project, project: [:namespace, :ci_cd_settings]) - - builds.each do |build| - BackfillCiQueuingTables::Ci::PendingBuild.upsert_from_build!(build) - end - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb deleted file mode 100644 index c45dcad5b2d..00000000000 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill group_features for an array of groups - class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob - job_arguments :batch_size - operation_name :upsert_group_features - feature_category :database - - def perform - each_sub_batch( - batching_arguments: { order_hint: :type }, - batching_scope: ->(relation) { relation.where(type: 'Group') } - ) do |sub_batch| - upsert_group_features(sub_batch) - end - end - - private - - def upsert_group_features(relation) - connection.execute( - <<~SQL - INSERT INTO group_features (group_id, created_at, updated_at) - SELECT namespaces.id as group_id, now(), now() - FROM namespaces - WHERE namespaces.type = 'Group' AND namespaces.id IN(#{relation.select(:id).limit(batch_size).to_sql}) - ON CONFLICT (group_id) DO NOTHING; - SQL - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb deleted file mode 100644 index 0585924cb7b..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfills the `routes.namespace_id` column, by copying source_id value - # (for groups and user namespaces source_id == namespace_id) - class BackfillNamespaceIdForNamespaceRoute - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all('namespace_id=source_id') - end - - pause_ms = [0, pause_ms].max - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ApplicationRecord.connection) - .joins('inner join namespaces on routes.source_id = namespaces.id') - .where(source_key_column => start_id..stop_id) - .where(namespace_id: nil) - .where(source_type: 'Namespace') - end - end - end -end diff --git a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb b/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb deleted file mode 100644 index b703faf6a6c..00000000000 --- a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Cleanup draft column data inserted by a faulty regex - # - class CleanupDraftDataFromFaultyRegex - # Migration only version of MergeRequest table - ## - class MergeRequest < ActiveRecord::Base - LEAKY_REGEXP_STR = "^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP" - CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" - - include EachBatch - - self.table_name = 'merge_requests' - - def self.eligible - where(state_id: 1) - .where(draft: true) - .where("title ~* ?", LEAKY_REGEXP_STR) - .where("title !~* ?", CORRECTED_REGEXP_STR) - end - end - - def perform(start_id, end_id) - eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) - - return if eligible_mrs.empty? - - eligible_mrs.each_slice(10) do |slice| - MergeRequest.where(id: slice).update_all(draft: false) - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'CleanupDraftDataFromFaultyRegex', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb deleted file mode 100644 index 28c28ae48eb..00000000000 --- a/lib/gitlab/background_migration/encrypt_integration_properties.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Migrates the integration.properties column from plaintext to encrypted text. - class EncryptIntegrationProperties - # The Integration model, with just the relevant bits. - class Integration < ActiveRecord::Base - include EachBatch - - ALGORITHM = 'aes-256-gcm' - - self.table_name = 'integrations' - self.inheritance_column = :_type_disabled - - scope :with_properties, -> { where.not(properties: nil) } - scope :not_already_encrypted, -> { where(encrypted_properties: nil) } - scope :for_batch, ->(range) { where(id: range) } - - attr_encrypted :encrypted_properties_tmp, - attribute: :encrypted_properties, - mode: :per_attribute_iv, - key: ::Settings.attr_encrypted_db_key_base_32, - algorithm: ALGORITHM, - marshal: true, - marshaler: ::Gitlab::Json, - encode: false, - encode_iv: false - - # See 'Integration#reencrypt_properties' - def encrypt_properties - data = ::Gitlab::Json.parse(properties) - iv = generate_iv(ALGORITHM) - ep = self.class.attr_encrypt(:encrypted_properties_tmp, data, { iv: iv }) - - [ep, iv] - end - end - - def perform(start_id, stop_id) - batch_query = Integration.with_properties.not_already_encrypted.for_batch(start_id..stop_id) - encrypt_batch(batch_query) - mark_job_as_succeeded(start_id, stop_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - - # represent binary string as a PSQL binary literal: - # https://www.postgresql.org/docs/9.4/datatype-binary.html - def bytea(value) - "'\\x#{value.unpack1('H*')}'::bytea" - end - - def encrypt_batch(batch_query) - values = batch_query.select(:id, :properties).map do |record| - encrypted_properties, encrypted_properties_iv = record.encrypt_properties - "(#{record.id}, #{bytea(encrypted_properties)}, #{bytea(encrypted_properties_iv)})" - end - - return if values.empty? - - Integration.connection.execute(<<~SQL.squish) - WITH cte(cte_id, cte_encrypted_properties, cte_encrypted_properties_iv) - AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT * - FROM (VALUES #{values.join(',')}) AS t (id, encrypted_properties, encrypted_properties_iv) - ) - UPDATE #{Integration.table_name} - SET encrypted_properties = cte_encrypted_properties - , encrypted_properties_iv = cte_encrypted_properties_iv - FROM cte - WHERE cte_id = id - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb deleted file mode 100644 index 961dea028c9..00000000000 --- a/lib/gitlab/background_migration/encrypt_static_object_token.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Populates "static_object_token_encrypted" field with encrypted versions - # of values from "static_object_token" field - class EncryptStaticObjectToken - # rubocop:disable Style/Documentation - class User < ActiveRecord::Base - include ::EachBatch - self.table_name = 'users' - scope :with_static_object_token, -> { where.not(static_object_token: nil) } - scope :without_static_object_token_encrypted, -> { where(static_object_token_encrypted: nil) } - end - # rubocop:enable Style/Documentation - - BATCH_SIZE = 100 - - def perform(start_id, end_id) - ranged_query = User - .where(id: start_id..end_id) - .with_static_object_token - .without_static_object_token_encrypted - - ranged_query.each_batch(of: BATCH_SIZE) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - - batch_query = User.unscoped - .where(id: first..last) - .with_static_object_token - .without_static_object_token_encrypted - - user_tokens = batch_query.pluck(:id, :static_object_token) - - user_encrypted_tokens = user_tokens.map do |(id, plaintext_token)| - next if plaintext_token.blank? - - [id, Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token)] - end - - encrypted_tokens_sql = user_encrypted_tokens.compact.map { |(id, token)| "(#{id}, '#{token}')" }.join(',') - - next unless user_encrypted_tokens.present? - - User.connection.execute(<<~SQL) - WITH cte(cte_id, cte_token) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT * - FROM (VALUES #{encrypted_tokens_sql}) AS t (id, token) - ) - UPDATE #{User.table_name} - SET static_object_token_encrypted = cte_token - FROM cte - WHERE cte_id = id - SQL - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb deleted file mode 100644 index 3772430d0b7..00000000000 --- a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Fix project name duplicates and backfill missing project namespace ids - class FixDuplicateProjectNameAndPath - SUB_BATCH_SIZE = 10 - # isolated project active record - class Project < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'projects' - - scope :without_project_namespace, -> { where(project_namespace_id: nil) } - scope :id_in, ->(ids) { where(id: ids) } - end - - def perform(start_id, end_id) - @project_ids = fetch_project_ids(start_id, end_id) - backfill_project_namespaces_service = init_backfill_service(project_ids) - backfill_project_namespaces_service.cleanup_gin_index('projects') - - project_ids.each_slice(SUB_BATCH_SIZE) do |ids| - ApplicationRecord.connection.execute(update_projects_name_and_path_sql(ids)) - end - - backfill_project_namespaces_service.backfill_project_namespaces - - mark_job_as_succeeded(start_id, end_id) - end - - private - - attr_accessor :project_ids - - def fetch_project_ids(start_id, end_id) - Project.without_project_namespace.where(id: start_id..end_id) - end - - def init_backfill_service(project_ids) - service = Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces.new - service.project_ids = project_ids - service.sub_batch_size = SUB_BATCH_SIZE - - service - end - - def update_projects_name_and_path_sql(project_ids) - <<~SQL - WITH cte (project_id, path_from_route ) AS ( - #{path_from_route_sql(project_ids).to_sql} - ) - UPDATE - projects - SET - name = concat(projects.name, '-', id), - path = CASE - WHEN projects.path <> cte.path_from_route THEN path_from_route - ELSE projects.path - END - FROM - cte - WHERE - projects.id = cte.project_id; - SQL - end - - def path_from_route_sql(project_ids) - Project.without_project_namespace.id_in(project_ids) - .joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'") - .select("projects.id, SUBSTRING(routes.path FROM '[^/]+(?=/$|$)') AS path_from_route") - end - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'FixDuplicateProjectNameAndPath', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb b/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb deleted file mode 100644 index 2c09b8c0b24..00000000000 --- a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class FixIncorrectMaxSeatsUsed - def perform(batch = nil) - end - end - end -end - -Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed.prepend_mod_with('Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed') diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb deleted file mode 100644 index db3f98bc2ba..00000000000 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -require 'parser/ruby27' - -module Gitlab - module BackgroundMigration - # This migration fixes raw_metadata entries which have incorrectly been passed a Ruby Hash instead of JSON data. - class FixVulnerabilityOccurrencesWithHashesAsRawMetadata - CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 - GENERIC_REPORT_TYPE = 99 - - # Type error is used to handle unexpected types when parsing stringified hashes. - class TypeError < ::StandardError - attr_reader :message, :type - - def initialize(message, type) - @message = message - @type = type - end - end - - # Migration model namespace isolated from application code. - class Finding < ActiveRecord::Base - include EachBatch - - self.table_name = 'vulnerability_occurrences' - - scope :by_api_report_types, -> { where(report_type: [CLUSTER_IMAGE_SCANNING_REPORT_TYPE, GENERIC_REPORT_TYPE]) } - end - - def perform(start_id, end_id) - Finding.by_api_report_types.where(id: start_id..end_id).each do |finding| - next if valid_json?(finding.raw_metadata) - - metadata = hash_from_s(finding.raw_metadata) - - finding.update(raw_metadata: metadata.to_json) if metadata - end - mark_job_as_succeeded(start_id, end_id) - end - - def hash_from_s(str_hash) - ast = Parser::Ruby27.parse(str_hash) - - unless ast.type == :hash - ::Gitlab::AppLogger.error(message: "expected raw_metadata to be a hash", type: ast.type) - return - end - - parse_hash(ast) - rescue Parser::SyntaxError => e - ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message) - nil - rescue TypeError => e - ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message, type: e.type) - nil - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'FixVulnerabilityOccurrencesWithHashesAsRawMetadata', - arguments - ) - end - - def valid_json?(metadata) - Oj.load(metadata) - true - rescue Oj::ParseError, EncodingError, JSON::ParserError, JSON::GeneratorError, Encoding::UndefinedConversionError - false - end - - def parse_hash(hash) - out = {} - hash.children.each do |node| - unless node.type == :pair - raise TypeError.new("expected child of hash to be a `pair`", node.type) - end - - key, value = node.children - - key = parse_key(key) - value = parse_value(value) - - out[key] = value - end - - out - end - - def parse_key(key) - case key.type - when :sym, :str, :int - key.children.first - else - raise TypeError.new("expected key to be either symbol, string, or integer", key.type) - end - end - - def parse_value(value) - case value.type - when :sym, :str, :int - value.children.first - # rubocop:disable Lint/BooleanSymbol - when :true - true - when :false - false - # rubocop:enable Lint/BooleanSymbol - when :nil - nil - when :array - value.children.map { |c| parse_value(c) } - when :hash - parse_hash(value) - else - raise TypeError.new("value of a pair was an unexpected type", value.type) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/merge_topics_with_same_name.rb b/lib/gitlab/background_migration/merge_topics_with_same_name.rb deleted file mode 100644 index 07231098a5f..00000000000 --- a/lib/gitlab/background_migration/merge_topics_with_same_name.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to merge project topics with the same case insensitive name - class MergeTopicsWithSameName - # Temporary AR model for topics - class Topic < ActiveRecord::Base - self.table_name = 'topics' - end - - # Temporary AR model for project topic assignment - class ProjectTopic < ActiveRecord::Base - self.table_name = 'project_topics' - end - - def perform(topic_names) - topic_names.each do |topic_name| - topics = Topic.where('LOWER(name) = ?', topic_name) - .order(total_projects_count: :desc, non_private_projects_count: :desc, id: :asc) - .to_a - topic_to_keep = topics.shift - merge_topics(topic_to_keep, topics) if topics.any? - end - end - - private - - def merge_topics(topic_to_keep, topics_to_remove) - description = topic_to_keep.description - - topics_to_remove.each do |topic| - description ||= topic.description if topic.description.present? - process_avatar(topic_to_keep, topic) if topic.avatar.present? - - ProjectTopic.transaction do - ProjectTopic.where(topic_id: topic.id) - .where.not(project_id: ProjectTopic.where(topic_id: topic_to_keep).select(:project_id)) - .update_all(topic_id: topic_to_keep.id) - ProjectTopic.where(topic_id: topic.id).delete_all - end - end - - Topic.where(id: topics_to_remove).delete_all - - topic_to_keep.update( - description: description, - total_projects_count: total_projects_count(topic_to_keep.id), - non_private_projects_count: non_private_projects_count(topic_to_keep.id) - ) - end - - # We intentionally use application code here because we need to copy/remove avatar files - def process_avatar(topic_to_keep, topic_to_remove) - topic_to_remove = ::Projects::Topic.find(topic_to_remove.id) - topic_to_keep = ::Projects::Topic.find(topic_to_keep.id) - unless topic_to_keep.avatar.present? - topic_to_keep.avatar = topic_to_remove.avatar - topic_to_keep.save! - end - - topic_to_remove.remove_avatar! - topic_to_remove.save! - end - - def total_projects_count(topic_id) - ProjectTopic.where(topic_id: topic_id).count - end - - def non_private_projects_count(topic_id) - ProjectTopic.joins('INNER JOIN projects ON project_topics.project_id = projects.id') - .where(project_topics: { topic_id: topic_id }).where('projects.visibility_level in (10, 20)').count - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb b/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb deleted file mode 100644 index 49eff6e2771..00000000000 --- a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Migrates personal namespace project `maintainer` memberships (for the associated user only) to OWNER - # Does not create any missing records, simply migrates existing ones - class MigratePersonalNamespaceProjectMaintainerToOwner - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all('access_level = 50') - end - - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - # members of projects within their own personal namespace - - # rubocop: disable CodeReuse/ActiveRecord - define_batchable_model(:members, connection: ApplicationRecord.connection) - .where(source_key_column => start_id..stop_id) - .joins("INNER JOIN projects ON members.source_id = projects.id") - .joins("INNER JOIN namespaces ON projects.namespace_id = namespaces.id") - .where(type: 'ProjectMember') - .where("namespaces.type = 'User'") - .where('members.access_level < 50') - .where('namespaces.owner_id = members.user_id') - end - end - # rubocop: enable CodeReuse/ActiveRecord - end -end diff --git a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb deleted file mode 100644 index d7d24960a41..00000000000 --- a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to migrate category of integrations to third_party_wiki for confluence and shimo - class MigrateShimoConfluenceIntegrationCategory - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id) - define_batchable_model('integrations', connection: ApplicationRecord.connection) - .where(id: start_id..end_id, type_new: %w[Integrations::Confluence Integrations::Shimo]) - .update_all(category: 'third_party_wiki') - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb deleted file mode 100644 index 13b66b2e02e..00000000000 --- a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify orphan runner_id on ci_builds table - class NullifyOrphanRunnerIdOnCiBuilds - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - pause_ms = 0 if pause_ms < 0 - - batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - filtered_sub_batch(sub_batch).update_all(runner_id: nil) - end - - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def connection - ::Ci::ApplicationRecord.connection - end - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection) - .where(source_key_column => start_id..stop_id) - end - - def filtered_sub_batch(sub_batch) - sub_batch - .joins('LEFT OUTER JOIN ci_runners ON ci_runners.id = ci_builds.runner_id') - .where('ci_builds.runner_id IS NOT NULL AND ci_runners.id IS NULL') - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb deleted file mode 100644 index a9611e9814c..00000000000 --- a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to populates the migration_plan column of container_repositories - # with the current plan of the namespaces that owns the container_repository - # - # The plan can be NULL, in which case no UPDATE - # will be executed. - class PopulateContainerRepositoryMigrationPlan - def perform(start_id, end_id) - (start_id..end_id).each do |id| - execute(<<~SQL) - WITH selected_plan AS ( - SELECT "plans"."name" - FROM "container_repositories" - INNER JOIN "projects" ON "projects"."id" = "container_repositories"."project_id" - INNER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id" - INNER JOIN "gitlab_subscriptions" ON "gitlab_subscriptions"."namespace_id" = "namespaces"."traversal_ids"[1] - INNER JOIN "plans" ON "plans"."id" = "gitlab_subscriptions"."hosted_plan_id" - WHERE "container_repositories"."id" = #{id} - ) - UPDATE container_repositories - SET migration_plan = selected_plan.name - FROM selected_plan - WHERE container_repositories.id = #{id}; - SQL - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def connection - @connection ||= ApplicationRecord.connection - end - - def execute(sql) - connection.execute(sql) - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb deleted file mode 100644 index 97927ef48c2..00000000000 --- a/lib/gitlab/background_migration/populate_namespace_statistics.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those namespace statistics - # that haven't been created nor initialized. - # It also updates the related namespace statistics - class PopulateNamespaceStatistics - def perform(group_ids, statistics) - # Updating group statistics might involve calling Gitaly. - # For example, when calculating `wiki_size`, we will need - # to perform the request to check if the repo exists and - # also the repository size. - # - # The `allow_n_plus_1_calls` method is only intended for - # dev and test. It won't be raised in prod. - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - relation(group_ids).each do |group| - upsert_namespace_statistics(group, statistics) - end - end - end - - private - - def upsert_namespace_statistics(group, statistics) - response = ::Groups::UpdateStatisticsService.new(group, statistics: statistics).execute - - error_message("#{response.message} group: #{group.id}") if response.error? - end - - def logger - @logger ||= ::Gitlab::BackgroundMigration::Logger.build - end - - def error_message(message) - logger.error(message: "Namespace Statistics Migration: #{message}") - end - - def relation(group_ids) - Group.includes(:namespace_statistics).where(id: group_ids) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateNamespaceStatistics') diff --git a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb b/lib/gitlab/background_migration/populate_test_reports_issue_id.rb deleted file mode 100644 index 301efd0c943..00000000000 --- a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true -# rubocop: disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateTestReportsIssueId - def perform(start_id, stop_id) - # NO OP - end - end - end -end - -Gitlab::BackgroundMigration::PopulateTestReportsIssueId.prepend_mod diff --git a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb deleted file mode 100644 index 1f2b55004e4..00000000000 --- a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to populates the non private projects counter of topics - class PopulateTopicsNonPrivateProjectsCount - SUB_BATCH_SIZE = 100 - - # Temporary AR model for topics - class Topic < ActiveRecord::Base - include EachBatch - - self.table_name = 'topics' - end - - def perform(start_id, stop_id) - Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ApplicationRecord.connection.execute(<<~SQL) - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) - UPDATE topics - SET non_private_projects_count = ( - SELECT COUNT(*) - FROM project_topics - INNER JOIN projects - ON project_topics.project_id = projects.id - WHERE project_topics.topic_id = batched_relation.id - AND projects.visibility_level > 0 - ) - FROM batched_relation - WHERE topics.id = batched_relation.id - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_vulnerability_reads.rb b/lib/gitlab/background_migration/populate_vulnerability_reads.rb deleted file mode 100644 index 656c62d9ee5..00000000000 --- a/lib/gitlab/background_migration/populate_vulnerability_reads.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class PopulateVulnerabilityReads - include Gitlab::Database::DynamicModelHelpers - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - vulnerability_model.where(id: start_id..end_id).each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - connection.execute(insert_query(first, last)) - - sleep PAUSE_SECONDS - end - - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - def vulnerability_model - define_batchable_model('vulnerabilities', connection: connection) - end - - def connection - ApplicationRecord.connection - end - - def insert_query(start_id, end_id) - <<~SQL - INSERT INTO vulnerability_reads ( - vulnerability_id, - project_id, - scanner_id, - report_type, - severity, - state, - has_issues, - resolved_on_default_branch, - uuid, - location_image - ) - SELECT - vulnerabilities.id, - vulnerabilities.project_id, - vulnerability_scanners.id, - vulnerabilities.report_type, - vulnerabilities.severity, - vulnerabilities.state, - CASE - WHEN - vulnerability_issue_links.vulnerability_id IS NOT NULL - THEN - true - ELSE - false - END - has_issues, - vulnerabilities.resolved_on_default_branch, - vulnerability_occurrences.uuid::uuid, - vulnerability_occurrences.location ->> 'image' - FROM - vulnerabilities - INNER JOIN vulnerability_occurrences ON vulnerability_occurrences.vulnerability_id = vulnerabilities.id - INNER JOIN vulnerability_scanners ON vulnerability_scanners.id = vulnerability_occurrences.scanner_id - LEFT JOIN vulnerability_issue_links ON vulnerability_issue_links.vulnerability_id = vulnerabilities.id - WHERE vulnerabilities.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT(vulnerability_id) DO NOTHING; - SQL - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - # rubocop:enable Style/Documentation - end -end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb deleted file mode 100644 index 9a42d035285..00000000000 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ /dev/null @@ -1,218 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # rubocop:disable Metrics/ClassLength - # rubocop: disable Gitlab/NamespacedClass - class VulnerabilitiesIdentifier < ActiveRecord::Base - self.table_name = "vulnerability_identifiers" - has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id' - end - - class VulnerabilitiesFinding < ActiveRecord::Base - include EachBatch - include ShaAttribute - - self.table_name = "vulnerability_occurrences" - - has_many :signatures, foreign_key: 'finding_id', class_name: 'VulnerabilityFindingSignature', inverse_of: :finding - belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' - - REPORT_TYPES = { - sast: 0, - dependency_scanning: 1, - container_scanning: 2, - dast: 3, - secret_detection: 4, - coverage_fuzzing: 5, - api_fuzzing: 6, - cluster_image_scanning: 7, - generic: 99 - }.with_indifferent_access.freeze - enum report_type: REPORT_TYPES - - sha_attribute :fingerprint - sha_attribute :location_fingerprint - end - - class VulnerabilityFindingSignature < ActiveRecord::Base - include ShaAttribute - - self.table_name = 'vulnerability_finding_signatures' - belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'VulnerabilitiesFinding' - - sha_attribute :signature_sha - end - - class VulnerabilitiesFindingPipeline < ActiveRecord::Base - include EachBatch - self.table_name = "vulnerability_occurrence_pipelines" - end - - class Vulnerability < ActiveRecord::Base - include EachBatch - self.table_name = "vulnerabilities" - end - - class CalculateFindingUUID - FINDING_NAMESPACES_IDS = { - development: "a143e9e2-41b3-47bc-9a19-081d089229f4", - test: "a143e9e2-41b3-47bc-9a19-081d089229f4", - staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", - production: "58dc0f06-936c-43b3-93bb-71693f1b6570" - }.freeze - - NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN" - - def self.call(value) - Digest::UUID.uuid_v5(namespace_id, value) - end - - def self.namespace_id - namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym) - # Digest::UUID is broken when using an UUID in namespace_id - # https://github.com/rails/rails/issues/37681#issue-520718028 - namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN) - end - end - # rubocop: enable Gitlab/NamespacedClass - - # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength - def perform(start_id, end_id) - log_info('Migration started', start_id: start_id, end_id: end_id) - - VulnerabilitiesFinding - .joins(:primary_identifier) - .includes(:signatures) - .select(:id, :report_type, :primary_identifier_id, :fingerprint, :location_fingerprint, :project_id, :created_at, :vulnerability_id, :uuid) - .where(id: start_id..end_id) - .each_batch(of: 50) do |relation| - duplicates = find_duplicates(relation) - remove_findings(ids: duplicates) if duplicates.present? - - to_update = relation.reject { |finding| duplicates.include?(finding.id) } - - begin - known_uuids = Set.new - to_be_deleted = [] - - mappings = to_update.each_with_object({}) do |finding, hash| - uuid = calculate_uuid_v5_for_finding(finding) - - if known_uuids.add?(uuid) - hash[finding] = { uuid: uuid } - else - to_be_deleted << finding.id - end - end - - # It is technically still possible to have duplicate uuids - # if the data integrity is broken somehow and the primary identifiers of - # the findings are pointing to different projects with the same fingerprint values. - if to_be_deleted.present? - log_info('Conflicting UUIDs found within the batch', finding_ids: to_be_deleted) - - remove_findings(ids: to_be_deleted) - end - - ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) if mappings.present? - - log_info('Recalculation is done', finding_ids: mappings.keys.pluck(:id)) - rescue ActiveRecord::RecordNotUnique => error - log_info('RecordNotUnique error received') - - match_data = /\(uuid\)=\((?<uuid>\S{36})\)/.match(error.message) - - # This exception returns the **correct** UUIDv5 which probably comes from a later record - # and it's the one we can drop in the easiest way before retrying the UPDATE query - if match_data - uuid = match_data[:uuid] - log_info('Conflicting UUID found', uuid: uuid) - - id = VulnerabilitiesFinding.find_by(uuid: uuid)&.id - remove_findings(ids: id) if id - retry - else - log_error('Couldnt find conflicting uuid') - - Gitlab::ErrorTracking.track_and_raise_exception(error) - end - end - end - - mark_job_as_succeeded(start_id, end_id) - rescue StandardError => error - log_error('An exception happened') - - Gitlab::ErrorTracking.track_and_raise_exception(error) - end - # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength - - private - - def find_duplicates(relation) - to_exclude = [] - relation.flat_map do |record| - # Assuming we're scanning id 31 and the duplicate is id 40 - # first we'd process 31 and add 40 to the list of ids to remove - # then we would process record 40 and add 31 to the list of removals - # so we would drop both records - to_exclude << record.id - - VulnerabilitiesFinding.where( - report_type: record.report_type, - location_fingerprint: record.location_fingerprint, - primary_identifier_id: record.primary_identifier_id, - project_id: record.project_id - ).where.not(id: to_exclude).pluck(:id) - end - end - - def remove_findings(ids:) - ids = Array(ids) - log_info('Removing Findings and associated records', ids: ids) - - vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).uniq.compact - - VulnerabilitiesFindingPipeline.where(occurrence_id: ids).each_batch { |batch| batch.delete_all } - Vulnerability.where(id: vulnerability_ids).each_batch { |batch| batch.delete_all } - VulnerabilitiesFinding.where(id: ids).delete_all - end - - def calculate_uuid_v5_for_finding(vulnerability_finding) - return unless vulnerability_finding - - signatures = vulnerability_finding.signatures.sort_by { |signature| signature.algorithm_type_before_type_cast } - location_fingerprint = signatures.last&.signature_sha || vulnerability_finding.location_fingerprint - - uuid_v5_name_components = { - report_type: vulnerability_finding.report_type, - primary_identifier_fingerprint: vulnerability_finding.fingerprint, - location_fingerprint: location_fingerprint, - project_id: vulnerability_finding.project_id - } - - name = uuid_v5_name_components.values.join('-') - - CalculateFindingUUID.call(name) - end - - def log_info(message, **extra) - logger.info(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) - end - - def log_error(message, **extra) - logger.error(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'RecalculateVulnerabilitiesOccurrencesUuid', - arguments - ) - end -end diff --git a/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb b/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb deleted file mode 100644 index 20200a1d508..00000000000 --- a/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class RecalculateVulnerabilityFindingSignaturesForFindings - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RecalculateVulnerabilityFindingSignaturesForFindings.prepend_mod diff --git a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb deleted file mode 100644 index d47aa76f24b..00000000000 --- a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Removing expire_at timestamps that shouldn't have - # been written to traces on gitlab.com. - class RemoveAllTraceExpirationDates - include Gitlab::Database::MigrationHelpers - - BATCH_SIZE = 1_000 - - # Stubbed class to connect to the CI database - # connects_to has to be called in abstract classes. - class MultiDbAdaptableClass < ActiveRecord::Base - self.abstract_class = true - - if Gitlab::Database.has_config?(:ci) - connects_to database: { writing: :ci, reading: :ci } - end - end - - # Stubbed class to access the ci_job_artifacts table - class JobArtifact < MultiDbAdaptableClass - include EachBatch - - self.table_name = 'ci_job_artifacts' - - TARGET_TIMESTAMPS = [ - Date.new(2021, 04, 22).midnight.utc, - Date.new(2021, 05, 22).midnight.utc, - Date.new(2021, 06, 22).midnight.utc, - Date.new(2022, 01, 22).midnight.utc, - Date.new(2022, 02, 22).midnight.utc, - Date.new(2022, 03, 22).midnight.utc, - Date.new(2022, 04, 22).midnight.utc - ].freeze - - scope :traces, -> { where(file_type: 3) } - scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } - scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) } - end - - def perform(start_id, end_id) - return unless Gitlab.com? - - JobArtifact.traces - .between(start_id, end_id) - .in_targeted_timestamps - .each_batch(of: BATCH_SIZE) { |batch| batch.update_all(expire_at: nil) } - end - end - end -end diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb deleted file mode 100644 index 4acef9029f9..00000000000 --- a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Remove vulnerability finding link records - # The records will be repopulated from the `raw_metadata` - # column of `vulnerability_occurrences` once the unique - # index is in place. - class RemoveVulnerabilityFindingLinks - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, stop_id) - define_batchable_model('vulnerability_finding_links', connection: ApplicationRecord.connection) - .where(id: start_id..stop_id) - .delete_all - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb deleted file mode 100644 index 190e2fc22fb..00000000000 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify duplicate runners_token_encrypted values in projects table in batches - class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'projects' - - scope :base_query, -> { where.not(runners_token_encrypted: nil) } - end - - def perform(start_id, end_id) - # Reset duplicate runner tokens that would prevent creating an unique index. - batch_records = Project.base_query.where(id: start_id..end_id) - - duplicate_tokens = Project.base_query - .where(runners_token_encrypted: batch_records.select(:runners_token_encrypted).distinct) - .group(:runners_token_encrypted) - .having('COUNT(*) > 1') - .pluck(:runners_token_encrypted) - - batch_records.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb deleted file mode 100644 index b58eefa0ab3..00000000000 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify duplicate ci_runners_token values in projects table in batches - class ResetDuplicateCiRunnersTokenValuesOnProjects - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'projects' - - scope :base_query, -> { where.not(runners_token: nil) } - end - - def perform(start_id, end_id) - # Reset duplicate runner tokens that would prevent creating an unique index. - batch_records = Project.base_query.where(id: start_id..end_id) - - duplicate_tokens = Project.base_query - .where(runners_token: batch_records.select(:runners_token).distinct) - .group(:runners_token) - .having('COUNT(*) > 1') - .pluck(:runners_token) - - batch_records.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb deleted file mode 100644 index b61f2ee7f4c..00000000000 --- a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class to populate spent_at for timelogs - class UpdateTimelogsNullSpentAt - include Gitlab::Database::DynamicModelHelpers - - BATCH_SIZE = 100 - - def perform(start_id, stop_id) - define_batchable_model('timelogs', connection: connection) - .where(spent_at: nil, id: start_id..stop_id) - .each_batch(of: 100) do |subbatch| - batch_start, batch_end = subbatch.pick('min(id), max(id)') - - update_timelogs(batch_start, batch_end) - end - end - - def update_timelogs(batch_start, batch_stop) - execute(<<~SQL) - UPDATE timelogs - SET spent_at = created_at - WHERE spent_at IS NULL - AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop}; - SQL - end - - def connection - @connection ||= ApplicationRecord.connection - end - - def execute(sql) - connection.execute(sql) - end - end - end -end |