diff options
Diffstat (limited to 'spec/lib/gitlab/background_migration')
9 files changed, 599 insertions, 48 deletions
diff --git a/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb new file mode 100644 index 00000000000..fc4597fbb96 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillHasMergeRequestOfVulnerabilityReads, schema: 20230907155247, feature_category: :database do # rubocop:disable Layout/LineLength + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:scanners) { table(:vulnerability_scanners) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_reads) { table(:vulnerability_reads) } + let(:merge_requests) { table(:merge_requests) } + let(:merge_request_links) { table(:vulnerability_merge_request_links) } + + let(:namespace) { namespaces.create!(name: 'user', path: 'user') } + let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } + let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'external_id', name: 'Test Scanner') } + + let(:vulnerability) do + vulnerabilities.create!( + project_id: project.id, + author_id: user.id, + title: 'test', + severity: 1, + confidence: 1, + report_type: 1 + ) + end + + let(:merge_request) do + merge_requests.create!( + target_project_id: project.id, + source_branch: "other", + target_branch: "main", + author_id: user.id, + title: 'Feedback Merge Request' + ) + end + + let!(:vulnerability_read) do + vulnerability_reads.create!( + project_id: project.id, + vulnerability_id: vulnerability.id, + scanner_id: scanner.id, + severity: 1, + report_type: 1, + state: 1, + uuid: SecureRandom.uuid + ) + end + + let!(:merge_request_link) do + merge_request_links.create!( + vulnerability_id: vulnerability.id, merge_request_id: merge_request.id) + end + + subject(:perform_migration) do + described_class.new( + start_id: vulnerability_reads.first.vulnerability_id, + end_id: vulnerability_reads.last.vulnerability_id, + batch_table: :vulnerability_reads, + batch_column: :vulnerability_id, + sub_batch_size: vulnerability_reads.count, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + before do + # Unset since the trigger already sets during merge_request_link creation. + vulnerability_reads.update_all(has_merge_request: false) + end + + it 'sets the has_merge_request of existing record' do + expect { perform_migration }.to change { vulnerability_read.reload.has_merge_request }.from(false).to(true) + end + + it 'does not modify has_merge_request of other vulnerabilities which do not have merge request' do + vulnerability_2 = vulnerabilities.create!( + project_id: project.id, + author_id: user.id, + title: 'test 2', + severity: 1, + confidence: 1, + report_type: 1 + ) + + vulnerability_read_2 = vulnerability_reads.create!( + project_id: project.id, + vulnerability_id: vulnerability_2.id, + scanner_id: scanner.id, + severity: 1, + report_type: 1, + state: 1, + uuid: SecureRandom.uuid + ) + + expect { perform_migration }.not_to change { vulnerability_read_2.reload.has_merge_request }.from(false) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_nuget_normalized_version_spec.rb b/spec/lib/gitlab/background_migration/backfill_nuget_normalized_version_spec.rb new file mode 100644 index 00000000000..3f0bd417955 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_nuget_normalized_version_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillNugetNormalizedVersion, schema: 20230811103457, + feature_category: :package_registry do + let(:packages_nuget_metadata) { table(:packages_nuget_metadata) } + let(:versions) do + { + '1' => '1.0.0', + '1.0' => '1.0.0', + '1.0.0' => '1.0.0', + '1.00' => '1.0.0', + '1.00.01' => '1.0.1', + '1.01.1' => '1.1.1', + '1.0.0.0' => '1.0.0', + '1.0.01.0' => '1.0.1', + '1.0.7+r3456' => '1.0.7', + '1.0.0-Alpha' => '1.0.0-alpha', + '1.00.05-alpha.0' => '1.0.5-alpha.0' + } + end + + let!(:migration_attrs) do + { + start_id: packages_nuget_metadata.minimum(:package_id), + end_id: packages_nuget_metadata.maximum(:package_id), + batch_table: :packages_nuget_metadata, + batch_column: :package_id, + sub_batch_size: 1000, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let(:migration) { described_class.new(**migration_attrs) } + let(:packages) { table(:packages_packages) } + + let(:namespace) { table(:namespaces).create!(name: 'project', path: 'project', type: 'Project') } + let(:project) do + table(:projects).create!(name: 'project', path: 'project', project_namespace_id: namespace.id, + namespace_id: namespace.id) + end + + let(:package_ids) { [] } + + subject(:perform_migration) { migration.perform } + + before do + versions.each_key do |version| + packages.create!(name: 'test', version: version, package_type: 4, project_id: project.id).tap do |package| + package_ids << package.id + packages_nuget_metadata.create!(package_id: package.id) + end + end + end + + it 'executes 5 queries and updates the normalized_version column' do + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + # each_batch lower bound query + # each_batch upper bound query + # SELECT packages_nuget_metadata.package_id FROM packages_nuget_metadata.... + # SELECT packages_packages.id, packages_packages.version FROM packages_packages.... + # UPDATE packages_nuget_metadata SET normalized_version =.... + expect(queries.count).to eq(5) + + expect( + packages_nuget_metadata.where(package_id: package_ids).pluck(:normalized_version) + ).to match_array(versions.values) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_project_statistics_storage_size_with_recent_size_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_statistics_storage_size_with_recent_size_spec.rb new file mode 100644 index 00000000000..2884fb9b10b --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_statistics_storage_size_with_recent_size_spec.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithRecentSize, + schema: 20230823090001, + feature_category: :consumables_cost_management do + include MigrationHelpers::ProjectStatisticsHelper + + include_context 'when backfilling project statistics' + + let(:recent_size_enabled_at) { described_class::RECENT_OBJECTS_SIZE_ENABLED_AT } + let(:default_stats) do + { + repository_size: 1, + wiki_size: 1, + lfs_objects_size: 1, + build_artifacts_size: 1, + packages_size: 1, + snippets_size: 1, + uploads_size: 1, + storage_size: default_storage_size, + updated_at: recent_size_enabled_at - 1.month + } + end + + describe '#filter_batch' do + let!(:project_statistics) { generate_records(default_projects, project_statistics_table, default_stats) } + let!(:expected) { project_statistics.map(&:id) } + + it 'filters out project_statistics with no repository_size' do + project_statistics_table.create!( + project_id: proj5.id, + namespace_id: proj5.namespace_id, + repository_size: 0, + wiki_size: 1, + lfs_objects_size: 1, + build_artifacts_size: 1, + packages_size: 1, + snippets_size: 1, + uploads_size: 1, + storage_size: 6, + updated_at: recent_size_enabled_at - 1.month + ) + + actual = migration.filter_batch(project_statistics_table).pluck(:id) + + expect(actual).to match_array(expected) + end + + shared_examples 'filters out project_statistics updated since recent objects went live' do + it 'filters out project_statistics updated since recent objects went live' do + project_statistics_table.create!( + project_id: proj5.id, + namespace_id: proj5.namespace_id, + repository_size: 10, + wiki_size: 1, + lfs_objects_size: 1, + build_artifacts_size: 1, + packages_size: 1, + snippets_size: 1, + uploads_size: 1, + storage_size: 6, + updated_at: recent_size_enabled_at + 1.month + ) + + actual = migration.filter_batch(project_statistics_table).pluck(:id) + + expect(actual).to match_array(expected) + end + end + + context 'when on GitLab.com' do + before do + allow(Gitlab).to receive(:org_or_com?).and_return(true) + end + + it_behaves_like 'filters out project_statistics updated since recent objects went live' + end + + context 'when Gitlab.dev_or_test_env? is true ' do + before do + allow(Gitlab).to receive(:dev_or_test_env?).and_return(true) + end + + it_behaves_like 'filters out project_statistics updated since recent objects went live' + end + + context 'when on self-managed' do + before do + allow(Gitlab).to receive(:dev_or_test_env?).and_return(false) + allow(Gitlab).to receive(:org_or_com?).and_return(false) + end + + it 'does not filter out project_statistics updated since recent objects went live' do + latest = project_statistics_table.create!( + project_id: proj5.id, + namespace_id: proj5.namespace_id, + repository_size: 10, + wiki_size: 1, + lfs_objects_size: 1, + build_artifacts_size: 1, + packages_size: 1, + snippets_size: 1, + uploads_size: 1, + storage_size: 6, + updated_at: recent_size_enabled_at + 1.month + ) + + actual = migration.filter_batch(project_statistics_table).pluck(:id) + + expect(actual).to match_array(expected.push(latest.id)) + end + end + end + + describe '#perform' do + subject(:perform_migration) { migration.perform } + + before do + allow_next_instance_of(Repository) do |repo| + allow(repo).to receive(:recent_objects_size).and_return(10) + end + end + + context 'when project_statistics backfill runs' do + before do + generate_records(default_projects, project_statistics_table, default_stats) + allow(::Namespaces::ScheduleAggregationWorker).to receive(:perform_async) + end + + it 'uses repository#recent_objects_size for repository_size' do + project_statistics = create_project_stats(projects, namespaces, default_stats) + migration = create_migration(end_id: project_statistics.project_id) + + migration.perform + + project_statistics.reload + expect(project_statistics.storage_size).to eq(6 + 10.megabytes) + end + end + + it 'coerces a null wiki_size to 0' do + project_statistics = create_project_stats(projects, namespaces, default_stats, { wiki_size: nil }) + allow(::Namespaces::ScheduleAggregationWorker).to receive(:perform_async) + migration = create_migration(end_id: project_statistics.project_id) + + migration.perform + + project_statistics.reload + expect(project_statistics.storage_size).to eq(5 + 10.megabytes) + end + + it 'coerces a null snippets_size to 0' do + project_statistics = create_project_stats(projects, namespaces, default_stats, { snippets_size: nil }) + allow(::Namespaces::ScheduleAggregationWorker).to receive(:perform_async) + migration = create_migration(end_id: project_statistics.project_id) + + migration.perform + + project_statistics.reload + expect(project_statistics.storage_size).to eq(5 + 10.megabytes) + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index 9f76e4131b2..06b66b599ab 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -250,7 +250,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat end context 'when user name is invalid' do - let(:user_name) { '.' } + let(:user_name) { ',' } let!(:snippet) { snippets.create!(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) } let(:ids) { [4, 4] } @@ -262,7 +262,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat end context 'when both user name and snippet file_name are invalid' do - let(:user_name) { '.' } + let(:user_name) { ',' } let!(:other_user) do users.create!( id: 2, diff --git a/spec/lib/gitlab/background_migration/backfill_user_preferences_with_defaults_spec.rb b/spec/lib/gitlab/background_migration/backfill_user_preferences_with_defaults_spec.rb new file mode 100644 index 00000000000..b66b930b7ac --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_user_preferences_with_defaults_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillUserPreferencesWithDefaults, + schema: 20230818085219, + feature_category: :user_profile do + let(:user_preferences) { table(:user_preferences) } + let(:users) { table(:users) } + let(:columns) { [:tab_width, :time_display_relative, :render_whitespace_in_code] } + let(:initial_column_values) do + [ + [nil, nil, nil], + [10, nil, nil], + [nil, false, nil], + [nil, nil, true] + ] + .map { |row| columns.zip(row).to_h } + end + + let(:final_column_values) do + [ + [8, true, false], + [10, true, false], + [8, false, false], + [8, true, true] + ] + .map { |row| columns.zip(row).to_h } + end + + subject(:perform_migration) do + described_class + .new( + start_id: user_preferences.minimum(:id), + end_id: user_preferences.maximum(:id), + batch_table: :user_preferences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + .perform + end + + before do + initial_column_values.each_with_index do |attributes, index| + user = users.create!(projects_limit: 1, email: "user#{index}@gitlab.com") + user_preference = user_preferences.create!(attributes.merge(user_id: user.id)) + final_column_values[index].merge!(id: user_preference.id) + end + end + + it 'backfills the null values with the default values' do + perform_migration + + final_column_values.each { |attributes| match_attributes(attributes) } + end + + def match_attributes(attributes) + migrated_user_preference = user_preferences.find(attributes[:id]) + + expect(migrated_user_preference.tab_width).to eq(attributes[:tab_width]) + expect(migrated_user_preference.time_display_relative).to eq(attributes[:time_display_relative]) + expect(migrated_user_preference.render_whitespace_in_code).to eq(attributes[:render_whitespace_in_code]) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_users_with_defaults_spec.rb b/spec/lib/gitlab/background_migration/backfill_users_with_defaults_spec.rb new file mode 100644 index 00000000000..78f36933435 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_users_with_defaults_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillUsersWithDefaults, + schema: 20230818083610, + feature_category: :user_profile do + let(:users) { table(:users) } + let(:columns) { [:project_view, :hide_no_ssh_key, :hide_no_password, :notified_of_own_activity] } + let(:initial_column_values) do + [ + [nil, nil, nil, nil], + [0, nil, nil, nil], + [nil, true, nil, nil], + [nil, nil, true, nil], + [nil, nil, nil, true] + ] + .map { |row| columns.zip(row).to_h } + end + + let(:final_column_values) do + [ + [2, false, false, false], + [0, false, false, false], + [2, true, false, false], + [2, false, true, false], + [2, false, false, true] + ] + .map { |row| columns.zip(row).to_h } + end + + subject(:perform_migration) do + described_class + .new( + start_id: users.minimum(:id), + end_id: users.maximum(:id), + batch_table: :users, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + .perform + end + + before do + initial_column_values.each_with_index do |attributes, index| + user = users.create!(**attributes.merge(projects_limit: 1, email: "user#{index}@gitlab.com")) + final_column_values[index].merge!(id: user.id) + end + end + + it 'backfills the null values with the default values' do + perform_migration + + final_column_values.each { |attributes| match_attributes(attributes) } + end + + private + + def match_attributes(attributes) + migrated_user = users.find(attributes[:id]) + expect(migrated_user.project_view).to eq(attributes[:project_view]) + expect(migrated_user.hide_no_ssh_key).to eq(attributes[:hide_no_ssh_key]) + expect(migrated_user.hide_no_password).to eq(attributes[:hide_no_password]) + expect(migrated_user.notified_of_own_activity).to eq(attributes[:notified_of_own_activity]) + end +end diff --git a/spec/lib/gitlab/background_migration/convert_credit_card_validation_data_to_hashes_spec.rb b/spec/lib/gitlab/background_migration/convert_credit_card_validation_data_to_hashes_spec.rb new file mode 100644 index 00000000000..97f69afca55 --- /dev/null +++ b/spec/lib/gitlab/background_migration/convert_credit_card_validation_data_to_hashes_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::ConvertCreditCardValidationDataToHashes, schema: 20230821081603, feature_category: :user_profile do # rubocop:disable Layout/LineLength + let(:users_table) { table(:users) } + let(:credit_card_validations_table) { table(:user_credit_card_validations) } + let(:rows) { 5 } + + describe '#perform' do + let(:network) { 'Visa' } + let(:holder_name) { 'John Smith' } + let(:last_digits) { 1111 } + let(:expiration_date) { 1.year.from_now.to_date } + + subject(:perform_migration) do + described_class.new( + start_id: 1, + end_id: rows, + batch_table: :user_credit_card_validations, + batch_column: :user_id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + before do + (1..rows).each do |i| + users_table.create!(id: i, username: "John #{i}", email: "johndoe_#{i}@gitlab.com", projects_limit: 10) + + credit_card_validations_table.create!( + id: i, + user_id: i, + network: network, + holder_name: holder_name, + last_digits: last_digits, + expiration_date: expiration_date, + credit_card_validated_at: Date.today + ) + end + end + + it 'updates values to hash for records in the specified batch', :aggregate_failures do + perform_migration + + (1..rows).each do |i| + credit_card = credit_card_validations_table.find_by(user_id: i) + + expect(credit_card.last_digits_hash).to eq(hashed_value(last_digits)) + expect(credit_card.holder_name_hash).to eq(hashed_value(holder_name.downcase)) + expect(credit_card.network_hash).to eq(hashed_value(network.downcase)) + expect(credit_card.expiration_date_hash).to eq(hashed_value(expiration_date.to_s)) + end + end + + context 'with NULL columns' do + let(:network) { nil } + let(:holder_name) { nil } + let(:last_digits) { nil } + let(:expiration_date) { nil } + + it 'does not update values for records in the specified batch', :aggregate_failures do + perform_migration + + (1..rows).each do |i| + credit_card = credit_card_validations_table.find_by(user_id: i) + + expect(credit_card.last_digits_hash).to eq(nil) + expect(credit_card.holder_name_hash).to eq(nil) + expect(credit_card.network_hash).to eq(nil) + expect(credit_card.expiration_date_hash).to eq(nil) + end + end + end + end + + def hashed_value(value) + Gitlab::CryptoHelper.sha256(value) + end +end diff --git a/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb b/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb deleted file mode 100644 index 195e57e4e59..00000000000 --- a/spec/lib/gitlab/background_migration/rebalance_partition_id_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RebalancePartitionId, - :migration, - schema: 20230125093723, - feature_category: :continuous_integration do - let(:ci_builds_table) { table(:ci_builds, database: :ci) } - let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) } - - let!(:valid_ci_pipeline) { ci_pipelines_table.create!(id: 1, partition_id: 100) } - let!(:invalid_ci_pipeline) { ci_pipelines_table.create!(id: 2, partition_id: 101) } - - describe '#perform' do - using RSpec::Parameterized::TableSyntax - - where(:table_name, :invalid_record, :valid_record) do - :ci_pipelines | invalid_ci_pipeline | valid_ci_pipeline - end - - subject(:perform) do - described_class.new( - start_id: 1, - end_id: 2, - batch_table: table_name, - batch_column: :id, - sub_batch_size: 1, - pause_ms: 0, - connection: Ci::ApplicationRecord.connection - ).perform - end - - shared_examples 'fix invalid records' do - it 'rebalances partition_id to 100 when partition_id is 101' do - expect { perform } - .to change { invalid_record.reload.partition_id }.from(101).to(100) - .and not_change { valid_record.reload.partition_id } - end - end - - with_them do - it_behaves_like 'fix invalid records' - end - end -end diff --git a/spec/lib/gitlab/background_migration/update_users_set_external_if_service_account_spec.rb b/spec/lib/gitlab/background_migration/update_users_set_external_if_service_account_spec.rb new file mode 100644 index 00000000000..19ad70337dc --- /dev/null +++ b/spec/lib/gitlab/background_migration/update_users_set_external_if_service_account_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::UpdateUsersSetExternalIfServiceAccount, feature_category: :system_access do + describe "#perform" do + let(:users_table) { table(:users) } + let(:service_account_user) do + users_table.create!(username: 'john_doe', email: 'johndoe@gitlab.com', + user_type: HasUserType::USER_TYPES[:service_account], projects_limit: 5) + end + + let(:service_user) do + users_table.create!(username: 'john_doe2', email: 'johndoe2@gitlab.com', + user_type: HasUserType::USER_TYPES[:service_user], projects_limit: 5) + end + + let(:table_name) { :users } + let(:batch_column) { :id } + let(:sub_batch_size) { 2 } + let(:pause_ms) { 0 } + let(:migration) do + described_class.new( + start_id: service_account_user.id, end_id: service_user.id, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) do + migration.perform + end + + it "changes external field for service_account user" do + perform_migration + + expect(service_account_user.reload.external).to eq(true) + expect(service_user.reload.external).to eq(false) + end + end +end |