From f64a639bcfa1fc2bc89ca7db268f594306edfd7c Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 16 Mar 2021 18:18:33 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-10-stable-ee --- ...ate_uuid_on_vulnerabilities_occurrences_spec.rb | 138 +++++++++++++++++++++ ...dd_environment_scope_to_group_variables_spec.rb | 46 +++++++ ...ith_bad_has_external_issue_tracker_data_spec.rb | 94 ++++++++++++++ ...l_from_namespaces_to_namespace_settings_spec.rb | 30 +++++ ...er_registry_enabled_to_project_features_spec.rb | 43 +++++++ .../reschedule_artifact_expiry_backfill_spec.rb | 38 ++++++ ...schedule_set_default_iteration_cadences_spec.rb | 41 ++++++ ...uest_assignees_migration_progress_check_spec.rb | 16 --- ...le_populate_personal_snippet_statistics_spec.rb | 10 +- 9 files changed, 437 insertions(+), 19 deletions(-) create mode 100644 spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb create mode 100644 spec/migrations/20210218040814_add_environment_scope_to_group_variables_spec.rb create mode 100644 spec/migrations/cleanup_projects_with_bad_has_external_issue_tracker_data_spec.rb create mode 100644 spec/migrations/migrate_delayed_project_removal_from_namespaces_to_namespace_settings_spec.rb create mode 100644 spec/migrations/move_container_registry_enabled_to_project_features_spec.rb create mode 100644 spec/migrations/reschedule_artifact_expiry_backfill_spec.rb create mode 100644 spec/migrations/reschedule_set_default_iteration_cadences_spec.rb delete mode 100644 spec/migrations/schedule_merge_request_assignees_migration_progress_check_spec.rb (limited to 'spec/migrations') diff --git a/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb b/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb new file mode 100644 index 00000000000..fce32be4683 --- /dev/null +++ b/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences.rb') + +RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences, :migration do + let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let(:users) { table(:users) } + let(:user) { create_user! } + let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) } + let(:scanners) { table(:vulnerability_scanners) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') } + let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerabilities_findings) { table(:vulnerability_occurrences) } + let(:vulnerability_identifiers) { table(:vulnerability_identifiers) } + let(:vulnerability_identifier) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: 'uuid-v5', + external_id: 'uuid-v5', + fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a', + name: 'Identifier for UUIDv5') + end + + let(:different_vulnerability_identifier) do + vulnerability_identifiers.create!( + project_id: project.id, + external_type: 'uuid-v4', + external_id: 'uuid-v4', + fingerprint: '772da93d34a1ba010bcb5efa9fb6f8e01bafcc89', + name: 'Identifier for UUIDv4') + end + + let!(:vulnerability_for_uuidv4) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let!(:vulnerability_for_uuidv5) do + create_vulnerability!( + project_id: project.id, + author_id: user.id + ) + end + + let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" } + let!(:finding_with_uuid_v4) do + create_finding!( + vulnerability_id: vulnerability_for_uuidv4.id, + project_id: project.id, + scanner_id: different_scanner.id, + primary_identifier_id: different_vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: "fa18f432f1d56675f4098d318739c3cd5b14eb3e", + uuid: known_uuid_v4 + ) + end + + let(:known_uuid_v5) { "e7d3d99d-04bb-5771-bb44-d80a9702d0a2" } + let!(:finding_with_uuid_v5) do + create_finding!( + vulnerability_id: vulnerability_for_uuidv5.id, + project_id: project.id, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + report_type: 0, # "sast" + location_fingerprint: "838574be0210968bf6b9f569df9c2576242cbf0a", + uuid: known_uuid_v5 + ) + end + + before do + stub_const("#{described_class}::BATCH_SIZE", 1) + end + + around do |example| + freeze_time { Sidekiq::Testing.fake! { example.run } } + end + + it 'schedules background migration' do + migrate! + + expect(BackgroundMigrationWorker.jobs.size).to eq(2) + expect(described_class::MIGRATION).to be_scheduled_migration(finding_with_uuid_v4.id, finding_with_uuid_v4.id) + expect(described_class::MIGRATION).to be_scheduled_migration(finding_with_uuid_v5.id, finding_with_uuid_v5.id) + end + + private + + def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0) + vulnerabilities.create!( + project_id: project_id, + author_id: author_id, + title: title, + severity: severity, + confidence: confidence, + report_type: report_type + ) + end + + # rubocop:disable Metrics/ParameterLists + def create_finding!( + vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:, + name: "test", severity: 7, confidence: 7, report_type: 0, + project_fingerprint: '123qweasdzxc', location_fingerprint: 'test', + metadata_version: 'test', raw_metadata: 'test', uuid: 'test') + vulnerabilities_findings.create!( + vulnerability_id: vulnerability_id, + project_id: project_id, + name: name, + severity: severity, + confidence: confidence, + report_type: report_type, + project_fingerprint: project_fingerprint, + scanner_id: scanner.id, + primary_identifier_id: vulnerability_identifier.id, + location_fingerprint: location_fingerprint, + metadata_version: metadata_version, + raw_metadata: raw_metadata, + uuid: uuid + ) + end + # rubocop:enable Metrics/ParameterLists + + def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.now, confirmed_at: Time.now) + users.create!( + name: name, + email: email, + username: name, + projects_limit: 0, + user_type: user_type, + confirmed_at: confirmed_at + ) + end +end diff --git a/spec/migrations/20210218040814_add_environment_scope_to_group_variables_spec.rb b/spec/migrations/20210218040814_add_environment_scope_to_group_variables_spec.rb new file mode 100644 index 00000000000..e525101f3a0 --- /dev/null +++ b/spec/migrations/20210218040814_add_environment_scope_to_group_variables_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration!('add_environment_scope_to_group_variables') + +RSpec.describe AddEnvironmentScopeToGroupVariables do + let(:migration) { described_class.new } + let(:ci_group_variables) { table(:ci_group_variables) } + let(:group) { table(:namespaces).create!(name: 'group', path: 'group') } + + def create_variable!(group, key:, environment_scope: '*') + table(:ci_group_variables).create!( + group_id: group.id, + key: key, + environment_scope: environment_scope + ) + end + + describe '#down' do + context 'group has variables with duplicate keys' do + it 'deletes all but the first record' do + migration.up + + remaining_variable = create_variable!(group, key: 'key') + create_variable!(group, key: 'key', environment_scope: 'staging') + create_variable!(group, key: 'key', environment_scope: 'production') + + migration.down + + expect(ci_group_variables.pluck(:id)).to eq [remaining_variable.id] + end + end + + context 'group does not have variables with duplicate keys' do + it 'does not delete any records' do + migration.up + + create_variable!(group, key: 'key') + create_variable!(group, key: 'staging') + create_variable!(group, key: 'production') + + expect { migration.down }.not_to change { ci_group_variables.count } + end + end + end +end diff --git a/spec/migrations/cleanup_projects_with_bad_has_external_issue_tracker_data_spec.rb b/spec/migrations/cleanup_projects_with_bad_has_external_issue_tracker_data_spec.rb new file mode 100644 index 00000000000..8aedd1f9607 --- /dev/null +++ b/spec/migrations/cleanup_projects_with_bad_has_external_issue_tracker_data_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe CleanupProjectsWithBadHasExternalIssueTrackerData, :migration do + let(:namespace) { table(:namespaces).create!(name: 'foo', path: 'bar') } + let(:projects) { table(:projects) } + let(:services) { table(:services) } + + def create_projects!(num) + Array.new(num) do + projects.create!(namespace_id: namespace.id) + end + end + + def create_active_external_issue_tracker_integrations!(*projects) + projects.each do |project| + services.create!(category: 'issue_tracker', project_id: project.id, active: true) + end + end + + def create_disabled_external_issue_tracker_integrations!(*projects) + projects.each do |project| + services.create!(category: 'issue_tracker', project_id: project.id, active: false) + end + end + + def create_active_other_integrations!(*projects) + projects.each do |project| + services.create!(category: 'not_an_issue_tracker', project_id: project.id, active: true) + end + end + + it 'sets `projects.has_external_issue_tracker` correctly' do + allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false) + + project_with_an_external_issue_tracker_1, + project_with_an_external_issue_tracker_2, + project_with_only_a_disabled_external_issue_tracker_1, + project_with_only_a_disabled_external_issue_tracker_2, + project_without_any_external_issue_trackers_1, + project_without_any_external_issue_trackers_2 = create_projects!(6) + + create_active_external_issue_tracker_integrations!( + project_with_an_external_issue_tracker_1, + project_with_an_external_issue_tracker_2 + ) + + create_disabled_external_issue_tracker_integrations!( + project_with_an_external_issue_tracker_1, + project_with_an_external_issue_tracker_2, + project_with_only_a_disabled_external_issue_tracker_1, + project_with_only_a_disabled_external_issue_tracker_2 + ) + + create_active_other_integrations!( + project_with_an_external_issue_tracker_1, + project_with_an_external_issue_tracker_2, + project_without_any_external_issue_trackers_1, + project_without_any_external_issue_trackers_2 + ) + + # PG triggers on the services table added in + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51852 will have set + # the `has_external_issue_tracker` columns to correct data when the services + # records were created above. + # + # We set the `has_external_issue_tracker` columns for projects to incorrect + # data manually below to emulate projects in a state before the PG + # triggers were added. + project_with_an_external_issue_tracker_2.update!(has_external_issue_tracker: false) + project_with_only_a_disabled_external_issue_tracker_2.update!(has_external_issue_tracker: true) + project_without_any_external_issue_trackers_2.update!(has_external_issue_tracker: true) + + migrate! + + expected_true = [ + project_with_an_external_issue_tracker_1, + project_with_an_external_issue_tracker_2 + ].each(&:reload).map(&:has_external_issue_tracker) + + expected_not_true = [ + project_without_any_external_issue_trackers_1, + project_without_any_external_issue_trackers_2, + project_with_only_a_disabled_external_issue_tracker_1, + project_with_only_a_disabled_external_issue_tracker_2 + ].each(&:reload).map(&:has_external_issue_tracker) + + expect(expected_true).to all(eq(true)) + expect(expected_not_true).to all(be_falsey) + end +end diff --git a/spec/migrations/migrate_delayed_project_removal_from_namespaces_to_namespace_settings_spec.rb b/spec/migrations/migrate_delayed_project_removal_from_namespaces_to_namespace_settings_spec.rb new file mode 100644 index 00000000000..28a8dcf0d4c --- /dev/null +++ b/spec/migrations/migrate_delayed_project_removal_from_namespaces_to_namespace_settings_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require Rails.root.join('db', 'post_migrate', '20210215095328_migrate_delayed_project_removal_from_namespaces_to_namespace_settings.rb') + +RSpec.describe MigrateDelayedProjectRemovalFromNamespacesToNamespaceSettings, :migration do + let(:namespaces) { table(:namespaces) } + let(:namespace_settings) { table(:namespace_settings) } + + let!(:namespace_wo_settings) { namespaces.create!(name: generate(:name), path: generate(:name), delayed_project_removal: true) } + let!(:namespace_wo_settings_delay_false) { namespaces.create!(name: generate(:name), path: generate(:name), delayed_project_removal: false) } + let!(:namespace_w_settings_delay_true) { namespaces.create!(name: generate(:name), path: generate(:name), delayed_project_removal: true) } + let!(:namespace_w_settings_delay_false) { namespaces.create!(name: generate(:name), path: generate(:name), delayed_project_removal: false) } + + let!(:namespace_settings_delay_true) { namespace_settings.create!(namespace_id: namespace_w_settings_delay_true.id, delayed_project_removal: false, created_at: DateTime.now, updated_at: DateTime.now) } + let!(:namespace_settings_delay_false) { namespace_settings.create!(namespace_id: namespace_w_settings_delay_false.id, delayed_project_removal: false, created_at: DateTime.now, updated_at: DateTime.now) } + + it 'migrates delayed_project_removal to namespace_settings' do + disable_migrations_output { migrate! } + + expect(namespace_settings.count).to eq(3) + + expect(namespace_settings.find_by(namespace_id: namespace_wo_settings.id).delayed_project_removal).to eq(true) + expect(namespace_settings.find_by(namespace_id: namespace_wo_settings_delay_false.id)).to be_nil + + expect(namespace_settings_delay_true.reload.delayed_project_removal).to eq(true) + expect(namespace_settings_delay_false.reload.delayed_project_removal).to eq(false) + end +end diff --git a/spec/migrations/move_container_registry_enabled_to_project_features_spec.rb b/spec/migrations/move_container_registry_enabled_to_project_features_spec.rb new file mode 100644 index 00000000000..c7b07f3ef37 --- /dev/null +++ b/spec/migrations/move_container_registry_enabled_to_project_features_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20210226120851_move_container_registry_enabled_to_project_features.rb') + +RSpec.describe MoveContainerRegistryEnabledToProjectFeatures, :migration do + let(:namespace) { table(:namespaces).create!(name: 'gitlab', path: 'gitlab-org') } + + let!(:projects) do + [ + table(:projects).create!(namespace_id: namespace.id, name: 'project 1'), + table(:projects).create!(namespace_id: namespace.id, name: 'project 2'), + table(:projects).create!(namespace_id: namespace.id, name: 'project 3'), + table(:projects).create!(namespace_id: namespace.id, name: 'project 4') + ] + end + + before do + stub_const("#{described_class.name}::BATCH_SIZE", 3) + end + + around do |example| + Sidekiq::Testing.fake! do + freeze_time do + example.call + end + end + end + + it 'schedules jobs for ranges of projects' do + migrate! + + expect(described_class::MIGRATION) + .to be_scheduled_delayed_migration(2.minutes, projects[0].id, projects[2].id) + + expect(described_class::MIGRATION) + .to be_scheduled_delayed_migration(4.minutes, projects[3].id, projects[3].id) + end + + it 'schedules jobs according to the configured batch size' do + expect { migrate! }.to change { BackgroundMigrationWorker.jobs.size }.by(2) + end +end diff --git a/spec/migrations/reschedule_artifact_expiry_backfill_spec.rb b/spec/migrations/reschedule_artifact_expiry_backfill_spec.rb new file mode 100644 index 00000000000..532035849cb --- /dev/null +++ b/spec/migrations/reschedule_artifact_expiry_backfill_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require Rails.root.join('db', 'post_migrate', '20210224150506_reschedule_artifact_expiry_backfill.rb') + +RSpec.describe RescheduleArtifactExpiryBackfill, :migration do + let(:migration_class) { Gitlab::BackgroundMigration::BackfillArtifactExpiryDate } + let(:migration_name) { migration_class.to_s.demodulize } + + before do + table(:namespaces).create!(id: 123, name: 'test_namespace', path: 'test_namespace') + table(:projects).create!(id: 123, name: 'sample_project', path: 'sample_project', namespace_id: 123) + end + + it 'correctly schedules background migrations' do + first_artifact = create_artifact(job_id: 0, expire_at: nil, created_at: Date.new(2020, 06, 21)) + second_artifact = create_artifact(job_id: 1, expire_at: nil, created_at: Date.new(2020, 06, 21)) + create_artifact(job_id: 2, expire_at: Date.yesterday, created_at: Date.new(2020, 06, 21)) + create_artifact(job_id: 3, expire_at: nil, created_at: Date.new(2020, 06, 23)) + + Sidekiq::Testing.fake! do + freeze_time do + migrate! + + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + expect(migration_name).to be_scheduled_migration_with_multiple_args(first_artifact.id, second_artifact.id) + end + end + end + + private + + def create_artifact(params) + table(:ci_builds).create!(id: params[:job_id], project_id: 123) + table(:ci_job_artifacts).create!(project_id: 123, file_type: 1, **params) + end +end diff --git a/spec/migrations/reschedule_set_default_iteration_cadences_spec.rb b/spec/migrations/reschedule_set_default_iteration_cadences_spec.rb new file mode 100644 index 00000000000..fb629c90d9f --- /dev/null +++ b/spec/migrations/reschedule_set_default_iteration_cadences_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe RescheduleSetDefaultIterationCadences do + let(:namespaces) { table(:namespaces) } + let(:iterations) { table(:sprints) } + + let(:group_1) { namespaces.create!(name: 'test_1', path: 'test_1') } + let!(:group_2) { namespaces.create!(name: 'test_2', path: 'test_2') } + let(:group_3) { namespaces.create!(name: 'test_3', path: 'test_3') } + let(:group_4) { namespaces.create!(name: 'test_4', path: 'test_4') } + let(:group_5) { namespaces.create!(name: 'test_5', path: 'test_5') } + let(:group_6) { namespaces.create!(name: 'test_6', path: 'test_6') } + let(:group_7) { namespaces.create!(name: 'test_7', path: 'test_7') } + let(:group_8) { namespaces.create!(name: 'test_8', path: 'test_8') } + + let!(:iteration_1) { iterations.create!(iid: 1, title: 'iteration 1', group_id: group_1.id, start_date: 2.days.from_now, due_date: 3.days.from_now) } + let!(:iteration_2) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_3.id, start_date: 2.days.from_now, due_date: 3.days.from_now) } + let!(:iteration_3) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_4.id, start_date: 2.days.from_now, due_date: 3.days.from_now) } + let!(:iteration_4) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_5.id, start_date: 2.days.from_now, due_date: 3.days.from_now) } + let!(:iteration_5) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_6.id, start_date: 2.days.from_now, due_date: 3.days.from_now) } + let!(:iteration_6) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_7.id, start_date: 2.days.from_now, due_date: 3.days.from_now) } + let!(:iteration_7) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_8.id, start_date: 2.days.from_now, due_date: 3.days.from_now) } + + around do |example| + freeze_time { Sidekiq::Testing.fake! { example.run } } + end + + it 'schedules the background jobs', :aggregate_failures do + stub_const("#{described_class.name}::BATCH_SIZE", 3) + + migrate! + + expect(BackgroundMigrationWorker.jobs.size).to be(3) + expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(2.minutes, group_1.id, group_3.id, group_4.id) + expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(4.minutes, group_5.id, group_6.id, group_7.id) + expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(6.minutes, group_8.id) + end +end diff --git a/spec/migrations/schedule_merge_request_assignees_migration_progress_check_spec.rb b/spec/migrations/schedule_merge_request_assignees_migration_progress_check_spec.rb deleted file mode 100644 index 0a69f49f10d..00000000000 --- a/spec/migrations/schedule_merge_request_assignees_migration_progress_check_spec.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20190402224749_schedule_merge_request_assignees_migration_progress_check.rb') - -RSpec.describe ScheduleMergeRequestAssigneesMigrationProgressCheck do - describe '#up' do - it 'schedules MergeRequestAssigneesMigrationProgressCheck background job' do - expect(BackgroundMigrationWorker).to receive(:perform_async) - .with(described_class::MIGRATION) - .and_call_original - - subject.up - end - end -end diff --git a/spec/migrations/schedule_populate_personal_snippet_statistics_spec.rb b/spec/migrations/schedule_populate_personal_snippet_statistics_spec.rb index 8678361fc64..faa993dfbc7 100644 --- a/spec/migrations/schedule_populate_personal_snippet_statistics_spec.rb +++ b/spec/migrations/schedule_populate_personal_snippet_statistics_spec.rb @@ -6,11 +6,15 @@ require Rails.root.join('db', 'post_migrate', '20200714075739_schedule_populate_ RSpec.describe SchedulePopulatePersonalSnippetStatistics do let(:users) { table(:users) } + let(:namespaces) { table(:namespaces) } let(:snippets) { table(:snippets) } let(:projects) { table(:projects) } - let(:user1) { users.create!(id: 1, email: 'user1@example.com', projects_limit: 10, username: 'test1', name: 'Test1', state: 'active') } - let(:user2) { users.create!(id: 2, email: 'user2@example.com', projects_limit: 10, username: 'test2', name: 'Test2', state: 'active') } - let(:user3) { users.create!(id: 3, email: 'user3@example.com', projects_limit: 10, username: 'test3', name: 'Test3', state: 'active') } + let!(:user1) { users.create!(id: 1, email: 'user1@example.com', projects_limit: 10, username: 'test1', name: 'Test1', state: 'active') } + let!(:user2) { users.create!(id: 2, email: 'user2@example.com', projects_limit: 10, username: 'test2', name: 'Test2', state: 'active') } + let!(:user3) { users.create!(id: 3, email: 'user3@example.com', projects_limit: 10, username: 'test3', name: 'Test3', state: 'active') } + let!(:namespace1) { namespaces.create!(id: 1, owner_id: user1.id, name: 'test1', path: 'test1') } + let!(:namespace2) { namespaces.create!(id: 2, owner_id: user2.id, name: 'test2', path: 'test2') } + let!(:namespace3) { namespaces.create!(id: 3, owner_id: user3.id, name: 'test3', path: 'test3') } def create_snippet(id, user_id, type = 'PersonalSnippet') params = { -- cgit v1.2.3