Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40

9 files changed, 437 insertions, 19 deletions

diff --git a/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb b/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb
new file mode 100644
index 00000000000..fce32be4683
--- /dev/null
+++ b/spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb
@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences.rb')
+
+RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences, :migration do
+  let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:users) { table(:users) }
+  let(:user) { create_user! }
+  let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
+  let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
+  let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
+  let(:vulnerability_identifier) do
+    vulnerability_identifiers.create!(
+      project_id: project.id,
+      external_type: 'uuid-v5',
+      external_id: 'uuid-v5',
+      fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a',
+      name: 'Identifier for UUIDv5')
+  end
+
+  let(:different_vulnerability_identifier) do
+    vulnerability_identifiers.create!(
+      project_id: project.id,
+      external_type: 'uuid-v4',
+      external_id: 'uuid-v4',
+      fingerprint: '772da93d34a1ba010bcb5efa9fb6f8e01bafcc89',
+      name: 'Identifier for UUIDv4')
+  end
+
+  let!(:vulnerability_for_uuidv4) do
+    create_vulnerability!(
+      project_id: project.id,
+      author_id: user.id
+    )
+  end
+
+  let!(:vulnerability_for_uuidv5) do
+    create_vulnerability!(
+      project_id: project.id,
+      author_id: user.id
+    )
+  end
+
+  let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" }
+  let!(:finding_with_uuid_v4) do
+    create_finding!(
+      vulnerability_id: vulnerability_for_uuidv4.id,
+      project_id: project.id,
+      scanner_id: different_scanner.id,
+      primary_identifier_id: different_vulnerability_identifier.id,
+      report_type: 0, # "sast"
+      location_fingerprint: "fa18f432f1d56675f4098d318739c3cd5b14eb3e",
+      uuid: known_uuid_v4
+    )
+  end
+
+  let(:known_uuid_v5) { "e7d3d99d-04bb-5771-bb44-d80a9702d0a2" }
+  let!(:finding_with_uuid_v5) do
+    create_finding!(
+      vulnerability_id: vulnerability_for_uuidv5.id,
+      project_id: project.id,
+      scanner_id: scanner.id,
+      primary_identifier_id: vulnerability_identifier.id,
+      report_type: 0, # "sast"
+      location_fingerprint: "838574be0210968bf6b9f569df9c2576242cbf0a",
+      uuid: known_uuid_v5
+    )
+  end
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+  end
+
+  around do |example|
+    freeze_time { Sidekiq::Testing.fake! { example.run } }
+  end
+
+  it 'schedules background migration' do
+    migrate!
+
+    expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+    expect(described_class::MIGRATION).to be_scheduled_migration(finding_with_uuid_v4.id, finding_with_uuid_v4.id)
+    expect(described_class::MIGRATION).to be_scheduled_migration(finding_with_uuid_v5.id, finding_with_uuid_v5.id)
+  end
+
+  private
+
+  def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
+    vulnerabilities.create!(
+      project_id: project_id,
+      author_id: author_id,
+      title: title,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type
+    )
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+  def create_finding!(
+    vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
+                      name: "test", severity: 7, confidence: 7, report_type: 0,
+                      project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
+                      metadata_version: 'test', raw_metadata: 'test', uuid: 'test')
+    vulnerabilities_findings.create!(
+      vulnerability_id: vulnerability_id,
+      project_id: project_id,
+      name: name,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_fingerprint: project_fingerprint,
+      scanner_id: scanner.id,
+      primary_identifier_id: vulnerability_identifier.id,
+      location_fingerprint: location_fingerprint,
+      metadata_version: metadata_version,
+      raw_metadata: raw_metadata,
+      uuid: uuid
+    )
+  end
+  # rubocop:enable Metrics/ParameterLists
+
+  def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.now, confirmed_at: Time.now)
+    users.create!(
+      name: name,
+      email: email,
+      username: name,
+      projects_limit: 0,
+      user_type: user_type,
+      confirmed_at: confirmed_at
+    )
+  end
+end
diff --git a/spec/migrations/20210218040814_add_environment_scope_to_group_variables_spec.rb b/spec/migrations/20210218040814_add_environment_scope_to_group_variables_spec.rb
new file mode 100644
index 00000000000..e525101f3a0
--- /dev/null
+++ b/spec/migrations/20210218040814_add_environment_scope_to_group_variables_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!('add_environment_scope_to_group_variables')
+
+RSpec.describe AddEnvironmentScopeToGroupVariables do
+  let(:migration) { described_class.new }
+  let(:ci_group_variables) { table(:ci_group_variables) }
+  let(:group) { table(:namespaces).create!(name: 'group', path: 'group') }
+
+  def create_variable!(group, key:, environment_scope: '*')
+    table(:ci_group_variables).create!(
+      group_id: group.id,
+      key: key,
+      environment_scope: environment_scope
+    )
+  end
+
+  describe '#down' do
+    context 'group has variables with duplicate keys' do
+      it 'deletes all but the first record' do
+        migration.up
+
+        remaining_variable = create_variable!(group, key: 'key')
+        create_variable!(group, key: 'key', environment_scope: 'staging')
+        create_variable!(group, key: 'key', environment_scope: 'production')
+
+        migration.down
+
+        expect(ci_group_variables.pluck(:id)).to eq [remaining_variable.id]
+      end
+    end
+
+    context 'group does not have variables with duplicate keys' do
+      it 'does not delete any records' do
+        migration.up
+
+        create_variable!(group, key: 'key')
+        create_variable!(group, key: 'staging')
+        create_variable!(group, key: 'production')
+
+        expect { migration.down }.not_to change { ci_group_variables.count }
+      end
+    end
+  end
+end
diff --git a/spec/migrations/cleanup_projects_with_bad_has_external_issue_tracker_data_spec.rb b/spec/migrations/cleanup_projects_with_bad_has_external_issue_tracker_data_spec.rb
new file mode 100644
index 00000000000..8aedd1f9607
--- /dev/null
+++ b/spec/migrations/cleanup_projects_with_bad_has_external_issue_tracker_data_spec.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require_migration!
+
+RSpec.describe CleanupProjectsWithBadHasExternalIssueTrackerData, :migration do
+  let(:namespace) { table(:namespaces).create!(name: 'foo', path: 'bar') }
+  let(:projects) { table(:projects) }
+  let(:services) { table(:services) }
+
+  def create_projects!(num)
+    Array.new(num) do
+      projects.create!(namespace_id: namespace.id)
+    end
+  end
+
+  def create_active_external_issue_tracker_integrations!(*projects)
+    projects.each do |project|
+      services.create!(category: 'issue_tracker', project_id: project.id, active: true)
+    end
+  end
+
+  def create_disabled_external_issue_tracker_integrations!(*projects)
+    projects.each do |project|
+      services.create!(category: 'issue_tracker', project_id: project.id, active: false)
+    end
+  end
+
+  def create_active_other_integrations!(*projects)
+    projects.each do |project|
+      services.create!(category: 'not_an_issue_tracker', project_id: project.id, active: true)
+    end
+  end
+
+  it 'sets `projects.has_external_issue_tracker` correctly' do
+    allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
+
+    project_with_an_external_issue_tracker_1,
+      project_with_an_external_issue_tracker_2,
+      project_with_only_a_disabled_external_issue_tracker_1,
+      project_with_only_a_disabled_external_issue_tracker_2,
+      project_without_any_external_issue_trackers_1,
+      project_without_any_external_issue_trackers_2 = create_projects!(6)
+
+    create_active_external_issue_tracker_integrations!(
+      project_with_an_external_issue_tracker_1,
+      project_with_an_external_issue_tracker_2
+    )
+
+    create_disabled_external_issue_tracker_integrations!(
+      project_with_an_external_issue_tracker_1,
+      project_with_an_external_issue_tracker_2,
+      project_with_only_a_disabled_external_issue_tracker_1,
+      project_with_only_a_disabled_external_issue_tracker_2
+    )
+
+    create_active_other_integrations!(
+      project_with_an_external_issue_tracker_1,
+      project_with_an_external_issue_tracker_2,
+      project_without_any_external_issue_trackers_1,
+      project_without_any_external_issue_trackers_2
+    )
+
+    # PG triggers on the services table added in
+    # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51852 will have set
+    # the `has_external_issue_tracker` columns to correct data when the services
+    # records were created above.
+    #
+    # We set the `has_external_issue_tracker` columns for projects to incorrect
+    # data manually below to emulate projects in a state before the PG
+    # triggers were added.
+    project_with_an_external_issue_tracker_2.update!(has_external_issue_tracker: false)
+    project_with_only_a_disabled_external_issue_tracker_2.update!(has_external_issue_tracker: true)
+    project_without_any_external_issue_trackers_2.update!(has_external_issue_tracker: true)
+
+    migrate!
+
+    expected_true = [
+      project_with_an_external_issue_tracker_1,
+      project_with_an_external_issue_tracker_2
+    ].each(&:reload).map(&:has_external_issue_tracker)
+
+    expected_not_true = [
+      project_without_any_external_issue_trackers_1,
+      project_without_any_external_issue_trackers_2,
+      project_with_only_a_disabled_external_issue_tracker_1,
+      project_with_only_a_disabled_external_issue_tracker_2
+    ].each(&:reload).map(&:has_external_issue_tracker)
+
+    expect(expected_true).to all(eq(true))
+    expect(expected_not_true).to all(be_falsey)
+  end
+end
diff --git a/spec/migrations/migrate_delayed_project_removal_from_namespaces_to_namespace_settings_spec.rb b/spec/migrations/migrate_delayed_project_removal_from_namespaces_to_namespace_settings_spec.rb
new file mode 100644
index 00000000000..28a8dcf0d4c
--- /dev/null
+++ b/spec/migrations/migrate_delayed_project_removal_from_namespaces_to_namespace_settings_spec.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20210215095328_migrate_delayed_project_removal_from_namespaces_to_namespace_settings.rb')
+
+RSpec.describe MigrateDelayedProjectRemovalFromNamespacesToNamespaceSettings, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:namespace_settings) { table(:namespace_settings) }
+
+  let!(:namespace_wo_settings) { namespaces.create!(name: generate(:name), path: generate(:name), delayed_project_removal: true) }
+  let!(:namespace_wo_settings_delay_false) { namespaces.create!(name: generate(:name), path: generate(:name), delayed_project_removal: false) }
+  let!(:namespace_w_settings_delay_true) { namespaces.create!(name: generate(:name), path: generate(:name), delayed_project_removal: true) }
+  let!(:namespace_w_settings_delay_false) { namespaces.create!(name: generate(:name), path: generate(:name), delayed_project_removal: false) }
+
+  let!(:namespace_settings_delay_true) { namespace_settings.create!(namespace_id: namespace_w_settings_delay_true.id, delayed_project_removal: false, created_at: DateTime.now, updated_at: DateTime.now) }
+  let!(:namespace_settings_delay_false) { namespace_settings.create!(namespace_id: namespace_w_settings_delay_false.id, delayed_project_removal: false, created_at: DateTime.now, updated_at: DateTime.now) }
+
+  it 'migrates delayed_project_removal to namespace_settings' do
+    disable_migrations_output { migrate! }
+
+    expect(namespace_settings.count).to eq(3)
+
+    expect(namespace_settings.find_by(namespace_id: namespace_wo_settings.id).delayed_project_removal).to eq(true)
+    expect(namespace_settings.find_by(namespace_id: namespace_wo_settings_delay_false.id)).to be_nil
+
+    expect(namespace_settings_delay_true.reload.delayed_project_removal).to eq(true)
+    expect(namespace_settings_delay_false.reload.delayed_project_removal).to eq(false)
+  end
+end
diff --git a/spec/migrations/move_container_registry_enabled_to_project_features_spec.rb b/spec/migrations/move_container_registry_enabled_to_project_features_spec.rb
new file mode 100644
index 00000000000..c7b07f3ef37
--- /dev/null
+++ b/spec/migrations/move_container_registry_enabled_to_project_features_spec.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20210226120851_move_container_registry_enabled_to_project_features.rb')
+
+RSpec.describe MoveContainerRegistryEnabledToProjectFeatures, :migration do
+  let(:namespace) { table(:namespaces).create!(name: 'gitlab', path: 'gitlab-org') }
+
+  let!(:projects) do
+    [
+      table(:projects).create!(namespace_id: namespace.id, name: 'project 1'),
+      table(:projects).create!(namespace_id: namespace.id, name: 'project 2'),
+      table(:projects).create!(namespace_id: namespace.id, name: 'project 3'),
+      table(:projects).create!(namespace_id: namespace.id, name: 'project 4')
+    ]
+  end
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 3)
+  end
+
+  around do |example|
+    Sidekiq::Testing.fake! do
+      freeze_time do
+        example.call
+      end
+    end
+  end
+
+  it 'schedules jobs for ranges of projects' do
+    migrate!
+
+    expect(described_class::MIGRATION)
+      .to be_scheduled_delayed_migration(2.minutes, projects[0].id, projects[2].id)
+
+    expect(described_class::MIGRATION)
+      .to be_scheduled_delayed_migration(4.minutes, projects[3].id, projects[3].id)
+  end
+
+  it 'schedules jobs according to the configured batch size' do
+    expect { migrate! }.to change { BackgroundMigrationWorker.jobs.size }.by(2)
+  end
+end
diff --git a/spec/migrations/reschedule_artifact_expiry_backfill_spec.rb b/spec/migrations/reschedule_artifact_expiry_backfill_spec.rb
new file mode 100644
index 00000000000..532035849cb
--- /dev/null
+++ b/spec/migrations/reschedule_artifact_expiry_backfill_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20210224150506_reschedule_artifact_expiry_backfill.rb')
+
+RSpec.describe RescheduleArtifactExpiryBackfill, :migration do
+  let(:migration_class) { Gitlab::BackgroundMigration::BackfillArtifactExpiryDate }
+  let(:migration_name)  { migration_class.to_s.demodulize }
+
+  before do
+    table(:namespaces).create!(id: 123, name: 'test_namespace', path: 'test_namespace')
+    table(:projects).create!(id: 123, name: 'sample_project', path: 'sample_project', namespace_id: 123)
+  end
+
+  it 'correctly schedules background migrations' do
+    first_artifact = create_artifact(job_id: 0, expire_at: nil, created_at: Date.new(2020, 06, 21))
+    second_artifact = create_artifact(job_id: 1, expire_at: nil, created_at: Date.new(2020, 06, 21))
+    create_artifact(job_id: 2, expire_at: Date.yesterday, created_at: Date.new(2020, 06, 21))
+    create_artifact(job_id: 3, expire_at: nil, created_at: Date.new(2020, 06, 23))
+
+    Sidekiq::Testing.fake! do
+      freeze_time do
+        migrate!
+
+        expect(BackgroundMigrationWorker.jobs.size).to eq(1)
+        expect(migration_name).to be_scheduled_migration_with_multiple_args(first_artifact.id, second_artifact.id)
+      end
+    end
+  end
+
+  private
+
+  def create_artifact(params)
+    table(:ci_builds).create!(id: params[:job_id], project_id: 123)
+    table(:ci_job_artifacts).create!(project_id: 123, file_type: 1, **params)
+  end
+end
diff --git a/spec/migrations/reschedule_set_default_iteration_cadences_spec.rb b/spec/migrations/reschedule_set_default_iteration_cadences_spec.rb
new file mode 100644
index 00000000000..fb629c90d9f
--- /dev/null
+++ b/spec/migrations/reschedule_set_default_iteration_cadences_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe RescheduleSetDefaultIterationCadences do
+  let(:namespaces) { table(:namespaces) }
+  let(:iterations) { table(:sprints) }
+
+  let(:group_1) { namespaces.create!(name: 'test_1', path: 'test_1') }
+  let!(:group_2) { namespaces.create!(name: 'test_2', path: 'test_2') }
+  let(:group_3) { namespaces.create!(name: 'test_3', path: 'test_3') }
+  let(:group_4) { namespaces.create!(name: 'test_4', path: 'test_4') }
+  let(:group_5) { namespaces.create!(name: 'test_5', path: 'test_5') }
+  let(:group_6) { namespaces.create!(name: 'test_6', path: 'test_6') }
+  let(:group_7) { namespaces.create!(name: 'test_7', path: 'test_7') }
+  let(:group_8) { namespaces.create!(name: 'test_8', path: 'test_8') }
+
+  let!(:iteration_1) { iterations.create!(iid: 1, title: 'iteration 1', group_id: group_1.id, start_date: 2.days.from_now, due_date: 3.days.from_now) }
+  let!(:iteration_2) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_3.id, start_date: 2.days.from_now, due_date: 3.days.from_now) }
+  let!(:iteration_3) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_4.id, start_date: 2.days.from_now, due_date: 3.days.from_now) }
+  let!(:iteration_4) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_5.id, start_date: 2.days.from_now, due_date: 3.days.from_now) }
+  let!(:iteration_5) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_6.id, start_date: 2.days.from_now, due_date: 3.days.from_now) }
+  let!(:iteration_6) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_7.id, start_date: 2.days.from_now, due_date: 3.days.from_now) }
+  let!(:iteration_7) { iterations.create!(iid: 1, title: 'iteration 2', group_id: group_8.id, start_date: 2.days.from_now, due_date: 3.days.from_now) }
+
+  around do |example|
+    freeze_time { Sidekiq::Testing.fake! { example.run } }
+  end
+
+  it 'schedules the background jobs', :aggregate_failures do
+    stub_const("#{described_class.name}::BATCH_SIZE", 3)
+
+    migrate!
+
+    expect(BackgroundMigrationWorker.jobs.size).to be(3)
+    expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(2.minutes, group_1.id, group_3.id, group_4.id)
+    expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(4.minutes, group_5.id, group_6.id, group_7.id)
+    expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(6.minutes, group_8.id)
+  end
+end
diff --git a/spec/migrations/schedule_merge_request_assignees_migration_progress_check_spec.rb b/spec/migrations/schedule_merge_request_assignees_migration_progress_check_spec.rb
deleted file mode 100644
index 0a69f49f10d..00000000000
--- a/spec/migrations/schedule_merge_request_assignees_migration_progress_check_spec.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-require Rails.root.join('db', 'post_migrate', '20190402224749_schedule_merge_request_assignees_migration_progress_check.rb')
-
-RSpec.describe ScheduleMergeRequestAssigneesMigrationProgressCheck do
-  describe '#up' do
-    it 'schedules MergeRequestAssigneesMigrationProgressCheck background job' do
-      expect(BackgroundMigrationWorker).to receive(:perform_async)
-                                             .with(described_class::MIGRATION)
-                                             .and_call_original
-
-      subject.up
-    end
-  end
-end
diff --git a/spec/migrations/schedule_populate_personal_snippet_statistics_spec.rb b/spec/migrations/schedule_populate_personal_snippet_statistics_spec.rb
index 8678361fc64..faa993dfbc7 100644
--- a/spec/migrations/schedule_populate_personal_snippet_statistics_spec.rb
+++ b/spec/migrations/schedule_populate_personal_snippet_statistics_spec.rb
@@ -6,11 +6,15 @@ require Rails.root.join('db', 'post_migrate', '20200714075739_schedule_populate_
 
 RSpec.describe SchedulePopulatePersonalSnippetStatistics do
   let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
   let(:snippets) { table(:snippets) }
   let(:projects) { table(:projects) }
-  let(:user1) { users.create!(id: 1, email: 'user1@example.com', projects_limit: 10, username: 'test1', name: 'Test1', state: 'active') }
-  let(:user2) { users.create!(id: 2, email: 'user2@example.com', projects_limit: 10, username: 'test2', name: 'Test2', state: 'active') }
-  let(:user3) { users.create!(id: 3, email: 'user3@example.com', projects_limit: 10, username: 'test3', name: 'Test3', state: 'active') }
+  let!(:user1) { users.create!(id: 1, email: 'user1@example.com', projects_limit: 10, username: 'test1', name: 'Test1', state: 'active') }
+  let!(:user2) { users.create!(id: 2, email: 'user2@example.com', projects_limit: 10, username: 'test2', name: 'Test2', state: 'active') }
+  let!(:user3) { users.create!(id: 3, email: 'user3@example.com', projects_limit: 10, username: 'test3', name: 'Test3', state: 'active') }
+  let!(:namespace1) { namespaces.create!(id: 1, owner_id: user1.id, name: 'test1', path: 'test1') }
+  let!(:namespace2) { namespaces.create!(id: 2, owner_id: user2.id, name: 'test2', path: 'test2') }
+  let!(:namespace3) { namespaces.create!(id: 3, owner_id: user3.id, name: 'test3', path: 'test3') }
 
   def create_snippet(id, user_id, type = 'PersonalSnippet')
     params = {