Add latest changes from gitlab-org/gitlab@master

8 files changed, 301 insertions, 40 deletions

diff --git a/spec/lib/generators/gitlab/snowplow_event_definition_generator_spec.rb b/spec/lib/generators/gitlab/snowplow_event_definition_generator_spec.rb
index d9fa6b931ad..4ccbf44af52 100644
--- a/spec/lib/generators/gitlab/snowplow_event_definition_generator_spec.rb
+++ b/spec/lib/generators/gitlab/snowplow_event_definition_generator_spec.rb
@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::SnowplowEventDefinitionGenerator, :silence_stdout do
+RSpec.describe Gitlab::SnowplowEventDefinitionGenerator, :silence_stdout, feature_category: :product_analytics do
   let(:ce_temp_dir) { Dir.mktmpdir }
   let(:ee_temp_dir) { Dir.mktmpdir }
   let(:timestamp) { Time.current.to_i }
@@ -30,7 +30,8 @@ RSpec.describe Gitlab::SnowplowEventDefinitionGenerator, :silence_stdout do
     let(:file_name) { Dir.children(ce_temp_dir).first }
 
     it 'creates CE event definition file using the template' do
-      sample_event = ::Gitlab::Config::Loader::Yaml.new(fixture_file(File.join(sample_event_dir, 'sample_event.yml'))).load_raw!
+      sample_event = ::Gitlab::Config::Loader::Yaml
+                       .new(fixture_file(File.join(sample_event_dir, 'sample_event.yml'))).load_raw!
 
       described_class.new([], generator_options).invoke_all
 
@@ -62,25 +63,13 @@ RSpec.describe Gitlab::SnowplowEventDefinitionGenerator, :silence_stdout do
       end
     end
 
-    context 'event definition already exists' do
+    context 'when event definition with same file name already exists' do
       before do
         stub_const('Gitlab::VERSION', '12.11.0-pre')
         described_class.new([], generator_options).invoke_all
       end
 
-      it 'overwrites event definition --force flag set to true' do
-        sample_event = ::Gitlab::Config::Loader::Yaml.new(fixture_file(File.join(sample_event_dir, 'sample_event.yml'))).load_raw!
-
-        stub_const('Gitlab::VERSION', '13.11.0-pre')
-        described_class.new([], generator_options.merge('force' => true)).invoke_all
-
-        event_definition_path = File.join(ce_temp_dir, file_name)
-        event_data = ::Gitlab::Config::Loader::Yaml.new(File.read(event_definition_path)).load_raw!
-
-        expect(event_data).to eq(sample_event)
-      end
-
-      it 'raises error when --force flag set to false' do
+      it 'raises error' do
         expect { described_class.new([], generator_options.merge('force' => false)).invoke_all }
           .to raise_error(StandardError, /Event definition already exists at/)
       end
@@ -90,7 +79,8 @@ RSpec.describe Gitlab::SnowplowEventDefinitionGenerator, :silence_stdout do
       let(:file_name) { Dir.children(ee_temp_dir).first }
 
       it 'creates EE event definition file using the template' do
-        sample_event = ::Gitlab::Config::Loader::Yaml.new(fixture_file(File.join(sample_event_dir, 'sample_event_ee.yml'))).load_raw!
+        sample_event = ::Gitlab::Config::Loader::Yaml
+                         .new(fixture_file(File.join(sample_event_dir, 'sample_event_ee.yml'))).load_raw!
 
         described_class.new([], generator_options.merge('ee' => true)).invoke_all
 
diff --git a/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb
new file mode 100644
index 00000000000..84259505683
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb
@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::MigrateRemediationsForVulnerabilityFindings,
+  feature_category: :vulnerability_management do
+  let(:vulnerability_occurrences) { table(:vulnerability_occurrences) }
+  let(:vulnerability_findings_remediations) { table(:vulnerability_findings_remediations) }
+  let(:vulnerability_remediations) { table(:vulnerability_remediations) }
+  let(:remediation_hash) { { summary: 'summary', diff: "ZGlmZiAtLWdp" } }
+  let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') }
+  let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) }
+  let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) }
+
+  let(:scanner1) do
+    table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1')
+  end
+
+  let(:stating_id) { vulnerability_occurrences.pluck(:id).min }
+  let(:end_id) { vulnerability_occurrences.pluck(:id).max }
+
+  let(:migration) do
+    described_class.new(
+      start_id: stating_id,
+      end_id: end_id,
+      batch_table: :vulnerability_occurrences,
+      batch_column: :id,
+      sub_batch_size: 2,
+      pause_ms: 2,
+      connection: ApplicationRecord.connection
+    )
+  end
+
+  subject(:perform_migration) { migration.perform }
+
+  context 'without the presence of remediation key' do
+    before do
+      create_finding!(project1.id, scanner1.id, { other_keys: 'test' })
+    end
+
+    it 'does not create any remediation' do
+      expect(Gitlab::AppLogger).not_to receive(:error)
+
+      expect { perform_migration }.not_to change { vulnerability_remediations.count }
+    end
+  end
+
+  context 'with remediation equals to an array of nil element' do
+    before do
+      create_finding!(project1.id, scanner1.id, { remediations: [nil] })
+    end
+
+    it 'does not create any remediation' do
+      expect(Gitlab::AppLogger).not_to receive(:error)
+
+      expect { perform_migration }.not_to change { vulnerability_remediations.count }
+    end
+  end
+
+  context 'with remediation equals to an array of duplicated elements' do
+    let!(:finding) do
+      create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash, remediation_hash] })
+    end
+
+    it 'creates new remediation' do
+      expect(Gitlab::AppLogger).not_to receive(:error)
+
+      expect { perform_migration }.to change { vulnerability_remediations.count }.by(1)
+      expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1)
+    end
+  end
+
+  context 'with existing remediations within raw_metadata' do
+    let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) }
+    let!(:finding2) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) }
+
+    it 'creates new remediation' do
+      expect(Gitlab::AppLogger).not_to receive(:error)
+
+      expect { perform_migration }.to change { vulnerability_remediations.count }.by(1)
+      expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1)
+      expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1)
+    end
+
+    context 'when create throws exception other than ActiveRecord::RecordNotUnique' do
+      before do
+        allow(migration).to receive(:create_finding_remediations).and_raise(StandardError)
+      end
+
+      it 'rolls back all related transactions' do
+        expect(Gitlab::AppLogger).to receive(:error).with({
+          class: described_class.name, message: StandardError.to_s, model_id: finding1.id
+        })
+        expect(Gitlab::AppLogger).to receive(:error).with({
+          class: described_class.name, message: StandardError.to_s, model_id: finding2.id
+        })
+        expect { perform_migration }.not_to change { vulnerability_remediations.count }
+        expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(0)
+        expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(0)
+      end
+    end
+  end
+
+  context 'with existing remediation records' do
+    let!(:finding) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) }
+
+    before do
+      vulnerability_remediations.create!(project_id: project1.id, summary: remediation_hash[:summary],
+        checksum: checksum(remediation_hash[:diff]), file: Tempfile.new.path)
+    end
+
+    it 'does not create new remediation' do
+      expect(Gitlab::AppLogger).not_to receive(:error)
+
+      expect { perform_migration }.not_to change { vulnerability_remediations.count }
+      expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1)
+    end
+  end
+
+  context 'with same raw_metadata for different projects' do
+    let(:namespace2) { table(:namespaces).create!(name: 'namespace 2', path: 'namespace2') }
+    let(:project2) { table(:projects).create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) }
+    let(:scanner2) do
+      table(:vulnerability_scanners).create!(project_id: project2.id, external_id: 'test 2', name: 'test scanner 2')
+    end
+
+    let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) }
+    let!(:finding2) { create_finding!(project2.id, scanner2.id, { remediations: [remediation_hash] }) }
+
+    it 'creates new remediation for each project' do
+      expect(Gitlab::AppLogger).not_to receive(:error)
+
+      expect { perform_migration }.to change { vulnerability_remediations.count }.by(2)
+      expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1)
+      expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1)
+    end
+  end
+
+  private
+
+  def create_finding!(project_id, scanner_id, raw_metadata)
+    vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test',
+      severity: 4, confidence: 4, report_type: 0)
+
+    identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5',
+      external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s),
+      name: 'Identifier for UUIDv5 2 2')
+
+    table(:vulnerability_occurrences).create!(
+      vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id,
+      primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0,
+      uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" },
+      location_fingerprint: 'test', metadata_version: 'test',
+      raw_metadata: raw_metadata.to_json)
+  end
+
+  def checksum(value)
+    sha = Digest::SHA256.hexdigest(value)
+    Gitlab::Database::ShaAttribute.new.serialize(sha)
+  end
+end
diff --git a/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb
index ab06c7c7e82..08d255a4bb8 100644
--- a/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb
+++ b/spec/lib/gitlab/database/async_constraints/migration_helpers_spec.rb
@@ -33,6 +33,7 @@ RSpec.describe Gitlab::Database::AsyncConstraints::MigrationHelpers, feature_cat
         record = constraint_model.find_by(table_name: table_name)
 
         expect(record.name).to start_with('fk_')
+        expect(record).to be_foreign_key
       end
 
       context 'when an explicit name is given' do
@@ -46,6 +47,7 @@ RSpec.describe Gitlab::Database::AsyncConstraints::MigrationHelpers, feature_cat
           record = constraint_model.find_by(name: fk_name)
 
           expect(record.table_name).to eq(table_name)
+          expect(record).to be_foreign_key
         end
       end
 
@@ -81,33 +83,50 @@ RSpec.describe Gitlab::Database::AsyncConstraints::MigrationHelpers, feature_cat
     end
 
     describe '#unprepare_async_foreign_key_validation' do
-      before do
-        migration.prepare_async_foreign_key_validation(table_name, column_name, name: fk_name)
-      end
-
-      it 'destroys the record' do
-        expect do
-          migration.unprepare_async_foreign_key_validation(table_name, column_name)
-        end.to change { constraint_model.where(table_name: table_name).count }.by(-1)
-      end
-
-      context 'when an explicit name is given' do
-        let(:fk_name) { 'my_test_async_fk' }
+      context 'with foreign keys' do
+        before do
+          migration.prepare_async_foreign_key_validation(table_name, column_name, name: fk_name)
+        end
 
         it 'destroys the record' do
           expect do
-            migration.unprepare_async_foreign_key_validation(table_name, name: fk_name)
-          end.to change { constraint_model.where(name: fk_name).count }.by(-1)
+            migration.unprepare_async_foreign_key_validation(table_name, column_name)
+          end.to change { constraint_model.where(table_name: table_name).count }.by(-1)
+        end
+
+        context 'when an explicit name is given' do
+          let(:fk_name) { 'my_test_async_fk' }
+
+          it 'destroys the record' do
+            expect do
+              migration.unprepare_async_foreign_key_validation(table_name, name: fk_name)
+            end.to change { constraint_model.where(name: fk_name).count }.by(-1)
+          end
+        end
+
+        context 'when the async fk validation table does not exist' do
+          it 'does not raise an error' do
+            connection.drop_table(constraint_model.table_name)
+
+            expect(constraint_model).not_to receive(:find_by)
+
+            expect { migration.unprepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error
+          end
         end
       end
 
-      context 'when the async fk validation table does not exist' do
-        it 'does not raise an error' do
-          connection.drop_table(constraint_model.table_name)
+      context 'with other types of constraints' do
+        let(:name) { 'my_test_async_constraint' }
+        let(:constraint) { create(:postgres_async_constraint_validation, table_name: table_name, name: name) }
 
-          expect(constraint_model).not_to receive(:find_by)
+        it 'does not destroy the record' do
+          constraint.update_column(:constraint_type, 99)
+
+          expect do
+            migration.unprepare_async_foreign_key_validation(table_name, name: name)
+          end.not_to change { constraint_model.where(name: name).count }
 
-          expect { migration.unprepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error
+          expect(constraint).to be_present
         end
       end
     end
diff --git a/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb b/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb
index a8e136dd22c..7fea2eb1bcc 100644
--- a/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb
+++ b/spec/lib/gitlab/database/async_constraints/postgres_async_constraint_validation_spec.rb
@@ -30,6 +30,24 @@ RSpec.describe Gitlab::Database::AsyncConstraints::PostgresAsyncConstraintValida
 
       it { is_expected.to eq([new_validation, failed_validation]) }
     end
+
+    describe '.foreign_key_type' do
+      before do
+        new_validation.update_column(:constraint_type, 99)
+      end
+
+      subject { described_class.foreign_key_type }
+
+      it { is_expected.to eq([failed_validation]) }
+
+      it 'does not apply the filter if the column is not present' do
+        expect(described_class).to receive(:columns_hash).and_wrap_original do |method|
+          method.call.except('constraint_type')
+        end
+
+        is_expected.to match_array([failed_validation, new_validation])
+      end
+    end
   end
 
   describe '.table_available?' do
diff --git a/spec/lib/gitlab/import/import_failure_service_spec.rb b/spec/lib/gitlab/import/import_failure_service_spec.rb
index eb71b307b8d..fc02c6bd4ca 100644
--- a/spec/lib/gitlab/import/import_failure_service_spec.rb
+++ b/spec/lib/gitlab/import/import_failure_service_spec.rb
@@ -141,6 +141,7 @@ RSpec.describe Gitlab::Import::ImportFailureService, :aggregate_failures do
           .with("#{project.import_type}_importer", project)
           .and_return(metrics_double)
         expect(metrics_double).to receive(:track_failed_import)
+        expect(metrics_double).to receive(:track_import_state)
 
         service.execute
       end
diff --git a/spec/lib/gitlab/import/metrics_spec.rb b/spec/lib/gitlab/import/metrics_spec.rb
index 9b8b58d00f3..f0140a975b1 100644
--- a/spec/lib/gitlab/import/metrics_spec.rb
+++ b/spec/lib/gitlab/import/metrics_spec.rb
@@ -60,6 +60,41 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do
     end
   end
 
+  describe '#track_import_state' do
+    context 'when project is not a github import' do
+      it 'does not emit importer metrics' do
+        subject.track_import_state
+
+        expect_no_snowplow_event(
+          category: :test_importer,
+          action: 'create',
+          label: 'github_import_project_state',
+          project: project,
+          extra: { import_type: 'github', state: 'failed' }
+        )
+      end
+    end
+
+    context 'when project is a github import' do
+      before do
+        project.import_type = 'github'
+        allow(project).to receive(:import_status).and_return('failed')
+      end
+
+      it 'emits importer metrics' do
+        subject.track_import_state
+
+        expect_snowplow_event(
+          category: :test_importer,
+          action: 'create',
+          label: 'github_import_project_state',
+          project: project,
+          extra: { import_type: 'github', state: 'failed' }
+        )
+      end
+    end
+  end
+
   describe '#track_finished_import' do
     before do
       allow(Gitlab::Metrics).to receive(:histogram) { histogram }
@@ -92,6 +127,33 @@ RSpec.describe Gitlab::Import::Metrics, :aggregate_failures do
         subject.track_finished_import
 
         expect(histogram).to have_received(:observe).with({ importer: :test_importer }, anything)
+        expect_no_snowplow_event(
+          category: :test_importer,
+          action: 'create',
+          label: 'github_import_project_state',
+          project: project,
+          extra: { import_type: 'github', state: 'completed' }
+        )
+      end
+    end
+
+    context 'when project is a github import' do
+      before do
+        project.import_type = 'github'
+        allow(project).to receive(:import_status).and_return('finished')
+        allow(project).to receive(:import_finished?).and_return(true)
+      end
+
+      it 'emits snowplow metrics' do
+        subject.track_finished_import
+
+        expect_snowplow_event(
+          category: :test_importer,
+          action: 'create',
+          label: 'github_import_project_state',
+          project: project,
+          extra: { import_type: 'github', state: 'completed' }
+        )
       end
     end
   end
diff --git a/spec/lib/sidebars/projects/super_sidebar_menus/operations_menu_spec.rb b/spec/lib/sidebars/projects/super_sidebar_menus/operations_menu_spec.rb
new file mode 100644
index 00000000000..df3f7e6cdab
--- /dev/null
+++ b/spec/lib/sidebars/projects/super_sidebar_menus/operations_menu_spec.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Sidebars::Projects::SuperSidebarMenus::OperationsMenu, feature_category: :navigation do
+  subject { described_class.new({}) }
+
+  it 'has title and sprite_icon' do
+    expect(subject.title).to eq(_("Operations"))
+    expect(subject.sprite_icon).to eq("deployments")
+  end
+end
diff --git a/spec/lib/sidebars/projects/super_sidebar_panel_spec.rb b/spec/lib/sidebars/projects/super_sidebar_panel_spec.rb
index a4df46ca493..d6fc3fd8fe1 100644
--- a/spec/lib/sidebars/projects/super_sidebar_panel_spec.rb
+++ b/spec/lib/sidebars/projects/super_sidebar_panel_spec.rb
@@ -30,9 +30,7 @@ RSpec.describe Sidebars::Projects::SuperSidebarPanel, feature_category: :navigat
         Sidebars::Projects::Menus::RepositoryMenu,
         Sidebars::Projects::Menus::CiCdMenu,
         Sidebars::Projects::Menus::SecurityComplianceMenu,
-        Sidebars::Projects::Menus::DeploymentsMenu,
-        Sidebars::Projects::Menus::PackagesRegistriesMenu,
-        Sidebars::Projects::Menus::InfrastructureMenu,
+        Sidebars::Projects::SuperSidebarMenus::OperationsMenu,
         Sidebars::Projects::Menus::MonitorMenu,
         Sidebars::Projects::Menus::AnalyticsMenu,
         Sidebars::UncategorizedMenu,
@@ -41,7 +39,7 @@ RSpec.describe Sidebars::Projects::SuperSidebarPanel, feature_category: :navigat
     end
 
     it "is exposed as a renderable menu" do
-      expect(subject.renderable_menus.map(&:class)).to eq(category_menu)
+      expect(subject.instance_variable_get(:@menus).map(&:class)).to eq(category_menu)
     end
   end
 end