Add latest changes from gitlab-org/gitlab@master

5 files changed, 353 insertions, 178 deletions

diff --git a/spec/frontend/google_tag_manager/index_spec.js b/spec/frontend/google_tag_manager/index_spec.js
index 6f1a14078c1..f9199f32f1e 100644
--- a/spec/frontend/google_tag_manager/index_spec.js
+++ b/spec/frontend/google_tag_manager/index_spec.js
@@ -234,7 +234,7 @@ describe('~/google_tag_manager/index', () => {
                     category: 'DevOps',
                     id: '0002',
                     name: 'Premium',
-                    price: 228,
+                    price: '228',
                     quantity: 1,
                     variant: 'SaaS',
                   },
@@ -264,7 +264,7 @@ describe('~/google_tag_manager/index', () => {
                   category: 'DevOps',
                   id: '0001',
                   name: 'Ultimate',
-                  price: 1188,
+                  price: '1188',
                   quantity: 1,
                   variant: 'SaaS',
                 },
@@ -301,7 +301,7 @@ describe('~/google_tag_manager/index', () => {
                   category: 'DevOps',
                   id: '0001',
                   name: 'Ultimate',
-                  price: 1188,
+                  price: '1188',
                   quantity: 5,
                   variant: 'SaaS',
                 },
@@ -354,8 +354,8 @@ describe('~/google_tag_manager/index', () => {
                     id: '123',
                     affiliation: 'GitLab',
                     option: 'visa',
-                    revenue,
-                    tax: 10,
+                    revenue: revenue.toString(),
+                    tax: '10',
                   },
                   products: [
                     {
@@ -363,7 +363,7 @@ describe('~/google_tag_manager/index', () => {
                       category: 'DevOps',
                       id,
                       name,
-                      price: revenue,
+                      price: revenue.toString(),
                       quantity: 1,
                       variant: 'SaaS',
                     },
diff --git a/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb
new file mode 100644
index 00000000000..a265fa95b23
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityReads do
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:vulnerability_reads) { table(:vulnerability_reads) }
+  let(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
+  let(:vulnerability_issue_links) { table(:vulnerability_issue_links) }
+  let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let(:project) { table(:projects).create!(namespace_id: namespace.id) }
+  let(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
+  let(:sub_batch_size) { 1000 }
+
+  before do
+    vulnerabilities_findings.connection.execute 'ALTER TABLE vulnerability_occurrences DISABLE TRIGGER "trigger_insert_or_update_vulnerability_reads_from_occurrences"'
+    vulnerabilities.connection.execute 'ALTER TABLE vulnerabilities DISABLE TRIGGER "trigger_update_vulnerability_reads_on_vulnerability_update"'
+    vulnerability_issue_links.connection.execute 'ALTER TABLE vulnerability_issue_links DISABLE TRIGGER "trigger_update_has_issues_on_vulnerability_issue_links_update"'
+
+    10.times.each do |x|
+      vulnerability = create_vulnerability!(
+        project_id: project.id,
+        report_type: 7,
+        author_id: user.id
+      )
+      identifier = table(:vulnerability_identifiers).create!(
+        project_id: project.id,
+        external_type: 'uuid-v5',
+        external_id: 'uuid-v5',
+        fingerprint: Digest::SHA1.hexdigest("#{vulnerability.id}"),
+        name: 'Identifier for UUIDv5')
+
+      create_finding!(
+        vulnerability_id: vulnerability.id,
+        project_id: project.id,
+        scanner_id: scanner.id,
+        primary_identifier_id: identifier.id
+      )
+    end
+  end
+
+  it 'creates vulnerability_reads for the given records' do
+    described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
+
+    expect(vulnerability_reads.count).to eq(10)
+  end
+
+  it 'does not create new records when records already exists' do
+    described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
+    described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
+
+    expect(vulnerability_reads.count).to eq(10)
+  end
+
+  private
+
+  def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
+    vulnerabilities.create!(
+      project_id: project_id,
+      author_id: author_id,
+      title: title,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type
+    )
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+  def create_finding!(
+    vulnerability_id: nil, project_id:, scanner_id:, primary_identifier_id:,
+    name: "test", severity: 7, confidence: 7, report_type: 0,
+    project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, location_fingerprint: 'test',
+    metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
+    vulnerabilities_findings.create!(
+      vulnerability_id: vulnerability_id,
+      project_id: project_id,
+      name: name,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_fingerprint: project_fingerprint,
+      scanner_id: scanner_id,
+      primary_identifier_id: primary_identifier_id,
+      location: location,
+      location_fingerprint: location_fingerprint,
+      metadata_version: metadata_version,
+      raw_metadata: raw_metadata,
+      uuid: uuid
+    )
+  end
+  # rubocop:enable Metrics/ParameterLists
+end
diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb
index 3ad147585dd..6f9d4e8a70d 100644
--- a/spec/lib/gitlab/ci/variables/builder_spec.rb
+++ b/spec/lib/gitlab/ci/variables/builder_spec.rb
@@ -3,9 +3,10 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Variables::Builder do
-  let_it_be(:project) { create(:project, :repository) }
-  let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
-  let_it_be(:user) { project.first_owner }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, :repository, namespace: group) }
+  let_it_be_with_reload(:pipeline) { create(:ci_pipeline, project: project) }
+  let_it_be(:user) { create(:user) }
   let_it_be(:job) do
     create(:ci_build,
       pipeline: pipeline,
@@ -153,7 +154,7 @@ RSpec.describe Gitlab::Ci::Variables::Builder do
 
       before do
         allow(builder).to receive(:predefined_variables) { [var('A', 1), var('B', 1)] }
-        allow(project).to receive(:predefined_variables) { [var('B', 2), var('C', 2)] }
+        allow(pipeline.project).to receive(:predefined_variables) { [var('B', 2), var('C', 2)] }
         allow(pipeline).to receive(:predefined_variables) { [var('C', 3), var('D', 3)] }
         allow(job).to receive(:runner) { double(predefined_variables: [var('D', 4), var('E', 4)]) }
         allow(builder).to receive(:kubernetes_variables) { [var('E', 5), var('F', 5)] }
@@ -201,4 +202,146 @@ RSpec.describe Gitlab::Ci::Variables::Builder do
       end
     end
   end
+
+  describe '#user_variables' do
+    context 'with user' do
+      subject { builder.user_variables(user).to_hash }
+
+      let(:expected_variables) do
+        {
+          'GITLAB_USER_EMAIL' => user.email,
+          'GITLAB_USER_ID' => user.id.to_s,
+          'GITLAB_USER_LOGIN' => user.username,
+          'GITLAB_USER_NAME' => user.name
+        }
+      end
+
+      it { is_expected.to eq(expected_variables) }
+    end
+
+    context 'without user' do
+      subject { builder.user_variables(nil).to_hash }
+
+      it { is_expected.to be_empty }
+    end
+  end
+
+  describe '#kubernetes_variables' do
+    let(:service) { double(execute: template) }
+    let(:template) { double(to_yaml: 'example-kubeconfig', valid?: template_valid) }
+    let(:template_valid) { true }
+
+    subject { builder.kubernetes_variables(job) }
+
+    before do
+      allow(Ci::GenerateKubeconfigService).to receive(:new).with(job).and_return(service)
+    end
+
+    it { is_expected.to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+
+    context 'generated config is invalid' do
+      let(:template_valid) { false }
+
+      it { is_expected.not_to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+    end
+  end
+
+  describe '#deployment_variables' do
+    let(:environment) { 'production' }
+    let(:kubernetes_namespace) { 'namespace' }
+    let(:project_variables) { double }
+
+    subject { builder.deployment_variables(environment: environment, job: job) }
+
+    before do
+      allow(job).to receive(:expanded_kubernetes_namespace)
+        .and_return(kubernetes_namespace)
+
+      allow(project).to receive(:deployment_variables)
+        .with(environment: environment, kubernetes_namespace: kubernetes_namespace)
+        .and_return(project_variables)
+    end
+
+    context 'environment is nil' do
+      let(:environment) { nil }
+
+      it { is_expected.to be_empty }
+    end
+  end
+
+  shared_examples "secret CI variables" do
+    context 'when ref is branch' do
+      context 'when ref is protected' do
+        before do
+          create(:protected_branch, :developers_can_merge, name: job.ref, project: project)
+        end
+
+        it { is_expected.to include(variable) }
+      end
+
+      context 'when ref is not protected' do
+        it { is_expected.not_to include(variable) }
+      end
+    end
+
+    context 'when ref is tag' do
+      let_it_be(:job) { create(:ci_build, ref: 'v1.1.0', tag: true, pipeline: pipeline) }
+
+      context 'when ref is protected' do
+        before do
+          create(:protected_tag, project: project, name: 'v*')
+        end
+
+        it { is_expected.to include(variable) }
+      end
+
+      context 'when ref is not protected' do
+        it { is_expected.not_to include(variable) }
+      end
+    end
+
+    context 'when ref is merge request' do
+      let_it_be(:merge_request) { create(:merge_request, :with_detached_merge_request_pipeline, source_project: project) }
+      let_it_be(:pipeline) { merge_request.pipelines_for_merge_request.first }
+      let_it_be(:job) { create(:ci_build, ref: merge_request.source_branch, tag: false, pipeline: pipeline) }
+
+      context 'when ref is protected' do
+        before do
+          create(:protected_branch, :developers_can_merge, name: merge_request.source_branch, project: project)
+        end
+
+        it 'does not return protected variables as it is not supported for merge request pipelines' do
+          is_expected.not_to include(variable)
+        end
+      end
+
+      context 'when ref is not protected' do
+        it { is_expected.not_to include(variable) }
+      end
+    end
+  end
+
+  describe '#secret_instance_variables' do
+    subject { builder.secret_instance_variables(ref: job.git_ref) }
+
+    let_it_be(:variable) { create(:ci_instance_variable, protected: true) }
+
+    include_examples "secret CI variables"
+  end
+
+  describe '#secret_group_variables' do
+    subject { builder.secret_group_variables(ref: job.git_ref, environment: job.expanded_environment_name) }
+
+    let_it_be(:variable) { create(:ci_group_variable, protected: true, group: group) }
+
+    include_examples "secret CI variables"
+  end
+
+  describe '#secret_project_variables' do
+    subject { builder.secret_project_variables(ref: job.git_ref, environment: job.expanded_environment_name) }
+
+    let_it_be(:variable) { create(:ci_variable, protected: true, project: project) }
+
+    include_examples "secret CI variables"
+  end
 end
diff --git a/spec/migrations/20220107064845_populate_vulnerability_reads_spec.rb b/spec/migrations/20220107064845_populate_vulnerability_reads_spec.rb
new file mode 100644
index 00000000000..ece971a50c9
--- /dev/null
+++ b/spec/migrations/20220107064845_populate_vulnerability_reads_spec.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+require_migration!
+
+RSpec.describe PopulateVulnerabilityReads, :migration do
+  let_it_be(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let_it_be(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let_it_be(:project) { table(:projects).create!(namespace_id: namespace.id) }
+  let_it_be(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
+  let_it_be(:background_migration_jobs) { table(:background_migration_jobs) }
+  let_it_be(:vulnerabilities) { table(:vulnerabilities) }
+  let_it_be(:vulnerability_reads) { table(:vulnerability_reads) }
+  let_it_be(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
+  let_it_be(:vulnerability_issue_links) { table(:vulnerability_issue_links) }
+  let_it_be(:vulnerability_ids) { [] }
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+    stub_const("#{described_class}::SUB_BATCH_SIZE", 1)
+
+    5.times.each do |x|
+      vulnerability = create_vulnerability!(
+        project_id: project.id,
+        report_type: 7,
+        author_id: user.id
+      )
+      identifier = table(:vulnerability_identifiers).create!(
+        project_id: project.id,
+        external_type: 'uuid-v5',
+        external_id: 'uuid-v5',
+        fingerprint: Digest::SHA1.hexdigest("#{vulnerability.id}"),
+        name: 'Identifier for UUIDv5')
+
+      create_finding!(
+        vulnerability_id: vulnerability.id,
+        project_id: project.id,
+        scanner_id: scanner.id,
+        primary_identifier_id: identifier.id
+      )
+
+      vulnerability_ids << vulnerability.id
+    end
+  end
+
+  around do |example|
+    freeze_time { Sidekiq::Testing.fake! { example.run } }
+  end
+
+  it 'schedules background migrations' do
+    migrate!
+
+    expect(background_migration_jobs.count).to eq(5)
+    expect(background_migration_jobs.first.arguments).to match_array([vulnerability_ids.first, vulnerability_ids.first, 1])
+    expect(background_migration_jobs.second.arguments).to match_array([vulnerability_ids.second, vulnerability_ids.second, 1])
+    expect(background_migration_jobs.third.arguments).to match_array([vulnerability_ids.third, vulnerability_ids.third, 1])
+    expect(background_migration_jobs.fourth.arguments).to match_array([vulnerability_ids.fourth, vulnerability_ids.fourth, 1])
+    expect(background_migration_jobs.fifth.arguments).to match_array([vulnerability_ids.fifth, vulnerability_ids.fifth, 1])
+
+    expect(BackgroundMigrationWorker.jobs.size).to eq(5)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(2.minutes, vulnerability_ids.first, vulnerability_ids.first, 1)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(4.minutes, vulnerability_ids.second, vulnerability_ids.second, 1)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(6.minutes, vulnerability_ids.third, vulnerability_ids.third, 1)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(8.minutes, vulnerability_ids.fourth, vulnerability_ids.fourth, 1)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(10.minutes, vulnerability_ids.fifth, vulnerability_ids.fifth, 1)
+  end
+
+  private
+
+  def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
+    vulnerabilities.create!(
+      project_id: project_id,
+      author_id: author_id,
+      title: title,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type
+    )
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+  def create_finding!(
+    id: nil,
+    vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
+    name: "test", severity: 7, confidence: 7, report_type: 0,
+    project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
+    metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
+    params = {
+      vulnerability_id: vulnerability_id,
+      project_id: project_id,
+      name: name,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_fingerprint: project_fingerprint,
+      scanner_id: scanner_id,
+      primary_identifier_id: primary_identifier_id,
+      location_fingerprint: location_fingerprint,
+      metadata_version: metadata_version,
+      raw_metadata: raw_metadata,
+      uuid: uuid
+    }
+    params[:id] = id unless id.nil?
+    vulnerabilities_findings.create!(params)
+  end
+  # rubocop:enable Metrics/ParameterLists
+end
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index b8c5af5a911..90298f0e973 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -3618,20 +3618,6 @@ RSpec.describe Ci::Build do
 
       build.scoped_variables
     end
-
-    context 'when variables builder is used' do
-      it 'returns the same variables' do
-        build.user = create(:user)
-
-        allow(build.pipeline).to receive(:use_variables_builder_definitions?).and_return(false)
-        legacy_variables = build.scoped_variables.to_hash
-
-        allow(build.pipeline).to receive(:use_variables_builder_definitions?).and_return(true)
-        new_variables = build.scoped_variables.to_hash
-
-        expect(new_variables).to eq(legacy_variables)
-      end
-    end
   end
 
   describe '#simple_variables_without_dependencies' do
@@ -3642,160 +3628,6 @@ RSpec.describe Ci::Build do
     end
   end
 
-  shared_examples "secret CI variables" do
-    context 'when ref is branch' do
-      let(:pipeline) { create(:ci_pipeline, project: project) }
-      let(:build) { create(:ci_build, ref: 'master', tag: false, pipeline: pipeline, project: project) }
-
-      context 'when ref is protected' do
-        before do
-          create(:protected_branch, :developers_can_merge, name: 'master', project: project)
-        end
-
-        it { is_expected.to include(variable) }
-      end
-
-      context 'when ref is not protected' do
-        it { is_expected.not_to include(variable) }
-      end
-    end
-
-    context 'when ref is tag' do
-      let(:pipeline) { create(:ci_pipeline, project: project) }
-      let(:build) { create(:ci_build, ref: 'v1.1.0', tag: true, pipeline: pipeline, project: project) }
-
-      context 'when ref is protected' do
-        before do
-          create(:protected_tag, project: project, name: 'v*')
-        end
-
-        it { is_expected.to include(variable) }
-      end
-
-      context 'when ref is not protected' do
-        it { is_expected.not_to include(variable) }
-      end
-    end
-
-    context 'when ref is merge request' do
-      let(:merge_request) { create(:merge_request, :with_detached_merge_request_pipeline) }
-      let(:pipeline) { merge_request.pipelines_for_merge_request.first }
-      let(:build) { create(:ci_build, ref: merge_request.source_branch, tag: false, pipeline: pipeline, project: project) }
-
-      context 'when ref is protected' do
-        before do
-          create(:protected_branch, :developers_can_merge, name: merge_request.source_branch, project: project)
-        end
-
-        it 'does not return protected variables as it is not supported for merge request pipelines' do
-          is_expected.not_to include(variable)
-        end
-      end
-
-      context 'when ref is not protected' do
-        it { is_expected.not_to include(variable) }
-      end
-    end
-  end
-
-  describe '#secret_instance_variables' do
-    subject { build.secret_instance_variables }
-
-    let_it_be(:variable) { create(:ci_instance_variable, protected: true) }
-
-    include_examples "secret CI variables"
-  end
-
-  describe '#secret_group_variables' do
-    subject { build.secret_group_variables }
-
-    let_it_be(:variable) { create(:ci_group_variable, protected: true, group: group) }
-
-    include_examples "secret CI variables"
-  end
-
-  describe '#secret_project_variables' do
-    subject { build.secret_project_variables }
-
-    let_it_be(:variable) { create(:ci_variable, protected: true, project: project) }
-
-    include_examples "secret CI variables"
-  end
-
-  describe '#kubernetes_variables' do
-    let(:build) { create(:ci_build) }
-    let(:service) { double(execute: template) }
-    let(:template) { double(to_yaml: 'example-kubeconfig', valid?: template_valid) }
-    let(:template_valid) { true }
-
-    subject { build.kubernetes_variables }
-
-    before do
-      allow(Ci::GenerateKubeconfigService).to receive(:new).with(build).and_return(service)
-    end
-
-    it { is_expected.to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
-
-    context 'generated config is invalid' do
-      let(:template_valid) { false }
-
-      it { is_expected.not_to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
-    end
-  end
-
-  describe '#deployment_variables' do
-    let(:build) { create(:ci_build, environment: environment) }
-    let(:environment) { 'production' }
-    let(:kubernetes_namespace) { 'namespace' }
-    let(:project_variables) { double }
-
-    subject { build.deployment_variables(environment: environment) }
-
-    before do
-      allow(build).to receive(:expanded_kubernetes_namespace)
-        .and_return(kubernetes_namespace)
-
-      allow(build.project).to receive(:deployment_variables)
-        .with(environment: environment, kubernetes_namespace: kubernetes_namespace)
-        .and_return(project_variables)
-    end
-
-    context 'environment is nil' do
-      let(:environment) { nil }
-
-      it { is_expected.to be_empty }
-    end
-  end
-
-  describe '#user_variables' do
-    subject { build.user_variables.to_hash }
-
-    context 'with user' do
-      let(:expected_variables) do
-        {
-          'GITLAB_USER_EMAIL' => user.email,
-          'GITLAB_USER_ID' => user.id.to_s,
-          'GITLAB_USER_LOGIN' => user.username,
-          'GITLAB_USER_NAME' => user.name
-        }
-      end
-
-      before do
-        build.user = user
-      end
-
-      it { is_expected.to eq(expected_variables) }
-    end
-
-    context 'without user' do
-      before do
-        expect(build).to receive(:user).and_return(nil)
-      end
-
-      it { is_expected.to be_empty }
-    end
-  end
-
   describe '#any_unmet_prerequisites?' do
     let(:build) { create(:ci_build, :created) }