Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42

7 files changed, 176 insertions, 149 deletions

diff --git a/spec/models/ci/bridge_spec.rb b/spec/models/ci/bridge_spec.rb
index 6dd3c40f228..8f1ae9c5f02 100644
--- a/spec/models/ci/bridge_spec.rb
+++ b/spec/models/ci/bridge_spec.rb
@@ -17,6 +17,8 @@ RSpec.describe Ci::Bridge do
     { trigger: { project: 'my/project', branch: 'master' } }
   end
 
+  it { is_expected.to respond_to(:runner_features) }
+
   it 'has many sourced pipelines' do
     expect(bridge).to have_many(:sourced_pipelines)
   end
@@ -76,7 +78,7 @@ RSpec.describe Ci::Bridge do
 
           bridge.enqueue!
 
-          expect(::Ci::CreateCrossProjectPipelineWorker.jobs.last['args']).to eq([bridge.id])
+          expect(::Ci::CreateDownstreamPipelineWorker.jobs.last['args']).to eq([bridge.id])
         end
       end
 
@@ -85,7 +87,7 @@ RSpec.describe Ci::Bridge do
 
         bridge.enqueue_waiting_for_resource!
 
-        expect(::Ci::CreateCrossProjectPipelineWorker.jobs.last['args']).to eq([bridge.id])
+        expect(::Ci::CreateDownstreamPipelineWorker.jobs.last['args']).to match_array([bridge.id])
       end
 
       it 'raises error when the status is failed' do
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 1e06d566c80..2ebf75a1d8a 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -29,11 +29,13 @@ RSpec.describe Ci::Build do
   it { is_expected.to have_one(:deployment) }
   it { is_expected.to have_one(:runner_session) }
   it { is_expected.to have_one(:trace_metadata) }
+  it { is_expected.to have_many(:terraform_state_versions).dependent(:nullify).inverse_of(:build) }
 
   it { is_expected.to validate_presence_of(:ref) }
 
   it { is_expected.to respond_to(:has_trace?) }
   it { is_expected.to respond_to(:trace) }
+  it { is_expected.to respond_to(:runner_features) }
 
   it { is_expected.to delegate_method(:merge_request?).to(:pipeline) }
   it { is_expected.to delegate_method(:merge_request_ref?).to(:pipeline) }
@@ -345,10 +347,10 @@ RSpec.describe Ci::Build do
   end
 
   describe '#stick_build_if_status_changed' do
-    it 'sticks the build if the status changed', :db_load_balancing do
+    it 'sticks the build if the status changed' do
       job = create(:ci_build, :pending)
 
-      expect(Gitlab::Database::LoadBalancing::Sticking).to receive(:stick)
+      expect(ApplicationRecord.sticking).to receive(:stick)
         .with(:build, job.id)
 
       job.update!(status: :running)
@@ -1288,7 +1290,7 @@ RSpec.describe Ci::Build do
     end
   end
 
-  describe 'state transition as a deployable' do
+  shared_examples_for 'state transition as a deployable' do
     subject { build.send(event) }
 
     let!(:build) { create(:ci_build, :with_deployment, :start_review_app, project: project, pipeline: pipeline) }
@@ -1397,6 +1399,36 @@ RSpec.describe Ci::Build do
     end
   end
 
+  it_behaves_like 'state transition as a deployable' do
+    context 'when transits to running' do
+      let(:event) { :run! }
+
+      context 'when deployment is already running state' do
+        before do
+          build.deployment.success!
+        end
+
+        it 'does not change deployment status and tracks an error' do
+          expect(Gitlab::ErrorTracking)
+            .to receive(:track_exception).with(
+              instance_of(Deployment::StatusSyncError), deployment_id: deployment.id, build_id: build.id)
+
+          with_cross_database_modification_prevented do
+            expect { subject }.not_to change { deployment.reload.status }
+          end
+        end
+      end
+    end
+  end
+
+  context 'when update_deployment_after_transaction_commit feature flag is disabled' do
+    before do
+      stub_feature_flags(update_deployment_after_transaction_commit: false)
+    end
+
+    it_behaves_like 'state transition as a deployable'
+  end
+
   describe '#on_stop' do
     subject { build.on_stop }
 
@@ -3946,7 +3978,7 @@ RSpec.describe Ci::Build do
       end
 
       it 'can drop the build' do
-        expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception)
+        expect(Gitlab::ErrorTracking).to receive(:track_exception)
 
         expect { build.drop! }.not_to raise_error
 
@@ -5288,4 +5320,10 @@ RSpec.describe Ci::Build do
       expect(build.reload.queuing_entry).not_to be_present
     end
   end
+
+  it 'does not generate cross DB queries when a record is created via FactoryBot' do
+    with_cross_database_modification_prevented do
+      create(:ci_build)
+    end
+  end
 end
diff --git a/spec/models/ci/build_trace_metadata_spec.rb b/spec/models/ci/build_trace_metadata_spec.rb
index 5e4645c5dc4..120e4289da2 100644
--- a/spec/models/ci/build_trace_metadata_spec.rb
+++ b/spec/models/ci/build_trace_metadata_spec.rb
@@ -88,14 +88,16 @@ RSpec.describe Ci::BuildTraceMetadata do
   describe '#track_archival!' do
     let(:trace_artifact) { create(:ci_job_artifact) }
     let(:metadata) { create(:ci_build_trace_metadata) }
+    let(:checksum) { SecureRandom.hex }
 
     it 'stores the artifact id and timestamp' do
       expect(metadata.trace_artifact_id).to be_nil
 
-      metadata.track_archival!(trace_artifact.id)
+      metadata.track_archival!(trace_artifact.id, checksum)
       metadata.reload
 
       expect(metadata.trace_artifact_id).to eq(trace_artifact.id)
+      expect(metadata.checksum).to eq(checksum)
       expect(metadata.archived_at).to be_like_time(Time.current)
     end
   end
@@ -131,4 +133,29 @@ RSpec.describe Ci::BuildTraceMetadata do
       end
     end
   end
+
+  describe '#remote_checksum_valid?' do
+    using RSpec::Parameterized::TableSyntax
+
+    let(:metadata) do
+      build(:ci_build_trace_metadata,
+        checksum: checksum,
+        remote_checksum: remote_checksum)
+    end
+
+    subject { metadata.remote_checksum_valid? }
+
+    where(:checksum, :remote_checksum, :result) do
+      nil         | nil         | false
+      nil         | 'a'         | false
+      'a'         | nil         | false
+      'a'         | 'b'         | false
+      'b'         | 'a'         | false
+      'a'         | 'a'         | true
+    end
+
+    with_them do
+      it { is_expected.to eq(result) }
+    end
+  end
 end
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index 1007d64438f..98b55ccb76b 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -35,6 +35,8 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep do
   it { is_expected.to have_many(:sourced_pipelines) }
   it { is_expected.to have_many(:triggered_pipelines) }
   it { is_expected.to have_many(:pipeline_artifacts) }
+  it { is_expected.to have_many(:package_build_infos).dependent(:nullify).inverse_of(:pipeline) }
+  it { is_expected.to have_many(:package_file_build_infos).dependent(:nullify).inverse_of(:pipeline) }
 
   it { is_expected.to have_one(:chat_data) }
   it { is_expected.to have_one(:source_pipeline) }
@@ -1219,32 +1221,6 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep do
                                     %w(test success),
                                     %w(deploy running)])
           end
-
-          context 'when commit status is retried' do
-            let!(:old_commit_status) do
-              create(:commit_status, pipeline: pipeline,
-                                     stage: 'build',
-                                     name: 'mac',
-                                     stage_idx: 0,
-                                     status: 'success')
-            end
-
-            context 'when FF ci_remove_update_retried_from_process_pipeline is disabled' do
-              before do
-                stub_feature_flags(ci_remove_update_retried_from_process_pipeline: false)
-
-                Ci::ProcessPipelineService
-                  .new(pipeline)
-                  .execute
-              end
-
-              it 'ignores the previous state' do
-                expect(statuses).to eq([%w(build success),
-                                        %w(test success),
-                                        %w(deploy running)])
-              end
-            end
-          end
         end
 
         context 'when there is a stage with warnings' do
@@ -2906,121 +2882,30 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep do
     end
   end
 
-  describe '#execute_hooks' do
+  describe 'hooks trigerring' do
     let_it_be(:pipeline) { create(:ci_empty_pipeline, :created) }
 
-    let!(:build_a) { create_build('a', 0) }
-    let!(:build_b) { create_build('b', 0) }
-
-    let!(:hook) do
-      create(:project_hook, pipeline_events: enabled)
-    end
-
-    before do
-      WebHookWorker.drain
-    end
-
-    context 'with pipeline hooks enabled' do
-      let(:enabled) { true }
-
-      before do
-        stub_full_request(hook.url, method: :post)
-      end
-
-      context 'with multiple builds', :sidekiq_inline do
-        context 'when build is queued' do
-          before do
-            build_a.reload.enqueue
-            build_b.reload.enqueue
-          end
-
-          it 'receives a pending event once' do
-            expect(WebMock).to have_requested_pipeline_hook('pending').once
-          end
-
-          it 'builds hook data once' do
-            create(:pipelines_email_integration)
-
-            expect(Gitlab::DataBuilder::Pipeline).to receive(:build).once.and_call_original
-
-            pipeline.execute_hooks
-          end
-        end
-
-        context 'when build is run' do
-          before do
-            build_a.reload.enqueue
-            build_a.reload.run!
-            build_b.reload.enqueue
-            build_b.reload.run!
-          end
-
-          it 'receives a running event once' do
-            expect(WebMock).to have_requested_pipeline_hook('running').once
-          end
-        end
-
-        context 'when all builds succeed' do
-          before do
-            build_a.success
-
-            # We have to reload build_b as this is in next stage and it gets triggered by PipelineProcessWorker
-            build_b.reload.success
-          end
-
-          it 'receives a success event once' do
-            expect(WebMock).to have_requested_pipeline_hook('success').once
-          end
-        end
+    %i[
+      enqueue
+      request_resource
+      prepare
+      run
+      skip
+      drop
+      succeed
+      cancel
+      block
+      delay
+    ].each do |action|
+      context "when pipeline action is #{action}" do
+        let(:pipeline_action) { action }
 
-        context 'when stage one failed' do
-          let!(:build_b) { create_build('b', 1) }
-
-          before do
-            build_a.drop
-          end
+        it 'schedules a new PipelineHooksWorker job' do
+          expect(PipelineHooksWorker).to receive(:perform_async).with(pipeline.id)
 
-          it 'receives a failed event once' do
-            expect(WebMock).to have_requested_pipeline_hook('failed').once
-          end
+          pipeline.reload.public_send(pipeline_action)
         end
-
-        def have_requested_pipeline_hook(status)
-          have_requested(:post, stubbed_hostname(hook.url)).with do |req|
-            json_body = Gitlab::Json.parse(req.body)
-            json_body['object_attributes']['status'] == status &&
-              json_body['builds'].length == 2
-          end
-        end
-      end
-    end
-
-    context 'with pipeline hooks disabled' do
-      let(:enabled) { false }
-
-      before do
-        build_a.enqueue
-        build_b.enqueue
-      end
-
-      it 'did not execute pipeline_hook after touched' do
-        expect(WebMock).not_to have_requested(:post, hook.url)
       end
-
-      it 'does not build hook data' do
-        expect(Gitlab::DataBuilder::Pipeline).not_to receive(:build)
-
-        pipeline.execute_hooks
-      end
-    end
-
-    def create_build(name, stage_idx)
-      create(:ci_build,
-             :created,
-             pipeline: pipeline,
-             name: name,
-             stage: "stage:#{stage_idx}",
-             stage_idx: stage_idx)
     end
   end
 
diff --git a/spec/models/ci/processable_spec.rb b/spec/models/ci/processable_spec.rb
index 0a43f785598..ac1a8247aaa 100644
--- a/spec/models/ci/processable_spec.rb
+++ b/spec/models/ci/processable_spec.rb
@@ -147,13 +147,20 @@ RSpec.describe Ci::Processable do
         end
 
         it 'releases a resource when build finished' do
-          expect(build.resource_group).to receive(:release_resource_from).with(build).and_call_original
+          expect(build.resource_group).to receive(:release_resource_from).with(build).and_return(true).and_call_original
           expect(Ci::ResourceGroups::AssignResourceFromResourceGroupWorker).to receive(:perform_async).with(build.resource_group_id)
 
           build.enqueue_waiting_for_resource!
           build.success!
         end
 
+        it 're-checks the resource group even if the processable does not retain a resource' do
+          expect(build.resource_group).to receive(:release_resource_from).with(build).and_return(false).and_call_original
+          expect(Ci::ResourceGroups::AssignResourceFromResourceGroupWorker).to receive(:perform_async).with(build.resource_group_id)
+
+          build.success!
+        end
+
         context 'when build has prerequisites' do
           before do
             allow(build).to receive(:any_unmet_prerequisites?) { true }
diff --git a/spec/models/ci/resource_group_spec.rb b/spec/models/ci/resource_group_spec.rb
index 50a786419f2..aae16157fbf 100644
--- a/spec/models/ci/resource_group_spec.rb
+++ b/spec/models/ci/resource_group_spec.rb
@@ -85,4 +85,61 @@ RSpec.describe Ci::ResourceGroup do
       end
     end
   end
+
+  describe '#upcoming_processables' do
+    subject { resource_group.upcoming_processables }
+
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:pipeline_1) { create(:ci_pipeline, project: project) }
+    let_it_be(:pipeline_2) { create(:ci_pipeline, project: project) }
+
+    let!(:resource_group) { create(:ci_resource_group, process_mode: process_mode, project: project) }
+
+    Ci::HasStatus::STATUSES_ENUM.keys.each do |status|
+      let!("build_1_#{status}") { create(:ci_build, pipeline: pipeline_1, status: status, resource_group: resource_group) }
+      let!("build_2_#{status}") { create(:ci_build, pipeline: pipeline_2, status: status, resource_group: resource_group) }
+    end
+
+    context 'when process mode is unordered' do
+      let(:process_mode) { :unordered }
+
+      it 'returns correct jobs in an indeterministic order' do
+        expect(subject).to contain_exactly(build_1_waiting_for_resource, build_2_waiting_for_resource)
+      end
+    end
+
+    context 'when process mode is oldest_first' do
+      let(:process_mode) { :oldest_first }
+
+      it 'returns correct jobs in a specific order' do
+        expect(subject[0]).to eq(build_1_waiting_for_resource)
+        expect(subject[1..2]).to contain_exactly(build_1_created, build_1_scheduled)
+        expect(subject[3]).to eq(build_2_waiting_for_resource)
+        expect(subject[4..5]).to contain_exactly(build_2_created, build_2_scheduled)
+      end
+    end
+
+    context 'when process mode is newest_first' do
+      let(:process_mode) { :newest_first }
+
+      it 'returns correct jobs in a specific order' do
+        expect(subject[0]).to eq(build_2_waiting_for_resource)
+        expect(subject[1..2]).to contain_exactly(build_2_created, build_2_scheduled)
+        expect(subject[3]).to eq(build_1_waiting_for_resource)
+        expect(subject[4..5]).to contain_exactly(build_1_created, build_1_scheduled)
+      end
+    end
+
+    context 'when process mode is unknown' do
+      let(:process_mode) { :unordered }
+
+      before do
+        resource_group.update_column(:process_mode, 3)
+      end
+
+      it 'returns empty' do
+        is_expected.to be_empty
+      end
+    end
+  end
 end
diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb
index 31e854c852e..826332268c5 100644
--- a/spec/models/ci/runner_spec.rb
+++ b/spec/models/ci/runner_spec.rb
@@ -5,6 +5,20 @@ require 'spec_helper'
 RSpec.describe Ci::Runner do
   it_behaves_like 'having unique enum values'
 
+  describe 'groups association' do
+    # Due to other assoctions such as projects this whole spec is allowed to
+    # generate cross-database queries. So we have this temporary spec to
+    # validate that at least groups association does not generate cross-DB
+    # queries.
+    it 'does not create a cross-database query' do
+      runner = create(:ci_runner, :group)
+
+      with_cross_joins_prevented do
+        expect(runner.groups.count).to eq(1)
+      end
+    end
+  end
+
   describe 'validation' do
     it { is_expected.to validate_presence_of(:access_level) }
     it { is_expected.to validate_presence_of(:runner_type) }
@@ -257,7 +271,7 @@ RSpec.describe Ci::Runner do
         expect(subject).to be_truthy
 
         expect(runner).to be_project_type
-        expect(runner.projects).to eq([project])
+        expect(runner.runner_projects.pluck(:project_id)).to match_array([project.id])
         expect(runner.only_for?(project)).to be_truthy
       end
     end
@@ -383,10 +397,7 @@ RSpec.describe Ci::Runner do
     it 'sticks the runner to the primary and calls the original method' do
       runner = create(:ci_runner)
 
-      allow(Gitlab::Database::LoadBalancing).to receive(:enable?)
-        .and_return(true)
-
-      expect(Gitlab::Database::LoadBalancing::Sticking).to receive(:stick)
+      expect(ApplicationRecord.sticking).to receive(:stick)
         .with(:runner, runner.id)
 
       expect(Gitlab::Workhorse).to receive(:set_key_and_notify)
@@ -724,7 +735,7 @@ RSpec.describe Ci::Runner do
 
       context 'with invalid runner' do
         before do
-          runner.projects = []
+          runner.runner_projects.delete_all
         end
 
         it 'still updates redis cache and database' do