Add latest changes from gitlab-org/gitlab@master

11 files changed, 83 insertions, 43 deletions

diff --git a/spec/services/ci/pipeline_processing/atomic_processing_service/status_collection_spec.rb b/spec/services/ci/pipeline_processing/atomic_processing_service/status_collection_spec.rb
index b487730d07f..de3c7713ac8 100644
--- a/spec/services/ci/pipeline_processing/atomic_processing_service/status_collection_spec.rb
+++ b/spec/services/ci/pipeline_processing/atomic_processing_service/status_collection_spec.rb
@@ -18,7 +18,7 @@ describe Ci::PipelineProcessing::AtomicProcessingService::StatusCollection do
     it 'does update existing status of processable' do
       collection.set_processable_status(test_a.id, 'success', 100)
 
-      expect(collection.status_for_names(['test-a'])).to eq('success')
+      expect(collection.status_for_names(['test-a'], dag: false)).to eq('success')
     end
 
     it 'ignores a missing processable' do
@@ -33,15 +33,18 @@ describe Ci::PipelineProcessing::AtomicProcessingService::StatusCollection do
   end
 
   describe '#status_for_names' do
-    where(:names, :status) do
-      %w[build-a]         | 'success'
-      %w[build-a build-b] | 'failed'
-      %w[build-a test-a]  | 'running'
+    where(:names, :status, :dag) do
+      %w[build-a]         | 'success' | false
+      %w[build-a build-b] | 'failed'  | false
+      %w[build-a test-a]  | 'running' | false
+      %w[build-a]         | 'success' | true
+      %w[build-a build-b] | 'failed'  | true
+      %w[build-a test-a]  | 'pending' | true
     end
 
     with_them do
       it 'returns composite status of given names' do
-        expect(collection.status_for_names(names)).to eq(status)
+        expect(collection.status_for_names(names, dag: dag)).to eq(status)
       end
     end
   end
diff --git a/spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb b/spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb
index 13448ea759b..3b66ecff196 100644
--- a/spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb
+++ b/spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb
@@ -2,17 +2,19 @@
 
 require 'spec_helper'
 require_relative 'shared_processing_service.rb'
-# require_relative 'shared_processing_service_tests_with_yaml.rb'
+require_relative 'shared_processing_service_tests_with_yaml.rb'
 
 describe Ci::PipelineProcessing::AtomicProcessingService do
   before do
     stub_feature_flags(ci_atomic_processing: true)
+
+    # This feature flag is implicit
+    # Atomic Processing does not process statuses differently
+    stub_feature_flags(ci_composite_status: true)
   end
 
   it_behaves_like 'Pipeline Processing Service'
-  # TODO: This needs to be enabled. There is a different behavior when using `needs` depending on
-  # a `manual` job. More info: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29405#note_327520605
-  # it_behaves_like 'Pipeline Processing Service Tests With Yaml'
+  it_behaves_like 'Pipeline Processing Service Tests With Yaml'
 
   private
 
diff --git a/spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb b/spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb
index d4fb03cf643..fd491bf461b 100644
--- a/spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb
+++ b/spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb
@@ -7,11 +7,25 @@ require_relative 'shared_processing_service_tests_with_yaml.rb'
 describe Ci::PipelineProcessing::LegacyProcessingService do
   before do
     stub_feature_flags(ci_atomic_processing: false)
-    stub_feature_flags(ci_composite_status: false)
   end
 
-  it_behaves_like 'Pipeline Processing Service'
-  it_behaves_like 'Pipeline Processing Service Tests With Yaml'
+  context 'when ci_composite_status is enabled' do
+    before do
+      stub_feature_flags(ci_composite_status: true)
+    end
+
+    it_behaves_like 'Pipeline Processing Service'
+    it_behaves_like 'Pipeline Processing Service Tests With Yaml'
+  end
+
+  context 'when ci_composite_status is disabled' do
+    before do
+      stub_feature_flags(ci_composite_status: false)
+    end
+
+    it_behaves_like 'Pipeline Processing Service'
+    it_behaves_like 'Pipeline Processing Service Tests With Yaml'
+  end
 
   private
 
diff --git a/spec/services/ci/pipeline_processing/shared_processing_service.rb b/spec/services/ci/pipeline_processing/shared_processing_service.rb
index ffe5eacfc48..29fa43001ae 100644
--- a/spec/services/ci/pipeline_processing/shared_processing_service.rb
+++ b/spec/services/ci/pipeline_processing/shared_processing_service.rb
@@ -816,10 +816,10 @@ shared_examples 'Pipeline Processing Service' do
   context 'when a needed job is skipped', :sidekiq_inline do
     let!(:linux_build) { create_build('linux:build', stage: 'build', stage_idx: 0) }
     let!(:linux_rspec) { create_build('linux:rspec', stage: 'test', stage_idx: 1) }
-    let!(:deploy) do
-      create_build('deploy', stage: 'deploy', stage_idx: 2, scheduling_type: :dag, needs: [
-        create(:ci_build_need, name: 'linux:rspec')
-      ])
+    let!(:deploy) { create_build('deploy', stage: 'deploy', stage_idx: 2, scheduling_type: :dag) }
+
+    before do
+      create(:ci_build_need, build: deploy, name: 'linux:build')
     end
 
     it 'skips the jobs depending on it' do
@@ -836,6 +836,23 @@ shared_examples 'Pipeline Processing Service' do
     end
   end
 
+  context 'when a needed job is manual', :sidekiq_inline do
+    let!(:linux_build) { create_build('linux:build', stage: 'build', stage_idx: 0, when: 'manual', allow_failure: true) }
+    let!(:deploy) { create_build('deploy', stage: 'deploy', stage_idx: 1, scheduling_type: :dag) }
+
+    before do
+      create(:ci_build_need, build: deploy, name: 'linux:build')
+    end
+
+    it 'makes deploy DAG to be waiting for optional manual to finish' do
+      expect(process_pipeline).to be_truthy
+
+      expect(stages).to eq(%w(skipped created))
+      expect(all_builds.manual).to contain_exactly(linux_build)
+      expect(all_builds.created).to contain_exactly(deploy)
+    end
+  end
+
   private
 
   def all_builds
diff --git a/spec/services/ci/pipeline_processing/shared_processing_service_tests_with_yaml.rb b/spec/services/ci/pipeline_processing/shared_processing_service_tests_with_yaml.rb
index ec3c10b3bb5..93f83f0ea3b 100644
--- a/spec/services/ci/pipeline_processing/shared_processing_service_tests_with_yaml.rb
+++ b/spec/services/ci/pipeline_processing/shared_processing_service_tests_with_yaml.rb
@@ -18,36 +18,40 @@ shared_context 'Pipeline Processing Service Tests With Yaml' do
       project.add_developer(user)
     end
 
-    it 'follows transitions', :sidekiq_inline do
+    it 'follows transitions' do
       expect(pipeline).to be_persisted
-      check_expectation(test_file.dig('init', 'expect'))
+      Sidekiq::Worker.drain_all # ensure that all async jobs are executed
+      check_expectation(test_file.dig('init', 'expect'), "init")
 
-      test_file['transitions'].each do |transition|
+      test_file['transitions'].each_with_index do |transition, idx|
         event_on_jobs(transition['event'], transition['jobs'])
-        check_expectation(transition['expect'])
+        Sidekiq::Worker.drain_all # ensure that all async jobs are executed
+        check_expectation(transition['expect'], "transition:#{idx}")
       end
     end
 
     private
 
-    def check_expectation(expectation)
-      expectation.each do |key, value|
-        case key
-        when 'pipeline'
-          expect(pipeline.reload.status).to eq(value)
-        when 'stages'
-          expect(pipeline.stages.pluck(:name, :status).to_h).to eq(value)
-        when 'jobs'
-          expect(pipeline.builds.latest.pluck(:name, :status).to_h).to eq(value)
-        end
-      end
+    def check_expectation(expectation, message)
+      expect(current_state.deep_stringify_keys).to eq(expectation), message
+    end
+
+    def current_state
+      # reload pipeline and all relations
+      pipeline.reload
+
+      {
+        pipeline: pipeline.status,
+        stages: pipeline.ordered_stages.pluck(:name, :status).to_h,
+        jobs: pipeline.statuses.latest.pluck(:name, :status).to_h
+      }
     end
 
     def event_on_jobs(event, job_names)
-      builds = pipeline.builds.latest.where(name: job_names).to_a
-      expect(builds.count).to eq(job_names.count) # ensure that we have the same counts
+      statuses = pipeline.statuses.latest.by_name(job_names).to_a
+      expect(statuses.count).to eq(job_names.count) # ensure that we have the same counts
 
-      builds.each { |build| build.public_send("#{event}!") }
+      statuses.each { |status| status.public_send("#{event}!") }
     end
   end
 end
diff --git a/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_false.yml b/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_false.yml
index 1a1412f2a11..1d61cd24f8c 100644
--- a/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_false.yml
+++ b/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_false.yml
@@ -24,9 +24,9 @@ transitions:
   - event: enqueue
     jobs: [test]
     expect:
-      pipeline: manual
+      pipeline: pending
       stages:
-        test: manual
+        test: pending
         deploy: created
       jobs:
         test: pending
diff --git a/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_true.yml b/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_true.yml
index 39545617926..d8ca563b141 100644
--- a/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_true.yml
+++ b/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_true.yml
@@ -26,7 +26,7 @@ transitions:
     expect:
       pipeline: pending
       stages:
-        test: running
+        test: pending
         deploy: created
       jobs:
         test: pending
diff --git a/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_true_deploy_on_failure.yml b/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_true_deploy_on_failure.yml
index 0c6a5a7e364..d375c6a49e0 100644
--- a/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_true_deploy_on_failure.yml
+++ b/spec/services/ci/pipeline_processing/test_cases/dag_test_manual_allow_failure_true_deploy_on_failure.yml
@@ -27,7 +27,7 @@ transitions:
     expect:
       pipeline: pending
       stages:
-        test: running
+        test: pending
         deploy: created
       jobs:
         test: pending
diff --git a/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_false.yml b/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_false.yml
index d37288b2075..2ffa35b56d7 100644
--- a/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_false.yml
+++ b/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_false.yml
@@ -23,9 +23,9 @@ transitions:
   - event: enqueue
     jobs: [test]
     expect:
-      pipeline: manual
+      pipeline: pending
       stages:
-        test: manual
+        test: pending
         deploy: created
       jobs:
         test: pending
diff --git a/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_true.yml b/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_true.yml
index bac339cc58d..088fab5ca09 100644
--- a/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_true.yml
+++ b/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_true.yml
@@ -36,7 +36,7 @@ transitions:
     expect:
       pipeline: running
       stages:
-        test: running
+        test: pending
         deploy: success
       jobs:
         test: pending
diff --git a/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_true_deploy_on_failure.yml b/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_true_deploy_on_failure.yml
index ab1b210848a..2b30316aef6 100644
--- a/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_true_deploy_on_failure.yml
+++ b/spec/services/ci/pipeline_processing/test_cases/stage_test_manual_allow_failure_true_deploy_on_failure.yml
@@ -26,7 +26,7 @@ transitions:
     expect:
       pipeline: pending
       stages:
-        test: running
+        test: pending
         deploy: skipped
       jobs:
         test: pending