Add latest changes from gitlab-org/gitlab@13-1-stable-ee

10 files changed, 349 insertions, 186 deletions

diff --git a/spec/services/ci/build_report_result_service_spec.rb b/spec/services/ci/build_report_result_service_spec.rb
new file mode 100644
index 00000000000..dbdfc774314
--- /dev/null
+++ b/spec/services/ci/build_report_result_service_spec.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::BuildReportResultService do
+  describe "#execute" do
+    subject(:build_report_result) { described_class.new.execute(build) }
+
+    context 'when build is finished' do
+      let(:build) { create(:ci_build, :success, :test_reports) }
+
+      it 'creates a build report result entry', :aggregate_failures do
+        expect(build_report_result.tests_name).to eq("test")
+        expect(build_report_result.tests_success).to eq(2)
+        expect(build_report_result.tests_failed).to eq(2)
+        expect(build_report_result.tests_errored).to eq(0)
+        expect(build_report_result.tests_skipped).to eq(0)
+        expect(build_report_result.tests_duration).to eq(0.010284)
+        expect(Ci::BuildReportResult.count).to eq(1)
+      end
+
+      context 'when feature is disable' do
+        it 'does not persist the data' do
+          stub_feature_flags(build_report_summary: false)
+
+          subject
+
+          expect(Ci::BuildReportResult.count).to eq(0)
+        end
+      end
+
+      context 'when data has already been persisted' do
+        it 'raises an error and do not persist the same data twice' do
+          expect { 2.times { described_class.new.execute(build) } }.to raise_error(ActiveRecord::RecordNotUnique)
+
+          expect(Ci::BuildReportResult.count).to eq(1)
+        end
+      end
+    end
+
+    context 'when build is running and test report does not exist' do
+      let(:build) { create(:ci_build, :running) }
+
+      it 'does not persist data' do
+        subject
+
+        expect(Ci::BuildReportResult.count).to eq(0)
+      end
+    end
+  end
+end
diff --git a/spec/services/ci/create_cross_project_pipeline_service_spec.rb b/spec/services/ci/create_cross_project_pipeline_service_spec.rb
index 5c59aaa4ce9..9e2497854bc 100644
--- a/spec/services/ci/create_cross_project_pipeline_service_spec.rb
+++ b/spec/services/ci/create_cross_project_pipeline_service_spec.rb
@@ -487,10 +487,11 @@ describe Ci::CreateCrossProjectPipelineService, '#execute' do
       end
 
       it 'does not create a pipeline and drops the bridge' do
-        service.execute(bridge)
+        expect { service.execute(bridge) }.not_to change(downstream_project.ci_pipelines, :count)
 
         expect(bridge.reload).to be_failed
         expect(bridge.failure_reason).to eq('downstream_pipeline_creation_failed')
+        expect(bridge.options[:downstream_errors]).to eq(['Reference not found'])
       end
     end
 
@@ -509,10 +510,35 @@ describe Ci::CreateCrossProjectPipelineService, '#execute' do
       end
 
       it 'does not create a pipeline and drops the bridge' do
-        service.execute(bridge)
+        expect { service.execute(bridge) }.not_to change(downstream_project.ci_pipelines, :count)
+
+        expect(bridge.reload).to be_failed
+        expect(bridge.failure_reason).to eq('downstream_pipeline_creation_failed')
+        expect(bridge.options[:downstream_errors]).to eq(['No stages / jobs for this pipeline.'])
+      end
+    end
+
+    context 'when downstream pipeline has invalid YAML' do
+      before do
+        stub_ci_pipeline_yaml_file(config)
+      end
+
+      let(:config) do
+        <<-EOY
+          test:
+            stage: testx
+            script: echo 1
+        EOY
+      end
+
+      it 'creates the pipeline but drops the bridge' do
+        expect { service.execute(bridge) }.to change(downstream_project.ci_pipelines, :count).by(1)
 
         expect(bridge.reload).to be_failed
         expect(bridge.failure_reason).to eq('downstream_pipeline_creation_failed')
+        expect(bridge.options[:downstream_errors]).to eq(
+          ['test job: chosen stage does not exist; available stages are .pre, build, test, deploy, .post']
+        )
       end
     end
   end
diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb
index 681ce9669e2..b9456d5fcd4 100644
--- a/spec/services/ci/create_pipeline_service_spec.rb
+++ b/spec/services/ci/create_pipeline_service_spec.rb
@@ -77,6 +77,18 @@ describe Ci::CreatePipelineService do
         pipeline
       end
 
+      it 'records pipeline size in a prometheus histogram' do
+        histogram = spy('pipeline size histogram')
+
+        allow(Gitlab::Ci::Pipeline::Chain::Metrics)
+          .to receive(:new).and_return(histogram)
+
+        execute_service
+
+        expect(histogram).to have_received(:observe)
+          .with({ source: 'push' }, 5)
+      end
+
       context 'when merge requests already exist for this source branch' do
         let(:merge_request_1) do
           create(:merge_request, source_branch: 'feature', target_branch: "master", source_project: project)
diff --git a/spec/services/ci/create_web_ide_terminal_service_spec.rb b/spec/services/ci/create_web_ide_terminal_service_spec.rb
new file mode 100644
index 00000000000..2cc67c7cd1d
--- /dev/null
+++ b/spec/services/ci/create_web_ide_terminal_service_spec.rb
@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::CreateWebIdeTerminalService do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
+  let(:ref) { 'master' }
+
+  describe '#execute' do
+    subject { described_class.new(project, user, ref: ref).execute }
+
+    context 'for maintainer' do
+      shared_examples 'be successful' do
+        it 'returns a success with pipeline object' do
+          is_expected.to include(status: :success)
+
+          expect(subject[:pipeline]).to be_a(Ci::Pipeline)
+          expect(subject[:pipeline]).to be_persisted
+          expect(subject[:pipeline].stages.count).to eq(1)
+          expect(subject[:pipeline].builds.count).to eq(1)
+        end
+      end
+
+      before do
+        project.add_maintainer(user)
+      end
+
+      context 'when web-ide has valid configuration' do
+        before do
+          stub_webide_config_file(config_content)
+        end
+
+        context 'for empty configuration' do
+          let(:config_content) do
+            'terminal: {}'
+          end
+
+          it_behaves_like 'be successful'
+        end
+
+        context 'for configuration with container image' do
+          let(:config_content) do
+            'terminal: { image: ruby }'
+          end
+
+          it_behaves_like 'be successful'
+        end
+
+        context 'for configuration with ports' do
+          let(:config_content) do
+            <<-EOS
+              terminal:
+                image:
+                  name: ruby:2.7
+                  ports:
+                    - 80
+                script: rspec
+                services:
+                  - name: test
+                    alias: test
+                    ports:
+                      - 8080
+            EOS
+          end
+
+          it_behaves_like 'be successful'
+        end
+      end
+    end
+
+    context 'error handling' do
+      shared_examples 'having an error' do |message|
+        it 'returns an error' do
+          is_expected.to eq(
+            status: :error,
+            message: message
+          )
+        end
+      end
+
+      shared_examples 'having insufficient permissions' do
+        it_behaves_like 'having an error', 'Insufficient permissions to create a terminal'
+      end
+
+      context 'when user is developer' do
+        before do
+          project.add_developer(user)
+        end
+
+        it_behaves_like 'having insufficient permissions'
+      end
+
+      context 'when user is maintainer' do
+        before do
+          project.add_maintainer(user)
+        end
+
+        context 'when terminal is already running' do
+          let!(:webide_pipeline) { create(:ci_pipeline, :webide, :running, project: project, user: user) }
+
+          it_behaves_like 'having an error', 'There is already a terminal running'
+        end
+
+        context 'when ref is non-existing' do
+          let(:ref) { 'non-existing-ref' }
+
+          it_behaves_like 'having an error', 'Ref does not exist'
+        end
+
+        context 'when ref is a tag' do
+          let(:ref) { 'v1.0.0' }
+
+          it_behaves_like 'having an error', 'Ref needs to be a branch'
+        end
+
+        context 'when terminal config is missing' do
+          let(:ref) { 'v1.0.0' }
+
+          it_behaves_like 'having an error', 'Ref needs to be a branch'
+        end
+
+        context 'when webide config is present' do
+          before do
+            stub_webide_config_file(config_content)
+          end
+
+          context 'config has invalid content' do
+            let(:config_content) { 'invalid' }
+
+            it_behaves_like 'having an error', 'Invalid configuration format'
+          end
+
+          context 'config is valid, but does not have terminal' do
+            let(:config_content) { '{}' }
+
+            it_behaves_like 'having an error', 'Terminal is not configured'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/services/ci/expire_pipeline_cache_service_spec.rb b/spec/services/ci/expire_pipeline_cache_service_spec.rb
index 78e1ba0109a..2962e9dd31e 100644
--- a/spec/services/ci/expire_pipeline_cache_service_spec.rb
+++ b/spec/services/ci/expire_pipeline_cache_service_spec.rb
@@ -10,9 +10,9 @@ describe Ci::ExpirePipelineCacheService do
 
   describe '#execute' do
     it 'invalidates Etag caching for project pipelines path' do
-      pipelines_path = "/#{project.full_path}/pipelines.json"
+      pipelines_path = "/#{project.full_path}/-/pipelines.json"
       new_mr_pipelines_path = "/#{project.full_path}/-/merge_requests/new.json"
-      pipeline_path = "/#{project.full_path}/pipelines/#{pipeline.id}.json"
+      pipeline_path = "/#{project.full_path}/-/pipelines/#{pipeline.id}.json"
 
       expect_any_instance_of(Gitlab::EtagCaching::Store).to receive(:touch).with(pipelines_path)
       expect_any_instance_of(Gitlab::EtagCaching::Store).to receive(:touch).with(new_mr_pipelines_path)
diff --git a/spec/services/ci/generate_terraform_reports_service_spec.rb b/spec/services/ci/generate_terraform_reports_service_spec.rb
index 4d2c60bed2c..008ecf17b3e 100644
--- a/spec/services/ci/generate_terraform_reports_service_spec.rb
+++ b/spec/services/ci/generate_terraform_reports_service_spec.rb
@@ -12,15 +12,23 @@ describe Ci::GenerateTerraformReportsService do
 
     context 'when head pipeline has terraform reports' do
       it 'returns status and data' do
-        result = subject.execute(nil, merge_request.head_pipeline)
-
-        expect(result).to match(
-          status: :parsed,
-          data: match(
-            a_hash_including('tfplan.json' => a_hash_including('create' => 0, 'update' => 1, 'delete' => 0))
-          ),
-          key: an_instance_of(Array)
-        )
+        pipeline = merge_request.head_pipeline
+        result = subject.execute(nil, pipeline)
+
+        pipeline.builds.each do |build|
+          expect(result).to match(
+            status: :parsed,
+            data: match(
+              a_hash_including(build.id.to_s => hash_including(
+                'create' => 0,
+                'delete' => 0,
+                'update' => 1,
+                'job_name' => build.options.dig(:artifacts, :name).to_s
+              ))
+            ),
+            key: an_instance_of(Array)
+          )
+        end
       end
     end
 
diff --git a/spec/services/ci/pipeline_bridge_status_service_spec.rb b/spec/services/ci/pipeline_bridge_status_service_spec.rb
index 0b6ae976d97..7e79d222349 100644
--- a/spec/services/ci/pipeline_bridge_status_service_spec.rb
+++ b/spec/services/ci/pipeline_bridge_status_service_spec.rb
@@ -4,7 +4,7 @@ require 'spec_helper'
 
 describe Ci::PipelineBridgeStatusService do
   let(:user) { build(:user) }
-  let(:project) { build(:project) }
+  let_it_be(:project) { create(:project) }
   let(:pipeline) { build(:ci_pipeline, project: project) }
 
   describe '#execute' do
diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb
index 0aa603b24ae..90c53d4a346 100644
--- a/spec/services/ci/retry_build_service_spec.rb
+++ b/spec/services/ci/retry_build_service_spec.rb
@@ -30,7 +30,7 @@ describe Ci::RetryBuildService do
        created_at updated_at started_at finished_at queued_at erased_by
        erased_at auto_canceled_by job_artifacts job_artifacts_archive
        job_artifacts_metadata job_artifacts_trace job_artifacts_junit
-       job_artifacts_sast job_artifacts_dependency_scanning
+       job_artifacts_sast job_artifacts_secret_detection job_artifacts_dependency_scanning
        job_artifacts_container_scanning job_artifacts_dast
        job_artifacts_license_management job_artifacts_license_scanning
        job_artifacts_performance job_artifacts_lsif
@@ -38,7 +38,8 @@ describe Ci::RetryBuildService do
        job_artifacts_codequality job_artifacts_metrics scheduled_at
        job_variables waiting_for_resource_at job_artifacts_metrics_referee
        job_artifacts_network_referee job_artifacts_dotenv
-       job_artifacts_cobertura needs job_artifacts_accessibility].freeze
+       job_artifacts_cobertura needs job_artifacts_accessibility
+       job_artifacts_requirements].freeze
 
   ignore_accessors =
     %i[type lock_version target_url base_tags trace_sections
@@ -49,7 +50,7 @@ describe Ci::RetryBuildService do
        metadata runner_session trace_chunks upstream_pipeline_id
        artifacts_file artifacts_metadata artifacts_size commands
        resource resource_group_id processed security_scans author
-       pipeline_id].freeze
+       pipeline_id report_results].freeze
 
   shared_examples 'build duplication' do
     let(:another_pipeline) { create(:ci_empty_pipeline, project: project) }
diff --git a/spec/services/ci/update_ci_ref_status_service_spec.rb b/spec/services/ci/update_ci_ref_status_service_spec.rb
deleted file mode 100644
index 8b60586318d..00000000000
--- a/spec/services/ci/update_ci_ref_status_service_spec.rb
+++ /dev/null
@@ -1,169 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe Ci::UpdateCiRefStatusService do
-  describe '#call' do
-    subject { described_class.new(pipeline) }
-
-    shared_examples 'creates ci_ref' do
-      it 'creates a ci_ref with the pipeline attributes' do
-        expect do
-          expect(subject.call).to eq(true)
-        end.to change { Ci::Ref.count }.by(1)
-
-        created_ref = pipeline.reload.ref_status
-        %w[ref tag project status].each do |attr|
-          expect(created_ref[attr]).to eq(pipeline[attr])
-        end
-      end
-
-      it 'calls PipelineNotificationWorker pasing the ref_status' do
-        expect(PipelineNotificationWorker).to receive(:perform_async).with(pipeline.id, ref_status: pipeline.status)
-
-        subject.call
-      end
-    end
-
-    shared_examples 'updates ci_ref' do
-      where(:ref_status, :pipeline_status, :next_status) do
-        [
-          %w[failed success fixed],
-          %w[failed failed failed],
-          %w[success success success],
-          %w[success failed failed]
-        ]
-      end
-
-      with_them do
-        let(:ci_ref) { create(:ci_ref, status: ref_status) }
-        let(:pipeline) { create(:ci_pipeline, status: pipeline_status, project: ci_ref.project, ref: ci_ref.ref) }
-
-        it 'sets ci_ref.status to next_status' do
-          expect do
-            expect(subject.call).to eq(true)
-            expect(ci_ref.reload.status).to eq(next_status)
-          end.not_to change { Ci::Ref.count }
-        end
-
-        it 'calls PipelineNotificationWorker pasing the ref_status' do
-          expect(PipelineNotificationWorker).to receive(:perform_async).with(pipeline.id, ref_status: next_status)
-
-          subject.call
-        end
-      end
-    end
-
-    shared_examples 'does a noop' do
-      it "doesn't change ci_ref" do
-        expect do
-          expect do
-            expect(subject.call).to eq(false)
-          end.not_to change { ci_ref.reload.status }
-        end.not_to change { Ci::Ref.count }
-      end
-
-      it "doesn't call PipelineNotificationWorker" do
-        expect(PipelineNotificationWorker).not_to receive(:perform_async)
-
-        subject.call
-      end
-    end
-
-    context "ci_ref doesn't exists" do
-      let(:pipeline) { create(:ci_pipeline, :success, ref: 'new-ref') }
-
-      it_behaves_like 'creates ci_ref'
-
-      context 'when an ActiveRecord::RecordNotUnique validation is raised' do
-        let(:ci_ref) { create(:ci_ref, status: 'failed') }
-        let(:pipeline) { create(:ci_pipeline, status: :success, project: ci_ref.project, ref: ci_ref.ref) }
-
-        it 'reloads the ci_ref and retries once' do
-          subject.instance_variable_set("@ref", subject.send(:build_ref))
-
-          expect do
-            expect(subject.call).to eq(true)
-          end.not_to change { Ci::Ref.count }
-          expect(ci_ref.reload.status).to eq('fixed')
-        end
-
-        it 'raises error on multiple retries' do
-          allow_any_instance_of(Ci::Ref).to receive(:update)
-            .and_raise(ActiveRecord::RecordNotUnique)
-
-          expect { subject.call }.to raise_error(ActiveRecord::RecordNotUnique)
-        end
-      end
-    end
-
-    context 'ci_ref exists' do
-      let!(:ci_ref) { create(:ci_ref, status: 'failed') }
-      let(:pipeline) { ci_ref.pipelines.first }
-
-      it_behaves_like 'updates ci_ref'
-
-      context 'pipeline status is invalid' do
-        let!(:pipeline) { create(:ci_pipeline, :running, project: ci_ref.project, ref: ci_ref.ref, tag: ci_ref.tag) }
-
-        it_behaves_like 'does a noop'
-      end
-
-      context 'newer pipeline finished' do
-        let(:newer_pipeline) { create(:ci_pipeline, :success, project: ci_ref.project, ref: ci_ref.ref, tag: ci_ref.tag) }
-
-        before do
-          ci_ref.update!(last_updated_by_pipeline: newer_pipeline)
-        end
-
-        it_behaves_like 'does a noop'
-      end
-
-      context 'pipeline is retried' do
-        before do
-          ci_ref.update!(last_updated_by_pipeline: pipeline)
-        end
-
-        it_behaves_like 'updates ci_ref'
-      end
-
-      context 'ref is stale' do
-        let(:pipeline1) { create(:ci_pipeline, :success, project: ci_ref.project, ref: ci_ref.ref, tag: ci_ref.tag) }
-        let(:pipeline2) { create(:ci_pipeline, :success, project: ci_ref.project, ref: ci_ref.ref, tag: ci_ref.tag) }
-
-        it 'reloads the ref and retry' do
-          service1 = described_class.new(pipeline1)
-          service2 = described_class.new(pipeline2)
-
-          service2.send(:ref)
-          service1.call
-          expect(ci_ref.reload.status).to eq('fixed')
-          expect do
-            expect(service2.call).to eq(true)
-            # We expect 'success' in this case rather than 'fixed' because
-            # the ref is correctly reloaded on stale error.
-            expect(ci_ref.reload.status).to eq('success')
-          end.not_to change { Ci::Ref.count }
-        end
-
-        it 'aborts when a newer pipeline finished' do
-          service1 = described_class.new(pipeline1)
-          service2 = described_class.new(pipeline2)
-
-          service2.call
-          expect do
-            expect(service1.call).to eq(false)
-            expect(ci_ref.reload.status).to eq('fixed')
-          end.not_to change { Ci::Ref.count }
-        end
-      end
-
-      context 'ref exists as both tag/branch and tag' do
-        let(:pipeline) { create(:ci_pipeline, :failed, project: ci_ref.project, ref: ci_ref.ref, tag: true) }
-        let!(:branch_pipeline) { create(:ci_pipeline, :success, project: ci_ref.project, ref: ci_ref.ref, tag: false) }
-
-        it_behaves_like 'creates ci_ref'
-      end
-    end
-  end
-end
diff --git a/spec/services/ci/web_ide_config_service_spec.rb b/spec/services/ci/web_ide_config_service_spec.rb
new file mode 100644
index 00000000000..7522103ccb7
--- /dev/null
+++ b/spec/services/ci/web_ide_config_service_spec.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::WebIdeConfigService do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
+  let(:sha) { 'sha' }
+
+  describe '#execute' do
+    subject { described_class.new(project, user, sha: sha).execute }
+
+    context 'when insufficient permission' do
+      it 'returns an error' do
+        is_expected.to include(
+          status: :error,
+          message: 'Insufficient permissions to read configuration')
+      end
+    end
+
+    context 'for developer' do
+      before do
+        project.add_developer(user)
+      end
+
+      context 'when file is missing' do
+        it 'returns an error' do
+          is_expected.to include(
+            status: :error,
+            message: "Failed to load Web IDE config file '.gitlab/.gitlab-webide.yml' for sha")
+        end
+      end
+
+      context 'when file is present' do
+        before do
+          allow(project.repository).to receive(:blob_data_at).with('sha', anything) do
+            config_content
+          end
+        end
+
+        context 'content is not valid' do
+          let(:config_content) { 'invalid content' }
+
+          it 'returns an error' do
+            is_expected.to include(
+              status: :error,
+              message: "Invalid configuration format")
+          end
+        end
+
+        context 'content is valid, but terminal not defined' do
+          let(:config_content) { '{}' }
+
+          it 'returns success' do
+            is_expected.to include(
+              status: :success,
+              terminal: nil)
+          end
+        end
+
+        context 'content is valid, with enabled terminal' do
+          let(:config_content) { 'terminal: {}' }
+
+          it 'returns success' do
+            is_expected.to include(
+              status: :success,
+              terminal: {
+                tag_list: [],
+                yaml_variables: [],
+                options: { script: ["sleep 60"] }
+              })
+          end
+        end
+
+        context 'content is valid, with custom terminal' do
+          let(:config_content) { 'terminal: { before_script: [ls] }' }
+
+          it 'returns success' do
+            is_expected.to include(
+              status: :success,
+              terminal: {
+                tag_list: [],
+                yaml_variables: [],
+                options: { before_script: ["ls"], script: ["sleep 60"] }
+              })
+          end
+        end
+      end
+    end
+  end
+end