Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

14 files changed, 526 insertions, 82 deletions

diff --git a/spec/lib/gitlab/ci/build/status/reason_spec.rb b/spec/lib/gitlab/ci/build/status/reason_spec.rb
new file mode 100644
index 00000000000..64f35c3f464
--- /dev/null
+++ b/spec/lib/gitlab/ci/build/status/reason_spec.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Build::Status::Reason do
+  let(:build) { double('build') }
+
+  describe '.fabricate' do
+    context 'when failure symbol reason is being passed' do
+      it 'correctly fabricates a status reason object' do
+        reason = described_class.fabricate(build, :script_failure)
+
+        expect(reason.failure_reason_enum).to eq 1
+      end
+    end
+
+    context 'when another status reason object is being passed' do
+      it 'correctly fabricates a status reason object' do
+        reason = described_class.fabricate(build, :script_failure)
+
+        new_reason = described_class.fabricate(build, reason)
+
+        expect(new_reason.failure_reason_enum).to eq 1
+      end
+    end
+  end
+
+  describe '#failure_reason_enum' do
+    it 'exposes a failure reason enum' do
+      reason = described_class.fabricate(build, :script_failure)
+
+      enum = ::CommitStatus.failure_reasons[:script_failure]
+
+      expect(reason.failure_reason_enum).to eq enum
+    end
+  end
+
+  describe '#force_allow_failure?' do
+    context 'when build is not allowed to fail' do
+      context 'when build is allowed to fail with a given exit code' do
+        it 'returns true' do
+          reason = described_class.new(build, :script_failure, 11)
+
+          allow(build).to receive(:allow_failure?).and_return(false)
+          allow(build).to receive(:allowed_to_fail_with_code?)
+            .with(11)
+            .and_return(true)
+
+          expect(reason.force_allow_failure?).to be true
+        end
+      end
+
+      context 'when build is not allowed to fail regardless of an exit code' do
+        it 'returns false' do
+          reason = described_class.new(build, :script_failure, 11)
+
+          allow(build).to receive(:allow_failure?).and_return(false)
+          allow(build).to receive(:allowed_to_fail_with_code?)
+            .with(11)
+            .and_return(false)
+
+          expect(reason.force_allow_failure?).to be false
+        end
+      end
+
+      context 'when an exit code is not specified' do
+        it 'returns false' do
+          reason = described_class.new(build, :script_failure)
+
+          expect(reason.force_allow_failure?).to be false
+        end
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/config/entry/root_spec.rb b/spec/lib/gitlab/ci/config/entry/root_spec.rb
index d862fbf5b78..749d1386ed9 100644
--- a/spec/lib/gitlab/ci/config/entry/root_spec.rb
+++ b/spec/lib/gitlab/ci/config/entry/root_spec.rb
@@ -3,7 +3,9 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Config::Entry::Root do
-  let(:root) { described_class.new(hash) }
+  let(:user) {}
+  let(:project) {}
+  let(:root) { described_class.new(hash, user: user, project: project) }
 
   describe '.nodes' do
     it 'returns a hash' do
@@ -53,6 +55,37 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do
         }
       end
 
+      context 'when deprecated types keyword is defined' do
+        let(:project) { create(:project, :repository) }
+        let(:user) { create(:user) }
+
+        let(:hash) do
+          { types: %w(test deploy),
+            rspec: { script: 'rspec' } }
+        end
+
+        before do
+          root.compose!
+        end
+
+        it 'returns array of types as stages with a warning' do
+          expect(root.stages_value).to eq %w[test deploy]
+          expect(root.warnings).to match_array(["root `types` is deprecated in 9.0 and will be removed in 15.0."])
+        end
+
+        it 'logs usage of types keyword' do
+          expect(Gitlab::AppJsonLogger).to(
+            receive(:info)
+              .with(event: 'ci_used_deprecated_keyword',
+                    entry: root[:stages].key.to_s,
+                    user_id: user.id,
+                    project_id: project.id)
+          )
+
+          root.compose!
+        end
+      end
+
       describe '#compose!' do
         before do
           root.compose!
@@ -108,17 +141,6 @@ RSpec.describe Gitlab::Ci::Config::Entry::Root do
               expect(root.stages_value).to eq %w[build pages release]
             end
           end
-
-          context 'when deprecated types key defined' do
-            let(:hash) do
-              { types: %w(test deploy),
-                rspec: { script: 'rspec' } }
-            end
-
-            it 'returns array of types as stages' do
-              expect(root.stages_value).to eq %w[test deploy]
-            end
-          end
         end
 
         describe '#jobs_value' do
diff --git a/spec/lib/gitlab/ci/jwt_v2_spec.rb b/spec/lib/gitlab/ci/jwt_v2_spec.rb
new file mode 100644
index 00000000000..33aaa145a39
--- /dev/null
+++ b/spec/lib/gitlab/ci/jwt_v2_spec.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::JwtV2 do
+  let(:namespace) { build_stubbed(:namespace) }
+  let(:project) { build_stubbed(:project, namespace: namespace) }
+  let(:user) { build_stubbed(:user) }
+  let(:pipeline) { build_stubbed(:ci_pipeline, ref: 'auto-deploy-2020-03-19') }
+  let(:build) do
+    build_stubbed(
+      :ci_build,
+      project: project,
+      user: user,
+      pipeline: pipeline
+    )
+  end
+
+  subject(:ci_job_jwt_v2) { described_class.new(build, ttl: 30) }
+
+  it { is_expected.to be_a Gitlab::Ci::Jwt }
+
+  describe '#payload' do
+    subject(:payload) { ci_job_jwt_v2.payload }
+
+    it 'has correct values for the standard JWT attributes' do
+      aggregate_failures do
+        expect(payload[:iss]).to eq(Settings.gitlab.base_url)
+        expect(payload[:aud]).to eq(Settings.gitlab.base_url)
+        expect(payload[:sub]).to eq("project_path:#{project.full_path}:ref_type:branch:ref:#{pipeline.source_ref}")
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb
index 28bc685286f..0a592395c3a 100644
--- a/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/create_deployments_spec.rb
@@ -38,20 +38,6 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::CreateDeployments do
         expect(job.deployment.environment).to eq(job.persisted_environment)
       end
 
-      context 'when creation failure occures' do
-        before do
-          allow_next_instance_of(Deployment) do |deployment|
-            allow(deployment).to receive(:save!) { raise ActiveRecord::RecordInvalid }
-          end
-        end
-
-        it 'trackes the exception' do
-          expect { subject }.to raise_error(described_class::DeploymentCreationError)
-
-          expect(Deployment.count).to eq(0)
-        end
-      end
-
       context 'when the corresponding environment does not exist' do
         let!(:environment) { }
 
diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb
index 4206483b228..1d020d3ea79 100644
--- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb
@@ -7,7 +7,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Create do
   let_it_be(:user) { create(:user) }
 
   let(:pipeline) do
-    build(:ci_empty_pipeline, project: project, ref: 'master')
+    build(:ci_empty_pipeline, project: project, ref: 'master', user: user)
   end
 
   let(:command) do
@@ -59,7 +59,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Create do
 
   context 'tags persistence' do
     let(:stage) do
-      build(:ci_stage_entity, pipeline: pipeline)
+      build(:ci_stage_entity, pipeline: pipeline, project: project)
     end
 
     let(:job) do
@@ -79,12 +79,11 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Create do
       it 'extracts an empty tag list' do
         expect(CommitStatus)
           .to receive(:bulk_insert_tags!)
-          .with(stage.statuses, {})
+          .with([job])
           .and_call_original
 
         step.perform!
 
-        expect(job.instance_variable_defined?(:@tag_list)).to be_falsey
         expect(job).to be_persisted
         expect(job.tag_list).to eq([])
       end
@@ -98,14 +97,13 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Create do
       it 'bulk inserts tags' do
         expect(CommitStatus)
           .to receive(:bulk_insert_tags!)
-          .with(stage.statuses, { job.name => %w[tag1 tag2] })
+          .with([job])
           .and_call_original
 
         step.perform!
 
-        expect(job.instance_variable_defined?(:@tag_list)).to be_falsey
         expect(job).to be_persisted
-        expect(job.tag_list).to match_array(%w[tag1 tag2])
+        expect(job.reload.tag_list).to match_array(%w[tag1 tag2])
       end
     end
 
@@ -120,7 +118,6 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Create do
 
         step.perform!
 
-        expect(job.instance_variable_defined?(:@tag_list)).to be_truthy
         expect(job).to be_persisted
         expect(job.reload.tag_list).to match_array(%w[tag1 tag2])
       end
diff --git a/spec/lib/gitlab/ci/pipeline/logger_spec.rb b/spec/lib/gitlab/ci/pipeline/logger_spec.rb
index 0b44e35dec1..a488bc184f8 100644
--- a/spec/lib/gitlab/ci/pipeline/logger_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/logger_spec.rb
@@ -41,6 +41,90 @@ RSpec.describe ::Gitlab::Ci::Pipeline::Logger do
     end
   end
 
+  describe '#instrument_with_sql', :request_store do
+    subject(:instrument_with_sql) do
+      logger.instrument_with_sql(:expensive_operation, &operation)
+    end
+
+    def loggable_data(count:, db_count: nil)
+      keys = %w[
+        expensive_operation_duration_s
+        expensive_operation_db_count
+        expensive_operation_db_primary_count
+        expensive_operation_db_primary_duration_s
+        expensive_operation_db_main_count
+        expensive_operation_db_main_duration_s
+      ]
+
+      data = keys.each.with_object({}) do |key, accumulator|
+        accumulator[key] = {
+          'count' => count,
+          'avg' => a_kind_of(Numeric),
+          'max' => a_kind_of(Numeric),
+          'min' => a_kind_of(Numeric)
+        }
+      end
+
+      if db_count
+        data['expensive_operation_db_count']['max'] = db_count
+        data['expensive_operation_db_count']['min'] = db_count
+        data['expensive_operation_db_count']['avg'] = db_count
+      end
+
+      data
+    end
+
+    context 'with a single query' do
+      let(:operation) { -> { Project.count } }
+
+      it { is_expected.to eq(operation.call) }
+
+      it 'includes SQL metrics' do
+        instrument_with_sql
+
+        expect(logger.observations_hash)
+          .to match(a_hash_including(loggable_data(count: 1, db_count: 1)))
+      end
+    end
+
+    context 'with multiple queries' do
+      let(:operation) { -> { Ci::Build.count + Ci::Bridge.count } }
+
+      it { is_expected.to eq(operation.call) }
+
+      it 'includes SQL metrics' do
+        instrument_with_sql
+
+        expect(logger.observations_hash)
+          .to match(a_hash_including(loggable_data(count: 1, db_count: 2)))
+      end
+    end
+
+    context 'with multiple observations' do
+      let(:operation) { -> { Ci::Build.count + Ci::Bridge.count } }
+
+      it 'includes SQL metrics' do
+        2.times { logger.instrument_with_sql(:expensive_operation, &operation) }
+
+        expect(logger.observations_hash)
+          .to match(a_hash_including(loggable_data(count: 2, db_count: 2)))
+      end
+    end
+
+    context 'when there are not SQL operations' do
+      let(:operation) { -> { 123 } }
+
+      it { is_expected.to eq(operation.call) }
+
+      it 'does not include SQL metrics' do
+        instrument_with_sql
+
+        expect(logger.observations_hash.keys)
+          .to match_array(['expensive_operation_duration_s'])
+      end
+    end
+  end
+
   describe '#observe' do
     it 'records durations of observed operations' do
       loggable_data = {
diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
index 68806fbf287..2f9fcd7caac 100644
--- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb
@@ -8,7 +8,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do
 
   let(:pipeline) { build(:ci_empty_pipeline, project: project, sha: head_sha) }
   let(:root_variables) { [] }
-  let(:seed_context) { double(pipeline: pipeline, root_variables: root_variables) }
+  let(:seed_context) { Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables) }
   let(:attributes) { { name: 'rspec', ref: 'master', scheduling_type: :stage, when: 'on_success' } }
   let(:previous_stages) { [] }
   let(:current_stage) { double(seeds_names: [attributes[:name]]) }
diff --git a/spec/lib/gitlab/ci/pipeline/seed/pipeline_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/pipeline_spec.rb
index 5d8a9358e10..a76b4874eca 100644
--- a/spec/lib/gitlab/ci/pipeline/seed/pipeline_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/seed/pipeline_spec.rb
@@ -6,7 +6,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Pipeline do
   let_it_be(:project) { create(:project, :repository) }
   let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
 
-  let(:seed_context) { double(pipeline: pipeline, root_variables: []) }
+  let(:seed_context) { Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: []) }
 
   let(:stages_attributes) do
     [
diff --git a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb
index 5b04d2abd88..a632b5dedcf 100644
--- a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb
@@ -6,7 +6,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Stage do
   let(:project) { create(:project, :repository) }
   let(:pipeline) { create(:ci_empty_pipeline, project: project) }
   let(:previous_stages) { [] }
-  let(:seed_context) { double(pipeline: pipeline, root_variables: []) }
+  let(:seed_context) { Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: []) }
 
   let(:attributes) do
     { name: 'test',
diff --git a/spec/lib/gitlab/ci/status/build/waiting_for_approval_spec.rb b/spec/lib/gitlab/ci/status/build/waiting_for_approval_spec.rb
new file mode 100644
index 00000000000..b703a8a47ac
--- /dev/null
+++ b/spec/lib/gitlab/ci/status/build/waiting_for_approval_spec.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Status::Build::WaitingForApproval do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
+
+  subject { described_class.new(Gitlab::Ci::Status::Core.new(build, user)) }
+
+  describe '#illustration' do
+    let(:build) { create(:ci_build, :manual, environment: 'production', project: project) }
+
+    before do
+      environment = create(:environment, name: 'production', project: project)
+      create(:deployment, :blocked, project: project, environment: environment, deployable: build)
+    end
+
+    it { expect(subject.illustration).to include(:image, :size) }
+    it { expect(subject.illustration[:title]).to eq('Waiting for approval') }
+    it { expect(subject.illustration[:content]).to include('This job deploys to the protected environment "production"') }
+  end
+
+  describe '.matches?' do
+    subject { described_class.matches?(build, user) }
+
+    let(:build) { create(:ci_build, :manual, environment: 'production', project: project) }
+
+    before do
+      create(:deployment, deployment_status, deployable: build, project: project)
+    end
+
+    context 'when build is waiting for approval' do
+      let(:deployment_status) { :blocked }
+
+      it 'is a correct match' do
+        expect(subject).to be_truthy
+      end
+    end
+
+    context 'when build is not waiting for approval' do
+      let(:deployment_status) { :created }
+
+      it 'does not match' do
+        expect(subject).to be_falsey
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb b/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb
index 6c1f56de840..6c4f69fb036 100644
--- a/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb
+++ b/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb
@@ -5,27 +5,37 @@ require 'spec_helper'
 RSpec.describe Gitlab::Ci::Tags::BulkInsert do
   let_it_be(:project) { create(:project, :repository) }
   let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
-  let_it_be_with_refind(:job) { create(:ci_build, :unique_name, pipeline: pipeline, project: project) }
-  let_it_be_with_refind(:other_job) { create(:ci_build, :unique_name, pipeline: pipeline, project: project) }
-  let_it_be_with_refind(:bridge) { create(:ci_bridge, pipeline: pipeline, project: project) }
+  let_it_be_with_refind(:job) { create(:ci_build, :unique_name, pipeline: pipeline) }
+  let_it_be_with_refind(:other_job) { create(:ci_build, :unique_name, pipeline: pipeline) }
 
-  let(:statuses) { [job, bridge, other_job] }
+  let(:statuses) { [job, other_job] }
 
-  subject(:service) { described_class.new(statuses, tags_list) }
+  subject(:service) { described_class.new(statuses) }
+
+  describe 'gem version' do
+    let(:acceptable_version) { '9.0.0' }
+
+    let(:error_message) do
+      <<~MESSAGE
+      A mechanism depending on internals of 'act-as-taggable-on` has been designed
+      to bulk insert tags for Ci::Build records.
+      Please review the code carefully before updating the gem version
+      https://gitlab.com/gitlab-org/gitlab/-/issues/350053
+      MESSAGE
+    end
+
+    it { expect(ActsAsTaggableOn::VERSION).to eq(acceptable_version), error_message }
+  end
 
   describe '#insert!' do
     context 'without tags' do
-      let(:tags_list) { {} }
-
       it { expect(service.insert!).to be_falsey }
     end
 
     context 'with tags' do
-      let(:tags_list) do
-        {
-          job.name => %w[tag1 tag2],
-          other_job.name => %w[tag2 tag3 tag4]
-        }
+      before do
+        job.tag_list = %w[tag1 tag2]
+        other_job.tag_list = %w[tag2 tag3 tag4]
       end
 
       it 'persists tags' do
@@ -35,5 +45,18 @@ RSpec.describe Gitlab::Ci::Tags::BulkInsert do
         expect(other_job.reload.tag_list).to match_array(%w[tag2 tag3 tag4])
       end
     end
+
+    context 'with tags for only one job' do
+      before do
+        job.tag_list = %w[tag1 tag2]
+      end
+
+      it 'persists tags' do
+        expect(service.insert!).to be_truthy
+
+        expect(job.reload.tag_list).to match_array(%w[tag1 tag2])
+        expect(other_job.reload.tag_list).to be_empty
+      end
+    end
   end
 end
diff --git a/spec/lib/gitlab/ci/trace/remote_checksum_spec.rb b/spec/lib/gitlab/ci/trace/remote_checksum_spec.rb
index 8837ebc3652..1cd88034166 100644
--- a/spec/lib/gitlab/ci/trace/remote_checksum_spec.rb
+++ b/spec/lib/gitlab/ci/trace/remote_checksum_spec.rb
@@ -30,14 +30,6 @@ RSpec.describe Gitlab::Ci::Trace::RemoteChecksum do
       context 'with remote files' do
         let(:file_store) { JobArtifactUploader::Store::REMOTE }
 
-        context 'when the feature flag is disabled' do
-          before do
-            stub_feature_flags(ci_archived_build_trace_checksum: false)
-          end
-
-          it { is_expected.to be_nil }
-        end
-
         context 'with AWS as provider' do
           it { is_expected.to eq(checksum) }
         end
diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb
index 5ff34592b2f..8a87cbe45c1 100644
--- a/spec/lib/gitlab/ci/variables/builder_spec.rb
+++ b/spec/lib/gitlab/ci/variables/builder_spec.rb
@@ -3,25 +3,201 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Variables::Builder do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
+  let_it_be(:user) { project.owner }
+  let_it_be(:job) do
+    create(:ci_build,
+      pipeline: pipeline,
+      user: user,
+      yaml_variables: [{ key: 'YAML_VARIABLE', value: 'value' }]
+    )
+  end
+
   let(:builder) { described_class.new(pipeline) }
-  let(:pipeline) { create(:ci_pipeline) }
-  let(:job) { create(:ci_build, pipeline: pipeline) }
 
   describe '#scoped_variables' do
     let(:environment) { job.expanded_environment_name }
     let(:dependencies) { true }
+    let(:predefined_variables) do
+      [
+        { key: 'CI_JOB_NAME',
+          value: job.name },
+        { key: 'CI_JOB_STAGE',
+          value: job.stage },
+        { key: 'CI_NODE_TOTAL',
+          value: '1' },
+        { key: 'CI_BUILD_NAME',
+          value: job.name },
+        { key: 'CI_BUILD_STAGE',
+          value: job.stage },
+        { key: 'CI',
+          value: 'true' },
+        { key: 'GITLAB_CI',
+          value: 'true' },
+        { key: 'CI_SERVER_URL',
+          value: Gitlab.config.gitlab.url },
+        { key: 'CI_SERVER_HOST',
+          value: Gitlab.config.gitlab.host },
+        { key: 'CI_SERVER_PORT',
+          value: Gitlab.config.gitlab.port.to_s },
+        { key: 'CI_SERVER_PROTOCOL',
+          value: Gitlab.config.gitlab.protocol },
+        { key: 'CI_SERVER_NAME',
+          value: 'GitLab' },
+        { key: 'CI_SERVER_VERSION',
+          value: Gitlab::VERSION },
+        { key: 'CI_SERVER_VERSION_MAJOR',
+          value: Gitlab.version_info.major.to_s },
+        { key: 'CI_SERVER_VERSION_MINOR',
+          value: Gitlab.version_info.minor.to_s },
+        { key: 'CI_SERVER_VERSION_PATCH',
+          value: Gitlab.version_info.patch.to_s },
+        { key: 'CI_SERVER_REVISION',
+          value: Gitlab.revision },
+        { key: 'GITLAB_FEATURES',
+          value: project.licensed_features.join(',') },
+        { key: 'CI_PROJECT_ID',
+          value: project.id.to_s },
+        { key: 'CI_PROJECT_NAME',
+          value: project.path },
+        { key: 'CI_PROJECT_TITLE',
+          value: project.title },
+        { key: 'CI_PROJECT_PATH',
+          value: project.full_path },
+        { key: 'CI_PROJECT_PATH_SLUG',
+          value: project.full_path_slug },
+        { key: 'CI_PROJECT_NAMESPACE',
+          value: project.namespace.full_path },
+        { key: 'CI_PROJECT_ROOT_NAMESPACE',
+          value: project.namespace.root_ancestor.path },
+        { key: 'CI_PROJECT_URL',
+          value: project.web_url },
+        { key: 'CI_PROJECT_VISIBILITY',
+          value: "private" },
+        { key: 'CI_PROJECT_REPOSITORY_LANGUAGES',
+          value: project.repository_languages.map(&:name).join(',').downcase },
+        { key: 'CI_PROJECT_CLASSIFICATION_LABEL',
+          value: project.external_authorization_classification_label },
+        { key: 'CI_DEFAULT_BRANCH',
+          value: project.default_branch },
+        { key: 'CI_CONFIG_PATH',
+          value: project.ci_config_path_or_default },
+        { key: 'CI_PAGES_DOMAIN',
+          value: Gitlab.config.pages.host },
+        { key: 'CI_PAGES_URL',
+          value: project.pages_url },
+        { key: 'CI_API_V4_URL',
+          value: API::Helpers::Version.new('v4').root_url },
+        { key: 'CI_PIPELINE_IID',
+          value: pipeline.iid.to_s },
+        { key: 'CI_PIPELINE_SOURCE',
+          value: pipeline.source },
+        { key: 'CI_PIPELINE_CREATED_AT',
+          value: pipeline.created_at.iso8601 },
+        { key: 'CI_COMMIT_SHA',
+          value: job.sha },
+        { key: 'CI_COMMIT_SHORT_SHA',
+          value: job.short_sha },
+        { key: 'CI_COMMIT_BEFORE_SHA',
+          value: job.before_sha },
+        { key: 'CI_COMMIT_REF_NAME',
+          value: job.ref },
+        { key: 'CI_COMMIT_REF_SLUG',
+          value: job.ref_slug },
+        { key: 'CI_COMMIT_BRANCH',
+          value: job.ref },
+        { key: 'CI_COMMIT_MESSAGE',
+          value: pipeline.git_commit_message },
+        { key: 'CI_COMMIT_TITLE',
+          value: pipeline.git_commit_title },
+        { key: 'CI_COMMIT_DESCRIPTION',
+          value: pipeline.git_commit_description },
+        { key: 'CI_COMMIT_REF_PROTECTED',
+          value: (!!pipeline.protected_ref?).to_s },
+        { key: 'CI_COMMIT_TIMESTAMP',
+          value: pipeline.git_commit_timestamp },
+        { key: 'CI_COMMIT_AUTHOR',
+          value: pipeline.git_author_full_text },
+        { key: 'CI_BUILD_REF',
+          value: job.sha },
+        { key: 'CI_BUILD_BEFORE_SHA',
+          value: job.before_sha },
+        { key: 'CI_BUILD_REF_NAME',
+          value: job.ref },
+        { key: 'CI_BUILD_REF_SLUG',
+          value: job.ref_slug },
+        { key: 'YAML_VARIABLE',
+          value: 'value' },
+        { key: 'GITLAB_USER_ID',
+          value: user.id.to_s },
+        { key: 'GITLAB_USER_EMAIL',
+         value: user.email },
+        { key: 'GITLAB_USER_LOGIN',
+         value: user.username },
+        { key: 'GITLAB_USER_NAME',
+         value: user.name }
+      ].map { |var| var.merge(public: true, masked: false) }
+    end
 
     subject { builder.scoped_variables(job, environment: environment, dependencies: dependencies) }
 
-    it 'returns the expected variables' do
-      keys = %w[CI_JOB_NAME
-                CI_JOB_STAGE
-                CI_NODE_TOTAL
-                CI_BUILD_NAME
-                CI_BUILD_STAGE]
+    it { is_expected.to be_instance_of(Gitlab::Ci::Variables::Collection) }
+
+    it { expect(subject.to_runner_variables).to eq(predefined_variables) }
+
+    context 'variables ordering' do
+      def var(name, value)
+        { key: name, value: value.to_s, public: true, masked: false }
+      end
+
+      before do
+        allow(builder).to receive(:predefined_variables) { [var('A', 1), var('B', 1)] }
+        allow(project).to receive(:predefined_variables) { [var('B', 2), var('C', 2)] }
+        allow(pipeline).to receive(:predefined_variables) { [var('C', 3), var('D', 3)] }
+        allow(job).to receive(:runner) { double(predefined_variables: [var('D', 4), var('E', 4)]) }
+        allow(builder).to receive(:kubernetes_variables) { [var('E', 5), var('F', 5)] }
+        allow(builder).to receive(:deployment_variables) { [var('F', 6), var('G', 6)] }
+        allow(job).to receive(:yaml_variables) { [var('G', 7), var('H', 7)] }
+        allow(builder).to receive(:user_variables) { [var('H', 8), var('I', 8)] }
+        allow(job).to receive(:dependency_variables) { [var('I', 9), var('J', 9)] }
+        allow(builder).to receive(:secret_instance_variables) { [var('J', 10), var('K', 10)] }
+        allow(builder).to receive(:secret_group_variables) { [var('K', 11), var('L', 11)] }
+        allow(builder).to receive(:secret_project_variables) { [var('L', 12), var('M', 12)] }
+        allow(job).to receive(:trigger_request) { double(user_variables: [var('M', 13), var('N', 13)]) }
+        allow(pipeline).to receive(:variables) { [var('N', 14), var('O', 14)] }
+        allow(pipeline).to receive(:pipeline_schedule) { double(job_variables: [var('O', 15), var('P', 15)]) }
+      end
+
+      it 'returns variables in order depending on resource hierarchy' do
+        expect(subject.to_runner_variables).to eq(
+          [var('A', 1), var('B', 1),
+           var('B', 2), var('C', 2),
+           var('C', 3), var('D', 3),
+           var('D', 4), var('E', 4),
+           var('E', 5), var('F', 5),
+           var('F', 6), var('G', 6),
+           var('G', 7), var('H', 7),
+           var('H', 8), var('I', 8),
+           var('I', 9), var('J', 9),
+           var('J', 10), var('K', 10),
+           var('K', 11), var('L', 11),
+           var('L', 12), var('M', 12),
+           var('M', 13), var('N', 13),
+           var('N', 14), var('O', 14),
+           var('O', 15), var('P', 15)])
+      end
 
-      subject.map { |env| env[:key] }.tap do |names|
-        expect(names).to include(*keys)
+      it 'overrides duplicate keys depending on resource hierarchy' do
+        expect(subject.to_hash).to match(
+          'A' => '1', 'B' => '2',
+          'C' => '3', 'D' => '4',
+          'E' => '5', 'F' => '6',
+          'G' => '7', 'H' => '8',
+          'I' => '9', 'J' => '10',
+          'K' => '11', 'L' => '12',
+          'M' => '13', 'N' => '14',
+          'O' => '15', 'P' => '15')
       end
     end
   end
diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb
index e8b38b21ef8..20af84ce648 100644
--- a/spec/lib/gitlab/ci/yaml_processor_spec.rb
+++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb
@@ -2097,6 +2097,12 @@ module Gitlab
           it_behaves_like 'returns errors', 'test1 job: need deploy is not defined in current or prior stages'
         end
 
+        context 'duplicate needs' do
+          let(:needs) { %w(build1 build1) }
+
+          it_behaves_like 'returns errors', 'test1 has duplicate entries in the needs section.'
+        end
+
         context 'needs and dependencies that are mismatching' do
           let(:needs) { %w(build1) }
           let(:dependencies) { %w(build2) }
@@ -2602,7 +2608,7 @@ module Gitlab
         end
 
         context 'returns errors if job stage is not a defined stage' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", type: "acceptance" } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", type: "acceptance" } }) }
 
           it_behaves_like 'returns errors', 'rspec job: chosen stage does not exist; available stages are .pre, build, test, .post'
         end
@@ -2638,37 +2644,37 @@ module Gitlab
         end
 
         context 'returns errors if job artifacts:name is not an a string' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", artifacts: { name: 1 } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", artifacts: { name: 1 } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:artifacts name should be a string'
         end
 
         context 'returns errors if job artifacts:when is not an a predefined value' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", artifacts: { when: 1 } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", artifacts: { when: 1 } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:artifacts when should be on_success, on_failure or always'
         end
 
         context 'returns errors if job artifacts:expire_in is not an a string' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", artifacts: { expire_in: 1 } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", artifacts: { expire_in: 1 } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:artifacts expire in should be a duration'
         end
 
         context 'returns errors if job artifacts:expire_in is not an a valid duration' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", artifacts: { expire_in: "7 elephants" } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", artifacts: { expire_in: "7 elephants" } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:artifacts expire in should be a duration'
         end
 
         context 'returns errors if job artifacts:untracked is not an array of strings' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", artifacts: { untracked: "string" } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", artifacts: { untracked: "string" } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:artifacts untracked should be a boolean value'
         end
 
         context 'returns errors if job artifacts:paths is not an array of strings' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", artifacts: { paths: "string" } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", artifacts: { paths: "string" } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:artifacts paths should be an array of strings'
         end
@@ -2692,49 +2698,49 @@ module Gitlab
         end
 
         context 'returns errors if job cache:key is not an a string' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: 1 } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", cache: { key: 1 } } }) }
 
           it_behaves_like 'returns errors', "jobs:rspec:cache:key should be a hash, a string or a symbol"
         end
 
         context 'returns errors if job cache:key:files is not an array of strings' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: { files: [1] } } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", cache: { key: { files: [1] } } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:cache:key:files config should be an array of strings'
         end
 
         context 'returns errors if job cache:key:files is an empty array' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: { files: [] } } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", cache: { key: { files: [] } } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:cache:key:files config requires at least 1 item'
         end
 
         context 'returns errors if job defines only cache:key:prefix' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: { prefix: 'prefix-key' } } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", cache: { key: { prefix: 'prefix-key' } } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:cache:key config missing required keys: files'
         end
 
         context 'returns errors if job cache:key:prefix is not an a string' do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: { prefix: 1, files: ['file'] } } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", cache: { key: { prefix: 1, files: ['file'] } } } }) }
 
           it_behaves_like 'returns errors', 'jobs:rspec:cache:key:prefix config should be a string or symbol'
         end
 
         context "returns errors if job cache:untracked is not an array of strings" do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { untracked: "string" } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", cache: { untracked: "string" } } }) }
 
           it_behaves_like 'returns errors', "jobs:rspec:cache:untracked config should be a boolean value"
         end
 
         context "returns errors if job cache:paths is not an array of strings" do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { paths: "string" } } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", cache: { paths: "string" } } }) }
 
           it_behaves_like 'returns errors', "jobs:rspec:cache:paths config should be an array of strings"
         end
 
         context "returns errors if job dependencies is not an array of strings" do
-          let(:config) { YAML.dump({ types: %w(build test), rspec: { script: "test", dependencies: "string" } }) }
+          let(:config) { YAML.dump({ stages: %w(build test), rspec: { script: "test", dependencies: "string" } }) }
 
           it_behaves_like 'returns errors', "jobs:rspec dependencies should be an array of strings"
         end