Add latest changes from gitlab-org/gitlab@master

15 files changed, 319 insertions, 15 deletions

diff --git a/spec/factories/ci/build_need.rb b/spec/factories/ci/build_need.rb
index fa72e696343..aa571ffabb7 100644
--- a/spec/factories/ci/build_need.rb
+++ b/spec/factories/ci/build_need.rb
@@ -2,7 +2,7 @@
 
 FactoryBot.define do
   factory :ci_build_need, class: 'Ci::BuildNeed' do
-    build factory: :ci_build
+    build factory: :ci_build, scheduling_type: :dag
     sequence(:name) { |n| "build_#{n}" }
   end
 end
diff --git a/spec/fixtures/lib/gitlab/import_export/group_exports/complex/group.json b/spec/fixtures/lib/gitlab/import_export/group_exports/complex/group.json
index 7cd91ff225b..504daae8abd 100644
--- a/spec/fixtures/lib/gitlab/import_export/group_exports/complex/group.json
+++ b/spec/fixtures/lib/gitlab/import_export/group_exports/complex/group.json
@@ -1,7 +1,7 @@
 {
   "name": "ymg09t5704clnxnqfgaj2h098gz4r7gyx4wc3fzmlqj1en24zf",
   "path": "ymg09t5704clnxnqfgaj2h098gz4r7gyx4wc3fzmlqj1en24zf",
-  "owner_id": null,
+  "owner_id": 123,
   "created_at": "2019-11-20 17:01:53 UTC",
   "updated_at": "2019-11-20 17:05:44 UTC",
   "description": "Group Description",
@@ -18,7 +18,7 @@
   "ldap_sync_last_successful_update_at": null,
   "ldap_sync_last_sync_at": null,
   "lfs_enabled": null,
-  "parent_id": null,
+  "parent_id": 7,
   "shared_runners_minutes_limit": null,
   "repository_size_limit": null,
   "require_two_factor_authentication": false,
@@ -33,6 +33,8 @@
   "extra_shared_runners_minutes_limit": null,
   "last_ci_minutes_notification_at": null,
   "last_ci_minutes_usage_notification_level": null,
+  "runners_token": "token",
+  "runners_token_encrypted": "encrypted",
   "subgroup_creation_level": 1,
   "emails_disabled": null,
   "max_pages_size": null,
diff --git a/spec/lib/gitlab/import_export/group_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group_tree_restorer_spec.rb
index 4d52f1fdda3..9aa9471d155 100644
--- a/spec/lib/gitlab/import_export/group_tree_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/group_tree_restorer_spec.rb
@@ -71,6 +71,44 @@ describe Gitlab::ImportExport::GroupTreeRestorer do
     end
   end
 
+  context 'excluded attributes' do
+    let!(:source_user) { create(:user, id: 123) }
+    let!(:importer_user) { create(:user) }
+    let(:group) { create(:group) }
+    let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+    let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group, group_hash: nil) }
+    let(:group_json) { ActiveSupport::JSON.decode(IO.read(File.join(shared.export_path, 'group.json'))) }
+
+    shared_examples 'excluded attributes' do
+      excluded_attributes = %w[
+        id
+        name
+        path
+        owner_id
+        parent_id
+        created_at
+        updated_at
+        runners_token
+        runners_token_encrypted
+        saml_discovery_token
+      ]
+
+      before do
+        group.add_owner(importer_user)
+
+        setup_import_export_config('group_exports/complex')
+      end
+
+      excluded_attributes.each do |excluded_attribute|
+        it 'does not allow override of excluded attributes' do
+          expect(group_json[excluded_attribute]).not_to eq(group.public_send(excluded_attribute))
+        end
+      end
+    end
+
+    include_examples 'excluded attributes'
+  end
+
   context 'group.json file access check' do
     let(:user) { create(:user) }
     let!(:group) { create(:group, name: 'group2', path: 'group2') }
diff --git a/spec/lib/gitlab/import_export/group_tree_saver_spec.rb b/spec/lib/gitlab/import_export/group_tree_saver_spec.rb
index 17538db3c68..d6aea8d1e69 100644
--- a/spec/lib/gitlab/import_export/group_tree_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/group_tree_saver_spec.rb
@@ -157,9 +157,28 @@ describe Gitlab::ImportExport::GroupTreeSaver do
       end
 
       context 'group attributes' do
-        it 'does not contain the runners token' do
-          expect(saved_group_json).not_to include("runners_token" => 'token')
+        shared_examples 'excluded attributes' do
+          excluded_attributes = %w[
+            id
+            name
+            path
+            owner_id
+            parent_id
+            created_at
+            updated_at
+            runners_token
+            runners_token_encrypted
+            saml_discovery_token
+          ]
+
+          excluded_attributes.each do |excluded_attribute|
+            it 'does not contain excluded attribute' do
+              expect(saved_group_json).not_to include(excluded_attribute => group.public_send(excluded_attribute))
+            end
+          end
         end
+
+        include_examples 'excluded attributes'
       end
     end
   end
diff --git a/spec/migrations/remove_packages_deprecated_dependencies_spec.rb b/spec/migrations/remove_packages_deprecated_dependencies_spec.rb
new file mode 100644
index 00000000000..0b7efe371a6
--- /dev/null
+++ b/spec/migrations/remove_packages_deprecated_dependencies_spec.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20200210135504_remove_packages_deprecated_dependencies.rb')
+
+describe RemovePackagesDeprecatedDependencies, :migration do
+  let(:projects) { table(:projects) }
+  let(:packages) { table(:packages_packages) }
+  let(:dependency_links) { table(:packages_dependency_links) }
+  let(:dependencies) { table(:packages_dependencies) }
+
+  before do
+    projects.create!(id: 123, name: 'gitlab', path: 'gitlab-org/gitlab-ce', namespace_id: 1)
+    packages.create!(id: 1, name: 'package', version: '1.0.0', package_type: 4, project_id: 123)
+    5.times do |i|
+      dependencies.create!(id: i, name: "pkg_dependency_#{i}", version_pattern: '~1.0.0')
+      dependency_links.create!(package_id: 1, dependency_id: i, dependency_type: 5)
+    end
+    dependencies.create!(id: 10, name: 'valid_pkg_dependency', version_pattern: '~2.5.0')
+    dependency_links.create!(package_id: 1, dependency_id: 10, dependency_type: 1)
+  end
+
+  it 'removes all dependency links with type 5' do
+    expect(dependency_links.count).to eq 6
+
+    migrate!
+
+    expect(dependency_links.count).to eq 1
+  end
+end
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 65fd9b049d6..4bfb5771bb8 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -762,8 +762,10 @@ describe Ci::Build do
       let(:needs) { }
 
       let!(:final) do
+        scheduling_type = needs.present? ? :dag : :stage
+
         create(:ci_build,
-          pipeline: pipeline, name: 'final',
+          pipeline: pipeline, name: 'final', scheduling_type: scheduling_type,
           stage_idx: 3, stage: 'deploy', options: {
             dependencies: dependencies
           }
diff --git a/spec/models/ci/processable_spec.rb b/spec/models/ci/processable_spec.rb
index f3d743fc272..1e0544c14c5 100644
--- a/spec/models/ci/processable_spec.rb
+++ b/spec/models/ci/processable_spec.rb
@@ -120,4 +120,29 @@ describe Ci::Processable do
       end
     end
   end
+
+  describe '.populate_scheduling_type!' do
+    let!(:build_without_needs) { create(:ci_build, project: project, pipeline: pipeline) }
+    let!(:build_with_needs) { create(:ci_build, project: project, pipeline: pipeline) }
+    let!(:needs_relation) { create(:ci_build_need, build: build_with_needs) }
+    let!(:another_build) { create(:ci_build, project: project) }
+
+    before do
+      Ci::Processable.update_all(scheduling_type: nil)
+    end
+
+    it 'populates scheduling_type of processables' do
+      expect do
+        pipeline.processables.populate_scheduling_type!
+      end.to change(pipeline.processables.where(scheduling_type: nil), :count).from(2).to(0)
+
+      expect(build_without_needs.reload.scheduling_type).to eq('stage')
+      expect(build_with_needs.reload.scheduling_type).to eq('dag')
+    end
+
+    it 'does not affect processables from other pipelines' do
+      pipeline.processables.populate_scheduling_type!
+      expect(another_build.reload.scheduling_type).to be_nil
+    end
+  end
 end
diff --git a/spec/models/project_services/youtrack_service_spec.rb b/spec/models/project_services/youtrack_service_spec.rb
index dcc40d8f343..0067793f8d8 100644
--- a/spec/models/project_services/youtrack_service_spec.rb
+++ b/spec/models/project_services/youtrack_service_spec.rb
@@ -37,6 +37,10 @@ describe YoutrackService do
     it 'does allow project prefix on the reference' do
       expect(described_class.reference_pattern.match('YT-123')[:issue]).to eq('YT-123')
     end
+
+    it 'does not allow issue number to be followed by a letter' do
+      expect(described_class.reference_pattern.match('YT-123A')).to eq(nil)
+    end
   end
 
   context 'overriding properties' do
diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb
index d14baa49341..12ac601c013 100644
--- a/spec/requests/api/users_spec.rb
+++ b/spec/requests/api/users_spec.rb
@@ -949,6 +949,45 @@ describe API::Users do
     end
   end
 
+  describe "DELETE /users/:id/identities/:provider" do
+    let(:test_user) { create(:omniauth_user, provider: 'ldapmain') }
+
+    context 'when unauthenticated' do
+      it 'returns authentication error' do
+        delete api("/users/#{test_user.id}/identities/ldapmain")
+
+        expect(response).to have_gitlab_http_status(:unauthorized)
+      end
+    end
+
+    context 'when authenticated' do
+      it 'deletes identity of given provider' do
+        expect do
+          delete api("/users/#{test_user.id}/identities/ldapmain", admin)
+        end.to change { test_user.identities.count }.by(-1)
+        expect(response).to have_gitlab_http_status(:no_content)
+      end
+
+      it_behaves_like '412 response' do
+        let(:request) { api("/users/#{test_user.id}/identities/ldapmain", admin) }
+      end
+
+      it 'returns 404 error if user not found' do
+        delete api("/users/0/identities/ldapmain", admin)
+
+        expect(response).to have_gitlab_http_status(:not_found)
+        expect(json_response['message']).to eq('404 User Not Found')
+      end
+
+      it 'returns 404 error if identity not found' do
+        delete api("/users/#{test_user.id}/identities/saml", admin)
+
+        expect(response).to have_gitlab_http_status(:not_found)
+        expect(json_response['message']).to eq('404 Identity Not Found')
+      end
+    end
+  end
+
   describe "POST /users/:id/keys" do
     before do
       admin
diff --git a/spec/services/ci/create_pipeline_service/needs_spec.rb b/spec/services/ci/create_pipeline_service/needs_spec.rb
index 757c076f9e6..17b9cf80cc1 100644
--- a/spec/services/ci/create_pipeline_service/needs_spec.rb
+++ b/spec/services/ci/create_pipeline_service/needs_spec.rb
@@ -175,5 +175,67 @@ describe Ci::CreatePipelineService do
           .to eq('jobs:test_a:needs:need artifacts should be a boolean value')
       end
     end
+
+    context 'when needs is empty array' do
+      let(:config) do
+        <<~YAML
+          build_a:
+            stage: build
+            script: ls
+          test_a:
+            stage: test
+            script: ls
+          test_b:
+            stage: test
+            script: ls
+            needs: []
+          deploy_a:
+            stage: deploy
+            script: ls
+            needs: [test_a]
+          deploy_b:
+            stage: deploy
+            script: ls
+            when: manual
+            needs: []
+        YAML
+      end
+
+      it 'creates a pipeline with build_a and test_b pending; deploy_b manual' do
+        processables = pipeline.processables
+
+        build_a = processables.find { |processable| processable.name == 'build_a' }
+        test_a = processables.find { |processable| processable.name == 'test_a' }
+        test_b = processables.find { |processable| processable.name == 'test_b' }
+        deploy_a = processables.find { |processable| processable.name == 'deploy_a' }
+        deploy_b = processables.find { |processable| processable.name == 'deploy_b' }
+
+        expect(pipeline).to be_persisted
+        expect(build_a.status).to eq('pending')
+        expect(test_a.status).to eq('created')
+        expect(test_b.status).to eq('pending')
+        expect(deploy_a.status).to eq('created')
+        expect(deploy_b.status).to eq('manual')
+      end
+    end
+
+    context 'when needs is empty hash' do
+      let(:config) do
+        <<~YAML
+          regular_job:
+            stage: build
+            script: echo 'hello'
+          invalid_dag_job:
+            stage: test
+            script: ls
+            needs: {}
+        YAML
+      end
+
+      it 'raises error' do
+        expect(pipeline.yaml_errors)
+          .to eq('jobs:invalid_dag_job:needs config can not be an empty hash')
+      end
+    end
   end
 end
diff --git a/spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb b/spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb
index 38686b41a22..cbeb45b92ff 100644
--- a/spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb
+++ b/spec/services/ci/pipeline_processing/atomic_processing_service_spec.rb
@@ -9,4 +9,10 @@ describe Ci::PipelineProcessing::AtomicProcessingService do
   end
 
   it_behaves_like 'Pipeline Processing Service'
+
+  private
+
+  def process_pipeline(initial_process: false)
+    described_class.new(pipeline).execute
+  end
 end
diff --git a/spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb b/spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb
index 2da1eb19818..09b462b7600 100644
--- a/spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb
+++ b/spec/services/ci/pipeline_processing/legacy_processing_service_spec.rb
@@ -9,4 +9,10 @@ describe Ci::PipelineProcessing::LegacyProcessingService do
   end
 
   it_behaves_like 'Pipeline Processing Service'
+
+  private
+
+  def process_pipeline(initial_process: false)
+    described_class.new(pipeline).execute(initial_process: initial_process)
+  end
 end
diff --git a/spec/services/ci/pipeline_processing/shared_processing_service.rb b/spec/services/ci/pipeline_processing/shared_processing_service.rb
index cae5ae3f09d..2349dedf370 100644
--- a/spec/services/ci/pipeline_processing/shared_processing_service.rb
+++ b/spec/services/ci/pipeline_processing/shared_processing_service.rb
@@ -710,10 +710,10 @@ shared_examples 'Pipeline Processing Service' do
   context 'when pipeline with needs is created', :sidekiq_inline do
     let!(:linux_build) { create_build('linux:build', stage: 'build', stage_idx: 0) }
     let!(:mac_build) { create_build('mac:build', stage: 'build', stage_idx: 0) }
-    let!(:linux_rspec) { create_build('linux:rspec', stage: 'test', stage_idx: 1) }
-    let!(:linux_rubocop) { create_build('linux:rubocop', stage: 'test', stage_idx: 1) }
-    let!(:mac_rspec) { create_build('mac:rspec', stage: 'test', stage_idx: 1) }
-    let!(:mac_rubocop) { create_build('mac:rubocop', stage: 'test', stage_idx: 1) }
+    let!(:linux_rspec) { create_build('linux:rspec', stage: 'test', stage_idx: 1, scheduling_type: :dag) }
+    let!(:linux_rubocop) { create_build('linux:rubocop', stage: 'test', stage_idx: 1, scheduling_type: :dag) }
+    let!(:mac_rspec) { create_build('mac:rspec', stage: 'test', stage_idx: 1, scheduling_type: :dag) }
+    let!(:mac_rubocop) { create_build('mac:rubocop', stage: 'test', stage_idx: 1, scheduling_type: :dag) }
     let!(:deploy) { create_build('deploy', stage: 'deploy', stage_idx: 2) }
 
     let!(:linux_rspec_on_build) { create(:ci_build_need, build: linux_rspec, name: 'linux:build') }
@@ -795,7 +795,7 @@ shared_examples 'Pipeline Processing Service' do
     end
 
     context 'when one of the jobs is run on a failure' do
-      let!(:linux_notify) { create_build('linux:notify', stage: 'deploy', stage_idx: 2, when: 'on_failure') }
+      let!(:linux_notify) { create_build('linux:notify', stage: 'deploy', stage_idx: 2, when: 'on_failure', scheduling_type: :dag) }
 
       let!(:linux_notify_on_build) { create(:ci_build_need, build: linux_notify, name: 'linux:build') }
 
@@ -837,10 +837,47 @@ shared_examples 'Pipeline Processing Service' do
         end
       end
     end
-  end
 
-  def process_pipeline
-    described_class.new(pipeline).execute
+    context 'when there is a job scheduled with dag but no need (needs: [])' do
+      let!(:deploy_pages) { create_build('deploy_pages', stage: 'deploy', stage_idx: 2, scheduling_type: :dag) }
+
+      it 'runs deploy_pages without waiting prior stages' do
+        # Ci::PipelineProcessing::LegacyProcessingService requires :initial_process parameter
+        expect(process_pipeline(initial_process: true)).to be_truthy
+
+        expect(stages).to eq(%w(pending created pending))
+        expect(builds.pending).to contain_exactly(linux_build, mac_build, deploy_pages)
+
+        linux_build.reset.success!
+        deploy_pages.reset.success!
+
+        expect(stages).to eq(%w(running pending running))
+        expect(builds.success).to contain_exactly(linux_build, deploy_pages)
+        expect(builds.pending).to contain_exactly(mac_build, linux_rspec, linux_rubocop)
+
+        linux_rspec.reset.success!
+        linux_rubocop.reset.success!
+        mac_build.reset.success!
+        mac_rspec.reset.success!
+        mac_rubocop.reset.success!
+
+        expect(stages).to eq(%w(success success running))
+        expect(builds.pending).to contain_exactly(deploy)
+      end
+
+      context 'when ci_dag_support is disabled' do
+        before do
+          stub_feature_flags(ci_dag_support: false)
+        end
+
+        it 'does run deploy_pages at the start' do
+          expect(process_pipeline).to be_truthy
+
+          expect(stages).to eq(%w(pending created created))
+          expect(builds.pending).to contain_exactly(linux_build, mac_build)
+        end
+      end
+    end
   end
 
   def all_builds
diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index 40ae1c4029b..6f5a070d73d 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -33,6 +33,25 @@ describe Ci::ProcessPipelineService do
     end
   end
 
+  context 'with a pipeline which has processables with nil scheduling_type', :clean_gitlab_redis_shared_state do
+    let!(:build1) { create_build('build1') }
+    let!(:build2) { create_build('build2') }
+    let!(:build3) { create_build('build3', scheduling_type: :dag) }
+    let!(:build3_on_build2) { create(:ci_build_need, build: build3, name: 'build2') }
+
+    before do
+      pipeline.processables.update_all(scheduling_type: nil)
+    end
+
+    it 'populates scheduling_type before processing' do
+      process_pipeline
+
+      expect(build1.scheduling_type).to eq('stage')
+      expect(build2.scheduling_type).to eq('stage')
+      expect(build3.scheduling_type).to eq('dag')
+    end
+  end
+
   def process_pipeline
     described_class.new(pipeline).execute
   end
diff --git a/spec/services/ci/retry_pipeline_service_spec.rb b/spec/services/ci/retry_pipeline_service_spec.rb
index e7a241ed335..7db871adc9a 100644
--- a/spec/services/ci/retry_pipeline_service_spec.rb
+++ b/spec/services/ci/retry_pipeline_service_spec.rb
@@ -95,7 +95,7 @@ describe Ci::RetryPipelineService, '#execute' do
       before do
         create_build('build', :success, 0)
         create_build('build2', :success, 0)
-        test_build = create_build('test', :failed, 1)
+        test_build = create_build('test', :failed, 1, scheduling_type: :dag)
         create(:ci_build_need, build: test_build, name: 'build')
         create(:ci_build_need, build: test_build, name: 'build2')
       end
@@ -108,6 +108,21 @@ describe Ci::RetryPipelineService, '#execute' do
         expect(build('test')).to be_pending
         expect(build('test').needs.map(&:name)).to match_array(%w(build build2))
       end
+
+      context 'when there is a failed DAG test without needs' do
+        before do
+          create_build('deploy', :failed, 2, scheduling_type: :dag)
+        end
+
+        it 'retries the test' do
+          service.execute(pipeline)
+
+          expect(build('build')).to be_success
+          expect(build('build2')).to be_success
+          expect(build('test')).to be_pending
+          expect(build('deploy')).to be_pending
+        end
+      end
     end
 
     context 'when the last stage was skipepd' do