Merge branch 'master' into backstage/gb/rename-ci-cd-processing-sidekiq-queues

* master: (115 commits)
  Use event-based waiting in Gitlab::JobWaiter
  Make sure repository's removal work for legacy and hashed storages
  Use `@hashed` prefix for hashed paths on disk, to avoid collision with existing ones
  Refactor project and storage types
  Prevent using gitlab import task when hashed storage is enabled
  Some codestyle changes and fixes for GitLab pages
  Removed some useless code, codestyle changes and removed an index
  Fix repository reloading in some specs
  Changelog
  Moving away from the "extend" based factory to a more traditional one.
  Enable automatic hashed storage for new projects by application settings
  New storage is now "Hashed" instead of "UUID"
  Add UUID Storage to Project
  Move create_repository back to project model as we can use disk_path and share it
  Codestyle: move hooks to the same place and move dependent methods to private
  Use non-i18n values for setting new group-level issue/MR button text
  indexes external issue tracker
  copyedit
  indexes user/search/ from /user/index
  Correctly encode string params for Gitaly's TreeEntries RPC
  ...

6 files changed, 154 insertions, 50 deletions

diff --git a/spec/lib/gitlab/background_migration/migrate_stage_status_spec.rb b/spec/lib/gitlab/background_migration/migrate_stage_status_spec.rb
new file mode 100644
index 00000000000..878158910be
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_stage_status_spec.rb
@@ -0,0 +1,80 @@
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateStageStatus, :migration, schema: 20170711145320 do
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:stages) { table(:ci_stages) }
+  let(:jobs) { table(:ci_builds) }
+
+  STATUSES = { created: 0, pending: 1, running: 2, success: 3,
+               failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+
+  before do
+    projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1')
+    pipelines.create!(id: 1, project_id: 1, ref: 'master', sha: 'adf43c3a')
+    stages.create!(id: 1, pipeline_id: 1, project_id: 1, name: 'test', status: nil)
+    stages.create!(id: 2, pipeline_id: 1, project_id: 1, name: 'deploy', status: nil)
+  end
+
+  context 'when stage status is known' do
+    before do
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'success')
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'running')
+      create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'failed')
+    end
+
+    it 'sets a correct stage status' do
+      described_class.new.perform(1, 2)
+
+      expect(stages.first.status).to eq STATUSES[:running]
+      expect(stages.second.status).to eq STATUSES[:failed]
+    end
+  end
+
+  context 'when stage status is not known' do
+    it 'sets a skipped stage status' do
+      described_class.new.perform(1, 2)
+
+      expect(stages.first.status).to eq STATUSES[:skipped]
+      expect(stages.second.status).to eq STATUSES[:skipped]
+    end
+  end
+
+  context 'when stage status includes status of a retried job' do
+    before do
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'canceled')
+      create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'failed', retried: true)
+      create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'success')
+    end
+
+    it 'sets a correct stage status' do
+      described_class.new.perform(1, 2)
+
+      expect(stages.first.status).to eq STATUSES[:canceled]
+      expect(stages.second.status).to eq STATUSES[:success]
+    end
+  end
+
+  context 'when some job in the stage is blocked / manual' do
+    before do
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'failed')
+      create_job(project: 1, pipeline: 1, stage: 'test', status: 'manual')
+      create_job(project: 1, pipeline: 1, stage: 'deploy', status: 'success', when: 'manual')
+    end
+
+    it 'sets a correct stage status' do
+      described_class.new.perform(1, 2)
+
+      expect(stages.first.status).to eq STATUSES[:manual]
+      expect(stages.second.status).to eq STATUSES[:success]
+    end
+  end
+
+  def create_job(project:, pipeline:, stage:, status:, **opts)
+    stages = { test: 1, build: 2, deploy: 3 }
+
+    jobs.create!(project_id: project, commit_id: pipeline,
+                 stage_idx: stages[stage.to_sym], stage: stage,
+                 status: status, **opts)
+  end
+end
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index a3ae0a4686d..8ec8dfe6acf 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -235,18 +235,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
     it { is_expected.to be < 2 }
   end
 
-  describe '#has_commits?' do
-    it { expect(repository.has_commits?).to be_truthy }
-  end
-
   describe '#empty?' do
     it { expect(repository.empty?).to be_falsey }
   end
 
-  describe '#bare?' do
-    it { expect(repository.bare?).to be_truthy }
-  end
-
   describe '#ref_names' do
     let(:ref_names) { repository.ref_names }
     subject { ref_names }
@@ -441,15 +433,6 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe "#remote_names" do
-    let(:remotes) { repository.remote_names }
-
-    it "should have one entry: 'origin'" do
-      expect(remotes.size).to eq(1)
-      expect(remotes.first).to eq("origin")
-    end
-  end
-
   describe "#refs_hash" do
     let(:refs) { repository.refs_hash }
 
diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb
index 7fe698fcb18..2eaf4222964 100644
--- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb
+++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb
@@ -111,6 +111,20 @@ describe Gitlab::GitalyClient::CommitService do
 
       client.tree_entries(repository, revision, path)
     end
+
+    context 'with UTF-8 params strings' do
+      let(:revision) { "branch\u011F" }
+      let(:path) { "foo/\u011F.txt" }
+
+      it 'handles string encodings correctly' do
+        expect_any_instance_of(Gitaly::CommitService::Stub)
+          .to receive(:get_tree_entries)
+          .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash))
+          .and_return([])
+
+        client.tree_entries(repository, revision, path)
+      end
+    end
   end
 
   describe '#find_commit' do
diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml
index ae3b0173160..a5e03e149a7 100644
--- a/spec/lib/gitlab/import_export/safe_model_attributes.yml
+++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml
@@ -227,6 +227,8 @@ Ci::Pipeline:
 Ci::Stage:
 - id
 - name
+- status
+- lock_version
 - project_id
 - pipeline_id
 - created_at
diff --git a/spec/lib/gitlab/job_waiter_spec.rb b/spec/lib/gitlab/job_waiter_spec.rb
index 6186cec2689..b0b4fdc09bc 100644
--- a/spec/lib/gitlab/job_waiter_spec.rb
+++ b/spec/lib/gitlab/job_waiter_spec.rb
@@ -1,30 +1,39 @@
 require 'spec_helper'
 
 describe Gitlab::JobWaiter do
-  describe '#wait' do
-    let(:waiter) { described_class.new(%w(a)) }
-    it 'returns when all jobs have been completed' do
-      expect(Gitlab::SidekiqStatus).to receive(:all_completed?).with(%w(a))
-        .and_return(true)
+  describe '.notify' do
+    it 'pushes the jid to the named queue' do
+      key = 'gitlab:job_waiter:foo'
+      jid = 1
 
-      expect(waiter).not_to receive(:sleep)
+      redis = double('redis')
+      expect(Gitlab::Redis::SharedState).to receive(:with).and_yield(redis)
+      expect(redis).to receive(:lpush).with(key, jid)
 
-      waiter.wait
+      described_class.notify(key, jid)
     end
+  end
+
+  describe '#wait' do
+    let(:waiter) { described_class.new(2) }
 
-    it 'sleeps between checking the job statuses' do
-      expect(Gitlab::SidekiqStatus).to receive(:all_completed?)
-        .with(%w(a))
-        .and_return(false, true)
+    it 'returns when all jobs have been completed' do
+      described_class.notify(waiter.key, 'a')
+      described_class.notify(waiter.key, 'b')
 
-      expect(waiter).to receive(:sleep).with(described_class::INTERVAL)
+      result = nil
+      expect { Timeout.timeout(1) { result = waiter.wait(2) } }.not_to raise_error
 
-      waiter.wait
+      expect(result).to contain_exactly('a', 'b')
     end
 
-    it 'returns when timing out' do
-      expect(waiter).not_to receive(:sleep)
-      waiter.wait(0)
+    it 'times out if not all jobs complete' do
+      described_class.notify(waiter.key, 'a')
+
+      result = nil
+      expect { Timeout.timeout(2) { result = waiter.wait(1) } }.not_to raise_error
+
+      expect(result).to contain_exactly('a')
     end
   end
 end
diff --git a/spec/lib/gitlab/sidekiq_throttler_spec.rb b/spec/lib/gitlab/sidekiq_throttler_spec.rb
index 6374ac80207..2dbb7bb7c34 100644
--- a/spec/lib/gitlab/sidekiq_throttler_spec.rb
+++ b/spec/lib/gitlab/sidekiq_throttler_spec.rb
@@ -1,28 +1,44 @@
 require 'spec_helper'
 
 describe Gitlab::SidekiqThrottler do
-  before do
-    Sidekiq.options[:concurrency] = 35
-
-    stub_application_setting(
-      sidekiq_throttling_enabled: true,
-      sidekiq_throttling_factor: 0.1,
-      sidekiq_throttling_queues: %w[build project_cache]
-    )
-  end
-
   describe '#execute!' do
-    it 'sets limits on the selected queues' do
-      described_class.execute!
+    context 'when job throttling is enabled' do
+      before do
+        Sidekiq.options[:concurrency] = 35
+
+        stub_application_setting(
+          sidekiq_throttling_enabled: true,
+          sidekiq_throttling_factor: 0.1,
+          sidekiq_throttling_queues: %w[build project_cache]
+        )
+      end
+
+      it 'requires sidekiq-limit_fetch' do
+        expect(described_class).to receive(:require).with('sidekiq-limit_fetch').and_call_original
+
+        described_class.execute!
+      end
+
+      it 'sets limits on the selected queues' do
+        described_class.execute!
+
+        expect(Sidekiq::Queue['build'].limit).to eq 4
+        expect(Sidekiq::Queue['project_cache'].limit).to eq 4
+      end
+
+      it 'does not set limits on other queues' do
+        described_class.execute!
 
-      expect(Sidekiq::Queue['build'].limit).to eq 4
-      expect(Sidekiq::Queue['project_cache'].limit).to eq 4
+        expect(Sidekiq::Queue['merge'].limit).to be_nil
+      end
     end
 
-    it 'does not set limits on other queues' do
-      described_class.execute!
+    context 'when job throttling is disabled' do
+      it 'does not require sidekiq-limit_fetch' do
+        expect(described_class).not_to receive(:require).with('sidekiq-limit_fetch')
 
-      expect(Sidekiq::Queue['merge'].limit).to be_nil
+        described_class.execute!
+      end
     end
   end
 end