Merge branch 'master' into '46448-add-timestamps-for-each-stage-of-gitlab-rake-gitlab-backup-restore'

# Conflicts:
#   spec/tasks/gitlab/backup_rake_spec.rb

12 files changed, 519 insertions, 177 deletions

diff --git a/spec/tasks/cache/clear/redis_spec.rb b/spec/tasks/cache/clear/redis_spec.rb
index cca2b864e9b..97c8c943f3a 100644
--- a/spec/tasks/cache/clear/redis_spec.rb
+++ b/spec/tasks/cache/clear/redis_spec.rb
@@ -6,7 +6,10 @@ describe 'clearing redis cache' do
   end
 
   describe 'clearing pipeline status cache' do
-    let(:pipeline_status) { create(:ci_pipeline).project.pipeline_status }
+    let(:pipeline_status) do
+      project = create(:project, :repository)
+      create(:ci_pipeline, project: project).project.pipeline_status
+    end
 
     before do
       allow(pipeline_status).to receive(:loaded).and_return(nil)
diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 70f3d77dc98..a8fae4a88a3 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -96,6 +96,27 @@ describe 'gitlab:app namespace rake task' do
         end
       end
     end
+
+    context 'when the restore directory is not empty' do
+      before do
+        # We only need a backup of the repositories for this test
+        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+      end
+
+      it 'removes stale data' do
+        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+
+        excluded_project = create(:project, :repository, name: 'mepmep')
+
+        expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+
+        raw_repo = excluded_project.repository.raw
+
+        # The restore will not find the repository in the backup, but will create
+        # an empty one in its place
+        expect(raw_repo.empty?).to be(true)
+      end
+    end
   end # backup_restore task
 
   describe 'backup' do
@@ -110,7 +131,9 @@ describe 'gitlab:app namespace rake task' do
 
       before do
         stub_env('SKIP', 'db')
-        path = File.join(project.repository.path_to_repo, 'custom_hooks')
+        path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          File.join(project.repository.path_to_repo, 'custom_hooks')
+        end
         FileUtils.mkdir_p(path)
         FileUtils.touch(File.join(path, "dummy.txt"))
       end
@@ -131,7 +154,10 @@ describe 'gitlab:app namespace rake task' do
           expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
           expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
 
-          expect(Dir.entries(File.join(project.repository.path, 'custom_hooks'))).to include("dummy.txt")
+          repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+            project.repository.path
+          end
+          expect(Dir.entries(File.join(repo_path, 'custom_hooks'))).to include("dummy.txt")
         end
       end
 
@@ -234,7 +260,7 @@ describe 'gitlab:app namespace rake task' do
 
         allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
 
-        # Avoid asking gitaly about the root ref (which will fail beacuse of the
+        # Avoid asking gitaly about the root ref (which will fail because of the
         # mocked storages)
         allow_any_instance_of(Repository).to receive(:empty?).and_return(false)
       end
@@ -252,10 +278,12 @@ describe 'gitlab:app namespace rake task' do
         FileUtils.mkdir_p(b_storage_dir)
 
         # Even when overriding the storage, we have to move it there, so it exists
-        FileUtils.mv(
-          File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'),
-          Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git')
-        )
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          FileUtils.mv(
+            File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'),
+            Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git')
+          )
+        end
 
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
 
diff --git a/spec/tasks/gitlab/check_rake_spec.rb b/spec/tasks/gitlab/check_rake_spec.rb
index 4eda618b6d6..06525e3c771 100644
--- a/spec/tasks/gitlab/check_rake_spec.rb
+++ b/spec/tasks/gitlab/check_rake_spec.rb
@@ -1,51 +1,101 @@
 require 'rake_helper'
 
-describe 'gitlab:ldap:check rake task' do
-  include LdapHelpers
-
+describe 'check.rake' do
   before do
     Rake.application.rake_require 'tasks/gitlab/check'
 
     stub_warn_user_is_not_gitlab
   end
 
-  context 'when LDAP is not enabled' do
-    it 'does not attempt to bind or search for users' do
-      expect(Gitlab::Auth::LDAP::Config).not_to receive(:providers)
-      expect(Gitlab::Auth::LDAP::Adapter).not_to receive(:open)
-
-      run_rake_task('gitlab:ldap:check')
+  shared_examples_for 'system check rake task' do
+    it 'runs the check' do
+      expect do
+        subject
+      end.to output(/Checking #{name} ... Finished/).to_stdout
     end
   end
 
-  context 'when LDAP is enabled' do
-    let(:ldap) { double(:ldap) }
-    let(:adapter) { ldap_adapter('ldapmain', ldap) }
+  describe 'gitlab:check rake task' do
+    subject { run_rake_task('gitlab:check') }
+    let(:name) { 'GitLab subtasks' }
 
-    before do
-      allow(Gitlab::Auth::LDAP::Config)
-        .to receive_messages(
-          enabled?: true,
-          providers: ['ldapmain']
-        )
-      allow(Gitlab::Auth::LDAP::Adapter).to receive(:open).and_yield(adapter)
-      allow(adapter).to receive(:users).and_return([])
-    end
+    it_behaves_like 'system check rake task'
+  end
+
+  describe 'gitlab:gitlab_shell:check rake task' do
+    subject { run_rake_task('gitlab:gitlab_shell:check') }
+    let(:name) { 'GitLab Shell' }
+
+    it_behaves_like 'system check rake task'
+  end
+
+  describe 'gitlab:gitaly:check rake task' do
+    subject { run_rake_task('gitlab:gitaly:check') }
+    let(:name) { 'Gitaly' }
+
+    it_behaves_like 'system check rake task'
+  end
+
+  describe 'gitlab:sidekiq:check rake task' do
+    subject { run_rake_task('gitlab:sidekiq:check') }
+    let(:name) { 'Sidekiq' }
 
-    it 'attempts to bind using credentials' do
-      stub_ldap_config(has_auth?: true)
+    it_behaves_like 'system check rake task'
+  end
 
-      expect(ldap).to receive(:bind)
+  describe 'gitlab:incoming_email:check rake task' do
+    subject { run_rake_task('gitlab:incoming_email:check') }
+    let(:name) { 'Incoming Email' }
 
-      run_rake_task('gitlab:ldap:check')
+    it_behaves_like 'system check rake task'
+  end
+
+  describe 'gitlab:ldap:check rake task' do
+    include LdapHelpers
+
+    subject { run_rake_task('gitlab:ldap:check') }
+    let(:name) { 'LDAP' }
+
+    it_behaves_like 'system check rake task'
+
+    context 'when LDAP is not enabled' do
+      it 'does not attempt to bind or search for users' do
+        expect(Gitlab::Auth::LDAP::Config).not_to receive(:providers)
+        expect(Gitlab::Auth::LDAP::Adapter).not_to receive(:open)
+
+        subject
+      end
     end
 
-    it 'searches for 100 LDAP users' do
-      stub_ldap_config(uid: 'uid')
+    context 'when LDAP is enabled' do
+      let(:ldap) { double(:ldap) }
+      let(:adapter) { ldap_adapter('ldapmain', ldap) }
+
+      before do
+        allow(Gitlab::Auth::LDAP::Config)
+          .to receive_messages(
+            enabled?: true,
+            providers: ['ldapmain']
+          )
+        allow(Gitlab::Auth::LDAP::Adapter).to receive(:open).and_yield(adapter)
+        allow(adapter).to receive(:users).and_return([])
+      end
+
+      it 'attempts to bind using credentials' do
+        stub_ldap_config(has_auth?: true)
+
+        expect(ldap).to receive(:bind)
+
+        subject
+      end
+
+      it 'searches for 100 LDAP users' do
+        stub_ldap_config(uid: 'uid')
 
-      expect(adapter).to receive(:users).with('uid', '*', 100)
+        expect(adapter).to receive(:users).with('uid', '*', 100)
 
-      run_rake_task('gitlab:ldap:check')
+        subject
+      end
     end
   end
 end
diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb
index 2bf873c923f..19794227d9f 100644
--- a/spec/tasks/gitlab/cleanup_rake_spec.rb
+++ b/spec/tasks/gitlab/cleanup_rake_spec.rb
@@ -5,7 +5,9 @@ describe 'gitlab:cleanup rake tasks' do
     Rake.application.rake_require 'tasks/gitlab/cleanup'
   end
 
-  describe 'cleanup' do
+  describe 'cleanup namespaces and repos' do
+    let(:gitlab_shell) { Gitlab::Shell.new }
+    let(:storage) { storages.keys.first }
     let(:storages) do
       {
         'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage'))
@@ -17,53 +19,140 @@ describe 'gitlab:cleanup rake tasks' do
     end
 
     before do
-      FileUtils.mkdir(Settings.absolute('tmp/tests/default_storage'))
       allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
     end
 
     after do
-      FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage'))
+      Gitlab::GitalyClient::StorageService.new(storage).delete_all_repositories
     end
 
     describe 'cleanup:repos' do
       before do
-        FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/broken/project.git'))
-        FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))
+        gitlab_shell.add_namespace(storage, 'broken/project.git')
+        gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git')
       end
 
       it 'moves it to an orphaned path' do
-        run_rake_task('gitlab:cleanup:repos')
-        repo_list = Dir['tmp/tests/default_storage/broken/*']
+        now = Time.now
+
+        Timecop.freeze(now) do
+          run_rake_task('gitlab:cleanup:repos')
+          repo_list = Gitlab::GitalyClient::StorageService.new(storage).list_directories(depth: 0)
 
-        expect(repo_list.first).to include('+orphaned+')
+          expect(repo_list.last).to include("broken+orphaned+#{now.to_i}")
+        end
       end
 
       it 'ignores @hashed repos' do
         run_rake_task('gitlab:cleanup:repos')
 
-        expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy
+        expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true)
       end
     end
 
     describe 'cleanup:dirs' do
       it 'removes missing namespaces' do
-        FileUtils.mkdir_p(Settings.absolute("tmp/tests/default_storage/namespace_1/project.git"))
-        FileUtils.mkdir_p(Settings.absolute("tmp/tests/default_storage/namespace_2/project.git"))
-        allow(Namespace).to receive(:pluck).and_return('namespace_1')
+        gitlab_shell.add_namespace(storage, "namespace_1/project.git")
+        gitlab_shell.add_namespace(storage, "namespace_2/project.git")
+        allow(Namespace).to receive(:pluck).and_return(['namespace_1'])
 
         stub_env('REMOVE', 'true')
         run_rake_task('gitlab:cleanup:dirs')
 
-        expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/namespace_1'))).to be_truthy
-        expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/namespace_2'))).to be_falsey
+        expect(gitlab_shell.exists?(storage, 'namespace_1')).to be(true)
+        expect(gitlab_shell.exists?(storage, 'namespace_2')).to be(false)
       end
 
       it 'ignores @hashed directory' do
-        FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))
+        gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git')
 
         run_rake_task('gitlab:cleanup:dirs')
 
-        expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy
+        expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true)
+      end
+    end
+  end
+
+  # A single integration test that is redundant with one part of the
+  # Gitlab::Cleanup::ProjectUploads spec.
+  #
+  # Additionally, this tests DRY_RUN env var values, and the extra line of
+  # output that says you can disable DRY_RUN if it's enabled.
+  describe 'cleanup:project_uploads' do
+    let!(:logger) { double(:logger) }
+
+    before do
+      expect(main_object).to receive(:logger).and_return(logger).at_least(1).times
+
+      allow(logger).to receive(:info).at_least(1).times
+      allow(logger).to receive(:debug).at_least(1).times
+    end
+
+    context 'with a fixable orphaned project upload file' do
+      let(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) }
+      let(:new_path) { orphaned.absolute_path }
+      let(:path) { File.join(FileUploader.root, 'some', 'wrong', 'location', orphaned.path) }
+
+      before do
+        FileUtils.mkdir_p(File.dirname(path))
+        FileUtils.mv(new_path, path)
+      end
+
+      context 'with DRY_RUN disabled' do
+        before do
+          stub_env('DRY_RUN', 'false')
+        end
+
+        it 'moves the file to its proper location' do
+          run_rake_task('gitlab:cleanup:project_uploads')
+
+          expect(File.exist?(path)).to be_falsey
+          expect(File.exist?(new_path)).to be_truthy
+        end
+
+        it 'logs action as done' do
+          expect(logger).to receive(:info).with("Looking for orphaned project uploads to clean up...")
+          expect(logger).to receive(:info).with("Did fix #{path} -> #{new_path}")
+
+          run_rake_task('gitlab:cleanup:project_uploads')
+        end
+      end
+
+      shared_examples_for 'does not move the file' do
+        it 'does not move the file' do
+          run_rake_task('gitlab:cleanup:project_uploads')
+
+          expect(File.exist?(path)).to be_truthy
+          expect(File.exist?(new_path)).to be_falsey
+        end
+
+        it 'logs action as able to be done' do
+          expect(logger).to receive(:info).with("Looking for orphaned project uploads to clean up. Dry run...")
+          expect(logger).to receive(:info).with("Can fix #{path} -> #{new_path}")
+          expect(logger).to receive(:info).with(/To clean up these files run this command with DRY_RUN=false/)
+
+          run_rake_task('gitlab:cleanup:project_uploads')
+        end
+      end
+
+      context 'with DRY_RUN explicitly enabled' do
+        before do
+          stub_env('DRY_RUN', 'true')
+        end
+
+        it_behaves_like 'does not move the file'
+      end
+
+      context 'with DRY_RUN set to an unknown value' do
+        before do
+          stub_env('DRY_RUN', 'foo')
+        end
+
+        it_behaves_like 'does not move the file'
+      end
+
+      context 'with DRY_RUN unset' do
+        it_behaves_like 'does not move the file'
       end
     end
   end
diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb
index fc52c04e78d..5818892d56a 100644
--- a/spec/tasks/gitlab/db_rake_spec.rb
+++ b/spec/tasks/gitlab/db_rake_spec.rb
@@ -20,7 +20,7 @@ describe 'gitlab:db namespace rake task' do
 
   describe 'configure' do
     it 'invokes db:migrate when schema has already been loaded' do
-      allow(ActiveRecord::Base.connection).to receive(:tables).and_return(['default'])
+      allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w[table1 table2])
       expect(Rake::Task['db:migrate']).to receive(:invoke)
       expect(Rake::Task['db:schema:load']).not_to receive(:invoke)
       expect(Rake::Task['db:seed_fu']).not_to receive(:invoke)
@@ -35,6 +35,14 @@ describe 'gitlab:db namespace rake task' do
       expect { run_rake_task('gitlab:db:configure') }.not_to raise_error
     end
 
+    it 'invokes db:shema:load and db:seed_fu when there is only a single table present' do
+      allow(ActiveRecord::Base.connection).to receive(:tables).and_return(['default'])
+      expect(Rake::Task['db:schema:load']).to receive(:invoke)
+      expect(Rake::Task['db:seed_fu']).to receive(:invoke)
+      expect(Rake::Task['db:migrate']).not_to receive(:invoke)
+      expect { run_rake_task('gitlab:db:configure') }.not_to raise_error
+    end
+
     it 'does not invoke any other rake tasks during an error' do
       allow(ActiveRecord::Base).to receive(:connection).and_raise(RuntimeError, 'error')
       expect(Rake::Task['db:migrate']).not_to receive(:invoke)
@@ -53,6 +61,39 @@ describe 'gitlab:db namespace rake task' do
       expect(Rake::Task['db:migrate']).not_to receive(:invoke)
       expect { run_rake_task('gitlab:db:configure') }.to raise_error(RuntimeError, 'error')
     end
+
+    context 'SKIP_POST_DEPLOYMENT_MIGRATIONS environment variable set' do
+      let(:rails_paths) { { 'db' => ['db'], 'db/migrate' => ['db/migrate'] } }
+
+      before do
+        allow(ENV).to receive(:[]).and_call_original
+        allow(ENV).to receive(:[]).with('SKIP_POST_DEPLOYMENT_MIGRATIONS').and_return true
+
+        # Our environment has already been loaded, so we need to pretend like post_migrations were not
+        allow(Rails.application.config).to receive(:paths).and_return(rails_paths)
+        allow(ActiveRecord::Migrator).to receive(:migrations_paths).and_return(rails_paths['db/migrate'].dup)
+      end
+
+      it 'adds post deployment migrations before schema load if the schema is not already loaded' do
+        allow(ActiveRecord::Base.connection).to receive(:tables).and_return([])
+        expect(Gitlab::Database).to receive(:add_post_migrate_path_to_rails).and_call_original
+        expect(Rake::Task['db:schema:load']).to receive(:invoke)
+        expect(Rake::Task['db:seed_fu']).to receive(:invoke)
+        expect(Rake::Task['db:migrate']).not_to receive(:invoke)
+        expect { run_rake_task('gitlab:db:configure') }.not_to raise_error
+        expect(rails_paths['db/migrate'].include?(File.join(Rails.root, 'db', 'post_migrate'))).to be(true)
+      end
+
+      it 'ignores post deployment migrations  when schema has already been loaded' do
+        allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w[table1 table2])
+        expect(Rake::Task['db:migrate']).to receive(:invoke)
+        expect(Gitlab::Database).not_to receive(:add_post_migrate_path_to_rails)
+        expect(Rake::Task['db:schema:load']).not_to receive(:invoke)
+        expect(Rake::Task['db:seed_fu']).not_to receive(:invoke)
+        expect { run_rake_task('gitlab:db:configure') }.not_to raise_error
+        expect(rails_paths['db/migrate'].include?(File.join(Rails.root, 'db', 'post_migrate'))).to be(false)
+      end
+    end
   end
 
   def run_rake_task(task_name)
diff --git a/spec/tasks/gitlab/git_rake_spec.rb b/spec/tasks/gitlab/git_rake_spec.rb
index 1efaecc63a5..57b006e1a39 100644
--- a/spec/tasks/gitlab/git_rake_spec.rb
+++ b/spec/tasks/gitlab/git_rake_spec.rb
@@ -1,41 +1,20 @@
 require 'rake_helper'
 
 describe 'gitlab:git rake tasks' do
-  before(:all) do
-    @default_storage_hash = Gitlab.config.repositories.storages.default.to_h
-  end
+  let(:base_path) { 'tmp/tests/default_storage' }
+  let!(:project) { create(:project, :repository) }
 
   before do
     Rake.application.rake_require 'tasks/gitlab/git'
-    storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) }
 
-    FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git'))
-    allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
     allow_any_instance_of(String).to receive(:color) { |string, _color| string }
 
     stub_warn_user_is_not_gitlab
   end
 
-  after do
-    FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage'))
-  end
-
   describe 'fsck' do
     it 'outputs the integrity check for a repo' do
-      expect { run_rake_task('gitlab:git:fsck') }.to output(%r{Performed Checking integrity at .*@hashed/1/2/test.git}).to_stdout
-    end
-
-    it 'errors out about config.lock issues' do
-      FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/config.lock'))
-
-      expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout
-    end
-
-    it 'errors out about ref lock issues' do
-      FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads'))
-      FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads/blah.lock'))
-
-      expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout
+      expect { run_rake_task('gitlab:git:fsck') }.to output(/Performed integrity check for/).to_stdout
     end
   end
 end
diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb
index 1e507c0236e..e6e4d9504d9 100644
--- a/spec/tasks/gitlab/gitaly_rake_spec.rb
+++ b/spec/tasks/gitlab/gitaly_rake_spec.rb
@@ -8,13 +8,23 @@ describe 'gitlab:gitaly namespace rake task' do
   describe 'install' do
     let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' }
     let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s }
+    let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s }
     let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp }
 
+    subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) }
+
     context 'no dir given' do
       it 'aborts and display a help message' do
         # avoid writing task output to spec progress
         allow($stderr).to receive :write
-        expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly/
+        expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
+      end
+    end
+
+    context 'no storage path given' do
+      it 'aborts and display a help message' do
+        allow($stderr).to receive :write
+        expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/
       end
     end
 
@@ -23,7 +33,7 @@ describe 'gitlab:gitaly namespace rake task' do
         expect(main_object)
           .to receive(:checkout_or_clone_version).and_raise 'Git error'
 
-        expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error 'Git error'
+        expect { subject }.to raise_error 'Git error'
       end
     end
 
@@ -36,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do
         expect(main_object)
           .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path)
 
-        run_rake_task('gitlab:gitaly:install', clone_path)
+        subject
       end
     end
 
@@ -59,7 +69,7 @@ describe 'gitlab:gitaly namespace rake task' do
           expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0])
           expect(main_object).to receive(:run_command!).with(command_preamble + %w[gmake]).and_return(true)
 
-          run_rake_task('gitlab:gitaly:install', clone_path)
+          subject
         end
       end
 
@@ -72,7 +82,7 @@ describe 'gitlab:gitaly namespace rake task' do
         it 'calls make in the gitaly directory' do
           expect(main_object).to receive(:run_command!).with(command_preamble + %w[make]).and_return(true)
 
-          run_rake_task('gitlab:gitaly:install', clone_path)
+          subject
         end
 
         context 'when Rails.env is test' do
@@ -89,53 +99,10 @@ describe 'gitlab:gitaly namespace rake task' do
           it 'calls make in the gitaly directory with --no-deployment flag for bundle' do
             expect(main_object).to receive(:run_command!).with(command_preamble + command).and_return(true)
 
-            run_rake_task('gitlab:gitaly:install', clone_path)
+            subject
           end
         end
       end
     end
   end
-
-  describe 'storage_config' do
-    it 'prints storage configuration in a TOML format' do
-      config = {
-        'default' => Gitlab::GitalyClient::StorageSettings.new(
-          'path' => '/path/to/default',
-          'gitaly_address' => 'unix:/path/to/my.socket'
-        ),
-        'nfs_01' => Gitlab::GitalyClient::StorageSettings.new(
-          'path' => '/path/to/nfs_01',
-          'gitaly_address' => 'unix:/path/to/my.socket'
-        )
-      }
-      allow(Gitlab.config.repositories).to receive(:storages).and_return(config)
-      allow(Rails.env).to receive(:test?).and_return(false)
-
-      expected_output = ''
-      Timecop.freeze do
-        expected_output = <<~TOML
-          # Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}
-          # This is in TOML format suitable for use in Gitaly's config.toml file.
-          bin_dir = "tmp/tests/gitaly"
-          socket_path = "/path/to/my.socket"
-          [gitlab-shell]
-          dir = "#{Gitlab.config.gitlab_shell.path}"
-          [[storage]]
-          name = "default"
-          path = "/path/to/default"
-          [[storage]]
-          name = "nfs_01"
-          path = "/path/to/nfs_01"
-        TOML
-      end
-
-      expect { run_rake_task('gitlab:gitaly:storage_config')}
-        .to output(expected_output).to_stdout
-
-      parsed_output = TomlRB.parse(expected_output)
-      config.each do |name, params|
-        expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path })
-      end
-    end
-  end
 end
diff --git a/spec/tasks/gitlab/shell_rake_spec.rb b/spec/tasks/gitlab/shell_rake_spec.rb
index 4a756c5742d..0ed5d3e27b9 100644
--- a/spec/tasks/gitlab/shell_rake_spec.rb
+++ b/spec/tasks/gitlab/shell_rake_spec.rb
@@ -7,11 +7,17 @@ describe 'gitlab:shell rake tasks' do
     stub_warn_user_is_not_gitlab
   end
 
+  after do
+    TestEnv.create_fake_git_hooks
+  end
+
   describe 'install task' do
     it 'invokes create_hooks task' do
       expect(Rake::Task['gitlab:shell:create_hooks']).to receive(:invoke)
 
-      storages = Gitlab.config.repositories.storages.values.map(&:legacy_disk_path)
+      storages = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        Gitlab.config.repositories.storages.values.map(&:legacy_disk_path)
+      end
       expect(Kernel).to receive(:system).with('bin/install', *storages).and_call_original
       expect(Kernel).to receive(:system).with('bin/compile').and_call_original
 
diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb
index 35e451b2f9a..6b50670c3c0 100644
--- a/spec/tasks/gitlab/storage_rake_spec.rb
+++ b/spec/tasks/gitlab/storage_rake_spec.rb
@@ -1,6 +1,6 @@
 require 'rake_helper'
 
-describe 'gitlab:storage:*' do
+describe 'rake gitlab:storage:*' do
   before do
     Rake.application.rake_require 'tasks/gitlab/storage'
 
@@ -44,28 +44,40 @@ describe 'gitlab:storage:*' do
   end
 
   describe 'gitlab:storage:migrate_to_hashed' do
+    let(:task) { 'gitlab:storage:migrate_to_hashed' }
+
+    context 'read-only database' do
+      it 'does nothing' do
+        expect(Gitlab::Database).to receive(:read_only?).and_return(true)
+
+        expect(Project).not_to receive(:with_unmigrated_storage)
+
+        expect { run_rake_task(task) }.to output(/This task requires database write access. Exiting./).to_stderr
+      end
+    end
+
     context '0 legacy projects' do
       it 'does nothing' do
-        expect(StorageMigratorWorker).not_to receive(:perform_async)
+        expect(::HashedStorage::MigratorWorker).not_to receive(:perform_async)
 
-        run_rake_task('gitlab:storage:migrate_to_hashed')
+        run_rake_task(task)
       end
     end
 
     context '3 legacy projects' do
-      let(:projects) { create_list(:project, 3, storage_version: 0) }
+      let(:projects) { create_list(:project, 3, :legacy_storage) }
 
       context 'in batches of 1' do
         before do
           stub_env('BATCH' => 1)
         end
 
-        it 'enqueues one StorageMigratorWorker per project' do
+        it 'enqueues one HashedStorage::MigratorWorker per project' do
           projects.each do |project|
-            expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id)
+            expect(::HashedStorage::MigratorWorker).to receive(:perform_async).with(project.id, project.id)
           end
 
-          run_rake_task('gitlab:storage:migrate_to_hashed')
+          run_rake_task(task)
         end
       end
 
@@ -74,29 +86,54 @@ describe 'gitlab:storage:*' do
           stub_env('BATCH' => 2)
         end
 
-        it 'enqueues one StorageMigratorWorker per 2 projects' do
+        it 'enqueues one HashedStorage::MigratorWorker per 2 projects' do
           projects.map(&:id).sort.each_slice(2) do |first, last|
             last ||= first
-            expect(StorageMigratorWorker).to receive(:perform_async).with(first, last)
+            expect(::HashedStorage::MigratorWorker).to receive(:perform_async).with(first, last)
           end
 
-          run_rake_task('gitlab:storage:migrate_to_hashed')
+          run_rake_task(task)
         end
       end
     end
+
+    context 'with same id in range' do
+      it 'displays message when project cant be found' do
+        stub_env('ID_FROM', 99999)
+        stub_env('ID_TO', 99999)
+
+        expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=99999/).to_stderr
+      end
+
+      it 'displays a message when project exists but its already migrated' do
+        project = create(:project)
+        stub_env('ID_FROM', project.id)
+        stub_env('ID_TO', project.id)
+
+        expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=#{project.id}/).to_stderr
+      end
+
+      it 'enqueues migration when project can be found' do
+        project = create(:project, :legacy_storage)
+        stub_env('ID_FROM', project.id)
+        stub_env('ID_TO', project.id)
+
+        expect { run_rake_task(task) }.to output(/Enqueueing storage migration .* \(ID=#{project.id}\)/).to_stdout
+      end
+    end
   end
 
   describe 'gitlab:storage:legacy_projects' do
     it_behaves_like 'rake entities summary', 'projects', 'Legacy' do
       let(:task) { 'gitlab:storage:legacy_projects' }
-      let(:create_collection) { create_list(:project, 3, storage_version: 0) }
+      let(:create_collection) { create_list(:project, 3, :legacy_storage) }
     end
   end
 
   describe 'gitlab:storage:list_legacy_projects' do
     it_behaves_like 'rake listing entities', 'projects', 'Legacy' do
       let(:task) { 'gitlab:storage:list_legacy_projects' }
-      let(:create_collection) { create_list(:project, 3, storage_version: 0) }
+      let(:create_collection) { create_list(:project, 3, :legacy_storage) }
     end
   end
 
@@ -133,7 +170,7 @@ describe 'gitlab:storage:*' do
   describe 'gitlab:storage:hashed_attachments' do
     it_behaves_like 'rake entities summary', 'attachments', 'Hashed' do
       let(:task) { 'gitlab:storage:hashed_attachments' }
-      let(:project) { create(:project, storage_version: 2) }
+      let(:project) { create(:project) }
       let(:create_collection) { create_list(:upload, 3, model: project) }
     end
   end
@@ -141,7 +178,7 @@ describe 'gitlab:storage:*' do
   describe 'gitlab:storage:list_hashed_attachments' do
     it_behaves_like 'rake listing entities', 'attachments', 'Hashed' do
       let(:task) { 'gitlab:storage:list_hashed_attachments' }
-      let(:project) { create(:project, storage_version: 2) }
+      let(:project) { create(:project) }
       let(:create_collection) { create_list(:upload, 3, model: project) }
     end
   end
diff --git a/spec/tasks/gitlab/traces_rake_spec.rb b/spec/tasks/gitlab/traces_rake_spec.rb
index bd18e8ffc1e..aaf0d7242dd 100644
--- a/spec/tasks/gitlab/traces_rake_spec.rb
+++ b/spec/tasks/gitlab/traces_rake_spec.rb
@@ -5,51 +5,109 @@ describe 'gitlab:traces rake tasks' do
     Rake.application.rake_require 'tasks/gitlab/traces'
   end
 
-  shared_examples 'passes the job id to worker' do
-    it do
-      expect(ArchiveTraceWorker).to receive(:bulk_perform_async).with([[job.id]])
+  describe 'gitlab:traces:archive' do
+    shared_examples 'passes the job id to worker' do
+      it do
+        expect(ArchiveTraceWorker).to receive(:bulk_perform_async).with([[job.id]])
 
-      run_rake_task('gitlab:traces:archive')
+        run_rake_task('gitlab:traces:archive')
+      end
     end
-  end
 
-  shared_examples 'does not pass the job id to worker' do
-    it do
-      expect(ArchiveTraceWorker).not_to receive(:bulk_perform_async)
+    shared_examples 'does not pass the job id to worker' do
+      it do
+        expect(ArchiveTraceWorker).not_to receive(:bulk_perform_async)
 
-      run_rake_task('gitlab:traces:archive')
+        run_rake_task('gitlab:traces:archive')
+      end
     end
-  end
 
-  context 'when trace file stored in default path' do
-    let!(:job) { create(:ci_build, :success, :trace_live) }
+    context 'when trace file stored in default path' do
+      let!(:job) { create(:ci_build, :success, :trace_live) }
 
-    it_behaves_like 'passes the job id to worker'
-  end
+      it_behaves_like 'passes the job id to worker'
+    end
 
-  context 'when trace is stored in database' do
-    let!(:job) { create(:ci_build, :success) }
+    context 'when trace is stored in database' do
+      let!(:job) { create(:ci_build, :success) }
 
-    before do
-      job.update_column(:trace, 'trace in db')
+      before do
+        job.update_column(:trace, 'trace in db')
+      end
+
+      it_behaves_like 'passes the job id to worker'
     end
 
-    it_behaves_like 'passes the job id to worker'
+    context 'when job has trace artifact' do
+      let!(:job) { create(:ci_build, :success) }
+
+      before do
+        create(:ci_job_artifact, :trace, job: job)
+      end
+
+      it_behaves_like 'does not pass the job id to worker'
+    end
+
+    context 'when job is not finished yet' do
+      let!(:build) { create(:ci_build, :running, :trace_live) }
+
+      it_behaves_like 'does not pass the job id to worker'
+    end
   end
 
-  context 'when job has trace artifact' do
-    let!(:job) { create(:ci_build, :success) }
+  describe 'gitlab:traces:migrate' do
+    let(:object_storage_enabled) { false }
 
     before do
-      create(:ci_job_artifact, :trace, job: job)
+      stub_artifacts_object_storage(enabled: object_storage_enabled)
     end
 
-    it_behaves_like 'does not pass the job id to worker'
-  end
+    subject { run_rake_task('gitlab:traces:migrate') }
 
-  context 'when job is not finished yet' do
-    let!(:build) { create(:ci_build, :running, :trace_live) }
+    let!(:job_trace) { create(:ci_job_artifact, :trace, file_store: store) }
 
-    it_behaves_like 'does not pass the job id to worker'
+    context 'when local storage is used' do
+      let(:store) { ObjectStorage::Store::LOCAL }
+
+      context 'and job does not have file store defined' do
+        let(:object_storage_enabled) { true }
+        let(:store) { nil }
+
+        it "migrates file to remote storage" do
+          subject
+
+          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+
+      context 'and remote storage is defined' do
+        let(:object_storage_enabled) { true }
+
+        it "migrates file to remote storage" do
+          subject
+
+          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+
+      context 'and remote storage is not defined' do
+        it "fails to migrate to remote storage" do
+          subject
+
+          expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        end
+      end
+    end
+
+    context 'when remote storage is used' do
+      let(:object_storage_enabled) { true }
+      let(:store) { ObjectStorage::Store::REMOTE }
+
+      it "file stays on remote storage" do
+        subject
+
+        expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+      end
+    end
   end
 end
diff --git a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
index 6fcfae358ec..9588e8be5dc 100644
--- a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
+++ b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
@@ -38,14 +38,6 @@ describe 'gitlab:uploads:migrate rake tasks' do
       let!(:projects) { create_list(:project, 10, :with_avatar) }
 
       it_behaves_like 'enqueue jobs in batch', batch: 4
-
-      context 'Upload has store = nil' do
-        before do
-          Upload.where(model: projects).update_all(store: nil)
-        end
-
-        it_behaves_like 'enqueue jobs in batch', batch: 4
-      end
     end
 
     context "for Group" do
diff --git a/spec/tasks/gitlab/web_hook_rake_spec.rb b/spec/tasks/gitlab/web_hook_rake_spec.rb
new file mode 100644
index 00000000000..7bdf33ff6b0
--- /dev/null
+++ b/spec/tasks/gitlab/web_hook_rake_spec.rb
@@ -0,0 +1,92 @@
+require 'rake_helper'
+
+describe 'gitlab:web_hook namespace rake tasks' do
+  set(:group) { create(:group) }
+
+  set(:project1) { create(:project, namespace: group) }
+  set(:project2) { create(:project, namespace: group) }
+  set(:other_group_project) { create(:project) }
+
+  let(:url) { 'http://example.com' }
+  let(:hook_urls) { (project1.hooks + project2.hooks).map(&:url) }
+  let(:other_group_hook_urls) { other_group_project.hooks.map(&:url) }
+
+  before do
+    Rake.application.rake_require 'tasks/gitlab/web_hook'
+  end
+
+  describe 'gitlab:web_hook:add' do
+    it 'adds a web hook to all projects' do
+      stub_env('URL' => url)
+      run_rake_task('gitlab:web_hook:add')
+
+      expect(hook_urls).to contain_exactly(url, url)
+      expect(other_group_hook_urls).to contain_exactly(url)
+    end
+
+    it 'adds a web hook to projects in the specified namespace' do
+      stub_env('URL' => url, 'NAMESPACE' => group.full_path)
+      run_rake_task('gitlab:web_hook:add')
+
+      expect(hook_urls).to contain_exactly(url, url)
+      expect(other_group_hook_urls).to be_empty
+    end
+
+    it 'raises an error if an unknown namespace is specified' do
+      stub_env('URL' => url, 'NAMESPACE' => group.full_path)
+
+      group.destroy
+
+      expect { run_rake_task('gitlab:web_hook:add') }.to raise_error(SystemExit)
+    end
+  end
+
+  describe 'gitlab:web_hook:rm' do
+    let!(:hook1) { create(:project_hook, project: project1, url: url) }
+    let!(:hook2) { create(:project_hook, project: project2, url: url) }
+    let!(:other_group_hook) { create(:project_hook, project: other_group_project, url: url) }
+    let!(:other_url_hook) { create(:project_hook, url: other_url, project: project1) }
+
+    let(:other_url) { 'http://other.example.com' }
+
+    it 'removes a web hook from all projects by URL' do
+      stub_env('URL' => url)
+      run_rake_task('gitlab:web_hook:rm')
+
+      expect(hook_urls).to contain_exactly(other_url)
+      expect(other_group_hook_urls).to be_empty
+    end
+
+    it 'removes a web hook from projects in the specified namespace by URL' do
+      stub_env('NAMESPACE' => group.full_path, 'URL' => url)
+      run_rake_task('gitlab:web_hook:rm')
+
+      expect(hook_urls).to contain_exactly(other_url)
+      expect(other_group_hook_urls).to contain_exactly(url)
+    end
+
+    it 'raises an error if an unknown namespace is specified' do
+      stub_env('URL' => url, 'NAMESPACE' => group.full_path)
+
+      group.destroy
+
+      expect { run_rake_task('gitlab:web_hook:rm') }.to raise_error(SystemExit)
+    end
+  end
+
+  describe 'gitlab:web_hook:list' do
+    let!(:hook1) { create(:project_hook, project: project1) }
+    let!(:hook2) { create(:project_hook, project: project2) }
+    let!(:other_group_hook) { create(:project_hook, project: other_group_project) }
+
+    it 'lists all web hooks' do
+      expect { run_rake_task('gitlab:web_hook:list') }.to output(/3 webhooks found/).to_stdout
+    end
+
+    it 'lists web hooks in a particular namespace' do
+      stub_env('NAMESPACE', group.full_path)
+
+      expect { run_rake_task('gitlab:web_hook:list') }.to output(/2 webhooks found/).to_stdout
+    end
+  end
+end