Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42

3 files changed, 145 insertions, 67 deletions

diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index a2cc2b12e5e..8963164ac53 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -284,62 +284,75 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     end
 
     context 'multiple repository storages' do
-      let_it_be(:default_storage_hash) { Gitlab.config.repositories.storages.default.to_h }
+      include StubConfiguration
+
+      let(:default_storage_name) { 'default' }
+      let(:second_storage_name) { 'test_second_storage' }
 
       before do
         # We only need a backup of the repositories for this test
         stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
-
-        allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
-
-        # Avoid asking gitaly about the root ref (which will fail because of the
-        # mocked storages)
-        allow_any_instance_of(Repository).to receive(:empty?).and_return(false)
-
-        FileUtils.mkdir_p(b_storage_dir)
-
-        # Even when overriding the storage, we have to move it there, so it exists
-        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          FileUtils.mv(
-            File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'),
-            Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git')
-          )
-        end
-      end
-
-      after do
-        FileUtils.rm_rf(test_second_storage_dir)
+        stub_storage_settings( second_storage_name => {
+          'gitaly_address' => Gitlab.config.repositories.storages.default.gitaly_address,
+          'path' => TestEnv::SECOND_STORAGE_PATH
+        })
       end
 
-      let(:test_second_storage_dir) { Dir.mktmpdir }
+      shared_examples 'includes repositories in all repository storages' do
+        specify :aggregate_failures do
+          project_a = create(:project, :repository)
+          project_a.track_project_repository
+          project_snippet_a = create(:project_snippet, :repository, project: project_a, author: project_a.owner)
+          project_b = create(:project, :repository, repository_storage: second_storage_name)
+          project_b.track_project_repository
+          project_snippet_b = create(:project_snippet, :repository, project: project_b, author: project_b.owner)
+          project_snippet_b.snippet_repository.update!(shard: project_b.project_repository.shard)
+          create(:wiki_page, container: project_a)
+          create(:design, :with_file, issue: create(:issue, project: project_a))
+
+          move_repository_to_secondary(project_b)
+          move_repository_to_secondary(project_snippet_b)
 
-      let(:test_second_storage) do
-        Gitlab::GitalyClient::StorageSettings.new(default_storage_hash.merge('path' => test_second_storage_dir))
-      end
-
-      let(:storages) do
-        {
-          'default' => Gitlab.config.repositories.storages.default,
-          'test_second_storage' => test_second_storage
-        }
-      end
-
-      let!(:project_a) { create(:project, :repository) }
-      let!(:project_b) { create(:project, :repository, repository_storage: 'test_second_storage') }
-      let!(:b_storage_dir) { File.join(test_second_storage_dir, File.dirname(project_b.disk_path)) }
-
-      context 'no concurrency' do
-        it 'includes repositories in all repository storages' do
           expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
 
           tar_contents, exit_status = Gitlab::Popen.popen(
             %W{tar -tvf #{backup_tar} repositories}
           )
 
+          tar_lines = tar_contents.lines.grep(/\.bundle/)
+
           expect(exit_status).to eq(0)
-          expect(tar_contents).to match("repositories/#{project_a.disk_path}.bundle")
-          expect(tar_contents).to match("repositories/#{project_b.disk_path}.bundle")
+
+          [
+            "#{project_a.disk_path}.bundle",
+            "#{project_a.disk_path}.wiki.bundle",
+            "#{project_a.disk_path}.design.bundle",
+            "#{project_b.disk_path}.bundle",
+            "#{project_snippet_a.disk_path}.bundle",
+            "#{project_snippet_b.disk_path}.bundle"
+          ].each do |repo_name|
+            expect(tar_lines.grep(/#{repo_name}/).size).to eq 1
+          end
         end
+
+        def move_repository_to_secondary(record)
+          Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+            default_shard_legacy_path = Gitlab.config.repositories.storages.default.legacy_disk_path
+            secondary_legacy_path = Gitlab.config.repositories.storages[second_storage_name].legacy_disk_path
+            dst_dir = File.join(secondary_legacy_path, File.dirname(record.disk_path))
+
+            FileUtils.mkdir_p(dst_dir) unless Dir.exist?(dst_dir)
+
+            FileUtils.mv(
+              File.join(default_shard_legacy_path, record.disk_path + '.git'),
+              File.join(secondary_legacy_path, record.disk_path + '.git')
+            )
+          end
+        end
+      end
+
+      context 'no concurrency' do
+        it_behaves_like 'includes repositories in all repository storages'
       end
 
       context 'with concurrency' do
@@ -347,17 +360,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 4)
         end
 
-        it 'includes repositories in all repository storages' do
-          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
-
-          tar_contents, exit_status = Gitlab::Popen.popen(
-            %W{tar -tvf #{backup_tar} repositories}
-          )
-
-          expect(exit_status).to eq(0)
-          expect(tar_contents).to match("repositories/#{project_a.disk_path}.bundle")
-          expect(tar_contents).to match("repositories/#{project_b.disk_path}.bundle")
-        end
+        it_behaves_like 'includes repositories in all repository storages'
       end
     end
 
@@ -370,7 +373,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       it 'has defaults' do
-        expect_next_instance_of(::Backup::Repository) do |instance|
+        expect_next_instance_of(::Backup::Repositories) do |instance|
           expect(instance).to receive(:dump)
             .with(max_concurrency: 1, max_storage_concurrency: 1)
             .and_call_original
@@ -383,7 +386,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 5)
         stub_env('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 2)
 
-        expect_next_instance_of(::Backup::Repository) do |instance|
+        expect_next_instance_of(::Backup::Repositories) do |instance|
           expect(instance).to receive(:dump)
             .with(max_concurrency: 5, max_storage_concurrency: 2)
             .and_call_original
diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb
index 99efd394e83..046e066a45f 100644
--- a/spec/tasks/gitlab/db_rake_spec.rb
+++ b/spec/tasks/gitlab/db_rake_spec.rb
@@ -98,6 +98,29 @@ RSpec.describe 'gitlab:db namespace rake task' do
     end
   end
 
+  describe 'unattended' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:schema_migration_table_exists, :needs_migrations, :rake_output) do
+      false | false | "unattended_migrations_completed"
+      false | true | "unattended_migrations_completed"
+      true | false | "unattended_migrations_static"
+      true | true | "unattended_migrations_completed"
+    end
+
+    before do
+      allow(Rake::Task['gitlab:db:configure']).to receive(:invoke).and_return(true)
+    end
+
+    with_them do
+      it 'outputs changed message for automation after operations happen' do
+        allow(ActiveRecord::Base.connection.schema_migration).to receive(:table_exists?).and_return(schema_migration_table_exists)
+        allow_any_instance_of(ActiveRecord::MigrationContext).to receive(:needs_migration?).and_return(needs_migrations)
+        expect { run_rake_task('gitlab:db:unattended') }. to output(/^#{rake_output}$/).to_stdout
+      end
+    end
+  end
+
   describe 'clean_structure_sql' do
     let_it_be(:clean_rake_task) { 'gitlab:db:clean_structure_sql' }
     let_it_be(:test_task_name) { 'gitlab:db:_test_multiple_structure_cleans' }
@@ -164,25 +187,77 @@ RSpec.describe 'gitlab:db namespace rake task' do
     end
   end
 
+  describe 'drop_tables' do
+    subject { run_rake_task('gitlab:db:drop_tables') }
+
+    let(:tables) { %w(one two) }
+    let(:views) { %w(three four) }
+    let(:connection) { ActiveRecord::Base.connection }
+
+    before do
+      allow(connection).to receive(:execute).and_return(nil)
+
+      allow(connection).to receive(:tables).and_return(tables)
+      allow(connection).to receive(:views).and_return(views)
+    end
+
+    it 'drops all tables, except schema_migrations' do
+      expect(connection).to receive(:execute).with('DROP TABLE IF EXISTS "one" CASCADE')
+      expect(connection).to receive(:execute).with('DROP TABLE IF EXISTS "two" CASCADE')
+
+      subject
+    end
+
+    it 'drops all views' do
+      expect(connection).to receive(:execute).with('DROP VIEW IF EXISTS "three" CASCADE')
+      expect(connection).to receive(:execute).with('DROP VIEW IF EXISTS "four" CASCADE')
+
+      subject
+    end
+
+    it 'truncates schema_migrations table' do
+      expect(connection).to receive(:execute).with('TRUNCATE schema_migrations')
+
+      subject
+    end
+
+    it 'drops extra schemas' do
+      Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
+        expect(connection).to receive(:execute).with("DROP SCHEMA IF EXISTS \"#{schema}\"")
+      end
+
+      subject
+    end
+  end
+
   describe 'reindex' do
+    let(:reindex) { double('reindex') }
+    let(:indexes) { double('indexes') }
+
     context 'when no index_name is given' do
-      it 'raises an error' do
-        expect do
-          run_rake_task('gitlab:db:reindex', '')
-        end.to raise_error(ArgumentError, /must give the index name/)
+      it 'rebuilds a random number of large indexes' do
+        expect(Gitlab::Database::Reindexing).to receive_message_chain('candidate_indexes.random_few').and_return(indexes)
+        expect(Gitlab::Database::Reindexing).to receive(:perform).with(indexes)
+
+        run_rake_task('gitlab:db:reindex')
       end
     end
 
-    it 'calls the index rebuilder with the proper arguments' do
-      reindex = double('rebuilder')
+    context 'with index name given' do
+      let(:index) { double('index') }
+
+      it 'calls the index rebuilder with the proper arguments' do
+        expect(Gitlab::Database::PostgresIndex).to receive(:by_identifier).with('public.foo_idx').and_return(index)
+        expect(Gitlab::Database::Reindexing).to receive(:perform).with([index])
 
-      expect(Gitlab::Database::ConcurrentReindex).to receive(:new)
-        .with('some_index_name', logger: instance_of(Logger))
-        .and_return(reindex)
+        run_rake_task('gitlab:db:reindex', '[public.foo_idx]')
+      end
 
-      expect(reindex).to receive(:execute)
+      it 'raises an error if the index does not exist' do
+        expect(Gitlab::Database::PostgresIndex).to receive(:by_identifier).with('public.absent_index').and_raise(ActiveRecord::RecordNotFound)
 
-      run_rake_task('gitlab:db:reindex', '[some_index_name]')
+        expect { run_rake_task('gitlab:db:reindex', '[public.absent_index]') }.to raise_error(ActiveRecord::RecordNotFound)
+      end
     end
   end
 
diff --git a/spec/tasks/gitlab/web_hook_rake_spec.rb b/spec/tasks/gitlab/web_hook_rake_spec.rb
index 3e18ce5ab29..9f373e3a20a 100644
--- a/spec/tasks/gitlab/web_hook_rake_spec.rb
+++ b/spec/tasks/gitlab/web_hook_rake_spec.rb
@@ -36,7 +36,7 @@ RSpec.describe 'gitlab:web_hook namespace rake tasks' do
     it 'raises an error if an unknown namespace is specified' do
       stub_env('URL' => url, 'NAMESPACE' => group.full_path)
 
-      group.destroy
+      group.destroy!
 
       expect { run_rake_task('gitlab:web_hook:add') }.to raise_error(SystemExit)
     end
@@ -69,7 +69,7 @@ RSpec.describe 'gitlab:web_hook namespace rake tasks' do
     it 'raises an error if an unknown namespace is specified' do
       stub_env('URL' => url, 'NAMESPACE' => group.full_path)
 
-      group.destroy
+      group.destroy!
 
       expect { run_rake_task('gitlab:web_hook:rm') }.to raise_error(SystemExit)
     end