Add latest changes from gitlab-org/gitlab@master

3 files changed, 140 insertions, 273 deletions

diff --git a/spec/lib/backup/gitaly_rpc_backup_spec.rb b/spec/lib/backup/gitaly_rpc_backup_spec.rb
deleted file mode 100644
index fb442f4a86f..00000000000
--- a/spec/lib/backup/gitaly_rpc_backup_spec.rb
+++ /dev/null
@@ -1,153 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Backup::GitalyRpcBackup do
-  let(:progress) { spy(:stdout) }
-
-  subject { described_class.new(progress) }
-
-  after do
-    # make sure we do not leave behind any backup files
-    FileUtils.rm_rf(File.join(Gitlab.config.backup.path, 'repositories'))
-  end
-
-  context 'unknown' do
-    it 'fails to start unknown' do
-      expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
-    end
-  end
-
-  context 'create' do
-    RSpec.shared_examples 'creates a repository backup' do
-      it 'creates repository bundles', :aggregate_failures do
-        # Add data to the wiki, design repositories, and snippets, so they will be included in the dump.
-        create(:wiki_page, container: project)
-        create(:design, :with_file, issue: create(:issue, project: project))
-        project_snippet = create(:project_snippet, :repository, project: project)
-        personal_snippet = create(:personal_snippet, :repository, author: project.owner)
-
-        subject.start(:create)
-        subject.enqueue(project, Gitlab::GlRepository::PROJECT)
-        subject.enqueue(project, Gitlab::GlRepository::WIKI)
-        subject.enqueue(project, Gitlab::GlRepository::DESIGN)
-        subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
-        subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
-        subject.wait
-
-        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
-        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
-        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle'))
-        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle'))
-        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle'))
-      end
-
-      context 'failure' do
-        before do
-          allow_next_instance_of(Repository) do |repository|
-            allow(repository).to receive(:bundle_to_disk) { raise 'Fail in tests' }
-          end
-        end
-
-        it 'logs an appropriate message', :aggregate_failures do
-          subject.start(:create)
-          subject.enqueue(project, Gitlab::GlRepository::PROJECT)
-          subject.wait
-
-          expect(progress).to have_received(:puts).with("[Failed] backing up #{project.full_path} (#{project.disk_path})")
-          expect(progress).to have_received(:puts).with("Error Fail in tests")
-        end
-      end
-    end
-
-    context 'hashed storage' do
-      let_it_be(:project) { create(:project, :repository) }
-
-      it_behaves_like 'creates a repository backup'
-    end
-
-    context 'legacy storage' do
-      let_it_be(:project) { create(:project, :repository, :legacy_storage) }
-
-      it_behaves_like 'creates a repository backup'
-    end
-  end
-
-  context 'restore' do
-    let_it_be(:project) { create(:project, :repository) }
-    let_it_be(:personal_snippet) { create(:personal_snippet, author: project.owner) }
-    let_it_be(:project_snippet) { create(:project_snippet, project: project, author: project.owner) }
-
-    def copy_bundle_to_backup_path(bundle_name, destination)
-      FileUtils.mkdir_p(File.join(Gitlab.config.backup.path, 'repositories', File.dirname(destination)))
-      FileUtils.cp(Rails.root.join('spec/fixtures/lib/backup', bundle_name), File.join(Gitlab.config.backup.path, 'repositories', destination))
-    end
-
-    it 'restores from repository bundles', :aggregate_failures do
-      copy_bundle_to_backup_path('project_repo.bundle', project.disk_path + '.bundle')
-      copy_bundle_to_backup_path('wiki_repo.bundle', project.disk_path + '.wiki.bundle')
-      copy_bundle_to_backup_path('design_repo.bundle', project.disk_path + '.design.bundle')
-      copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle')
-      copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle')
-
-      subject.start(:restore)
-      subject.enqueue(project, Gitlab::GlRepository::PROJECT)
-      subject.enqueue(project, Gitlab::GlRepository::WIKI)
-      subject.enqueue(project, Gitlab::GlRepository::DESIGN)
-      subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
-      subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
-      subject.wait
-
-      collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
-
-      expect(collect_commit_shas.call(project.repository)).to eq(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec'])
-      expect(collect_commit_shas.call(project.wiki.repository)).to eq(['c74b9948d0088d703ee1fafeddd9ed9add2901ea'])
-      expect(collect_commit_shas.call(project.design_repository)).to eq(['c3cd4d7bd73a51a0f22045c3a4c871c435dc959d'])
-      expect(collect_commit_shas.call(personal_snippet.repository)).to eq(['3b3c067a3bc1d1b695b51e2be30c0f8cf698a06e'])
-      expect(collect_commit_shas.call(project_snippet.repository)).to eq(['6e44ba56a4748be361a841e759c20e421a1651a1'])
-    end
-
-    it 'cleans existing repositories', :aggregate_failures do
-      expect_next_instance_of(DesignManagement::Repository) do |repository|
-        expect(repository).to receive(:remove)
-      end
-
-      # 4 times = project repo + wiki repo + project_snippet repo + personal_snippet repo
-      expect(Repository).to receive(:new).exactly(4).times.and_wrap_original do |method, *original_args|
-        full_path, container, kwargs = original_args
-
-        repository = method.call(full_path, container, **kwargs)
-
-        expect(repository).to receive(:remove)
-
-        repository
-      end
-
-      subject.start(:restore)
-      subject.enqueue(project, Gitlab::GlRepository::PROJECT)
-      subject.enqueue(project, Gitlab::GlRepository::WIKI)
-      subject.enqueue(project, Gitlab::GlRepository::DESIGN)
-      subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
-      subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
-      subject.wait
-    end
-
-    context 'failure' do
-      before do
-        allow_next_instance_of(Repository) do |repository|
-          allow(repository).to receive(:create_repository) { raise 'Fail in tests' }
-          allow(repository).to receive(:create_from_bundle) { raise 'Fail in tests' }
-        end
-      end
-
-      it 'logs an appropriate message', :aggregate_failures do
-        subject.start(:restore)
-        subject.enqueue(project, Gitlab::GlRepository::PROJECT)
-        subject.wait
-
-        expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} (#{project.disk_path})")
-        expect(progress).to have_received(:puts).with("Error Fail in tests")
-      end
-    end
-  end
-end
diff --git a/spec/lib/backup/repositories_spec.rb b/spec/lib/backup/repositories_spec.rb
index 85818038c9d..a991ddc62db 100644
--- a/spec/lib/backup/repositories_spec.rb
+++ b/spec/lib/backup/repositories_spec.rb
@@ -4,8 +4,7 @@ require 'spec_helper'
 
 RSpec.describe Backup::Repositories do
   let(:progress) { spy(:stdout) }
-  let(:parallel_enqueue) { true }
-  let(:strategy) { spy(:strategy, parallel_enqueue?: parallel_enqueue) }
+  let(:strategy) { spy(:strategy) }
 
   subject { described_class.new(progress, strategy: strategy) }
 
@@ -17,7 +16,7 @@ RSpec.describe Backup::Repositories do
         project_snippet = create(:project_snippet, :repository, project: project)
         personal_snippet = create(:personal_snippet, :repository, author: project.owner)
 
-        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        subject.dump
 
         expect(strategy).to have_received(:start).with(:create)
         expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
@@ -41,132 +40,30 @@ RSpec.describe Backup::Repositories do
       it_behaves_like 'creates repository bundles'
     end
 
-    context 'no concurrency' do
-      it 'creates the expected number of threads' do
-        expect(Thread).not_to receive(:new)
+    context 'command failure' do
+      it 'enqueue_project raises an error' do
+        allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError)
 
-        expect(strategy).to receive(:start).with(:create)
-        projects.each do |project|
-          expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
-        end
-        expect(strategy).to receive(:wait)
-
-        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
-      end
-
-      describe 'command failure' do
-        it 'enqueue_project raises an error' do
-          allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError)
-
-          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(IOError)
-        end
-
-        it 'project query raises an error' do
-          allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
-
-          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(ActiveRecord::StatementTimeout)
-        end
-      end
-
-      it 'avoids N+1 database queries' do
-        control_count = ActiveRecord::QueryRecorder.new do
-          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
-        end.count
-
-        create_list(:project, 2, :repository)
-
-        expect do
-          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
-        end.not_to exceed_query_limit(control_count)
+        expect { subject.dump }.to raise_error(IOError)
       end
-    end
 
-    context 'concurrency with a strategy without parallel enqueueing support' do
-      let(:parallel_enqueue) { false }
+      it 'project query raises an error' do
+        allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
 
-      it 'enqueues all projects sequentially' do
-        expect(Thread).not_to receive(:new)
-
-        expect(strategy).to receive(:start).with(:create)
-        projects.each do |project|
-          expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
-        end
-        expect(strategy).to receive(:wait)
-
-        subject.dump(max_concurrency: 2, max_storage_concurrency: 2)
+        expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
       end
     end
 
-    [4, 10].each do |max_storage_concurrency|
-      context "max_storage_concurrency #{max_storage_concurrency}", quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/241701' do
-        let(:storage_keys) { %w[default test_second_storage] }
-
-        before do
-          allow(Gitlab.config.repositories.storages).to receive(:keys).and_return(storage_keys)
-        end
+    it 'avoids N+1 database queries' do
+      control_count = ActiveRecord::QueryRecorder.new do
+        subject.dump
+      end.count
 
-        it 'creates the expected number of threads' do
-          expect(Thread).to receive(:new)
-            .exactly(storage_keys.length * (max_storage_concurrency + 1)).times
-            .and_call_original
+      create_list(:project, 2, :repository)
 
-          expect(strategy).to receive(:start).with(:create)
-          projects.each do |project|
-            expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
-          end
-          expect(strategy).to receive(:wait)
-
-          subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
-        end
-
-        it 'creates the expected number of threads with extra max concurrency' do
-          expect(Thread).to receive(:new)
-            .exactly(storage_keys.length * (max_storage_concurrency + 1)).times
-            .and_call_original
-
-          expect(strategy).to receive(:start).with(:create)
-          projects.each do |project|
-            expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
-          end
-          expect(strategy).to receive(:wait)
-
-          subject.dump(max_concurrency: 3, max_storage_concurrency: max_storage_concurrency)
-        end
-
-        describe 'command failure' do
-          it 'enqueue_project raises an error' do
-            allow(strategy).to receive(:enqueue).and_raise(IOError)
-
-            expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(IOError)
-          end
-
-          it 'project query raises an error' do
-            allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
-
-            expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(ActiveRecord::StatementTimeout)
-          end
-
-          context 'misconfigured storages' do
-            let(:storage_keys) { %w[test_second_storage] }
-
-            it 'raises an error' do
-              expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
-            end
-          end
-        end
-
-        it 'avoids N+1 database queries' do
-          control_count = ActiveRecord::QueryRecorder.new do
-            subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
-          end.count
-
-          create_list(:project, 2, :repository)
-
-          expect do
-            subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
-          end.not_to exceed_query_limit(control_count)
-        end
-      end
+      expect do
+        subject.dump
+      end.not_to exceed_query_limit(control_count)
     end
   end
 
diff --git a/spec/lib/gitlab/patch/legacy_database_config_spec.rb b/spec/lib/gitlab/patch/legacy_database_config_spec.rb
new file mode 100644
index 00000000000..e6c0bdbf360
--- /dev/null
+++ b/spec/lib/gitlab/patch/legacy_database_config_spec.rb
@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Patch::LegacyDatabaseConfig do
+  it 'module is included' do
+    expect(Rails::Application::Configuration).to include(described_class)
+  end
+
+  describe 'config/database.yml' do
+    let(:configuration) { Rails::Application::Configuration.new(Rails.root) }
+
+    before do
+      # The `AS::ConfigurationFile` calls `read` in `def initialize`
+      # thus we cannot use `expect_next_instance_of`
+      # rubocop:disable RSpec/AnyInstanceOf
+      expect_any_instance_of(ActiveSupport::ConfigurationFile)
+        .to receive(:read).with(Rails.root.join('config/database.yml')).and_return(database_yml)
+      # rubocop:enable RSpec/AnyInstanceOf
+    end
+
+    shared_examples 'hash containing main: connection name' do
+      it 'returns a hash containing only main:' do
+        database_configuration = configuration.database_configuration
+
+        expect(database_configuration).to match(
+          "production" => { "main" => a_hash_including("adapter") },
+          "development" => { "main" => a_hash_including("adapter" => "postgresql") },
+          "test" => { "main" => a_hash_including("adapter" => "postgresql") }
+        )
+      end
+    end
+
+    context 'when a new syntax is used' do
+      let(:database_yml) do
+        <<-EOS
+          production:
+            main:
+              adapter: postgresql
+              encoding: unicode
+              database: gitlabhq_production
+              username: git
+              password: "secure password"
+              host: localhost
+
+          development:
+            main:
+              adapter: postgresql
+              encoding: unicode
+              database: gitlabhq_development
+              username: postgres
+              password: "secure password"
+              host: localhost
+              variables:
+                statement_timeout: 15s
+
+          test: &test
+            main:
+              adapter: postgresql
+              encoding: unicode
+              database: gitlabhq_test
+              username: postgres
+              password:
+              host: localhost
+              prepared_statements: false
+              variables:
+                statement_timeout: 15s
+        EOS
+      end
+
+      include_examples 'hash containing main: connection name'
+
+      it 'configuration is not legacy one' do
+        configuration.database_configuration
+
+        expect(configuration.uses_legacy_database_config).to eq(false)
+      end
+    end
+
+    context 'when a legacy syntax is used' do
+      let(:database_yml) do
+        <<-EOS
+          production:
+            adapter: postgresql
+            encoding: unicode
+            database: gitlabhq_production
+            username: git
+            password: "secure password"
+            host: localhost
+
+          development:
+            adapter: postgresql
+            encoding: unicode
+            database: gitlabhq_development
+            username: postgres
+            password: "secure password"
+            host: localhost
+            variables:
+              statement_timeout: 15s
+
+          test: &test
+            adapter: postgresql
+            encoding: unicode
+            database: gitlabhq_test
+            username: postgres
+            password:
+            host: localhost
+            prepared_statements: false
+            variables:
+              statement_timeout: 15s
+        EOS
+      end
+
+      include_examples 'hash containing main: connection name'
+
+      it 'configuration is legacy' do
+        configuration.database_configuration
+
+        expect(configuration.uses_legacy_database_config).to eq(true)
+      end
+    end
+  end
+end