Add latest changes from gitlab-org/gitlab@13-4-stable-ee

7 files changed, 232 insertions, 2 deletions

diff --git a/spec/lib/backup/artifacts_spec.rb b/spec/lib/backup/artifacts_spec.rb
new file mode 100644
index 00000000000..2a3f1949ba5
--- /dev/null
+++ b/spec/lib/backup/artifacts_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Artifacts do
+  let(:progress) { StringIO.new }
+
+  subject(:backup) { described_class.new(progress) }
+
+  describe '#initialize' do
+    it 'uses the correct upload dir' do
+      Dir.mktmpdir do |tmpdir|
+        allow(JobArtifactUploader).to receive(:root) { "#{tmpdir}" }
+
+        expect(backup.app_files_dir).to eq("#{tmpdir}")
+      end
+    end
+  end
+
+  describe '#dump' do
+    before do
+      allow(File).to receive(:realpath).with('/var/gitlab-artifacts').and_return('/var/gitlab-artifacts')
+      allow(File).to receive(:realpath).with('/var/gitlab-artifacts/..').and_return('/var')
+      allow(JobArtifactUploader).to receive(:root) { '/var/gitlab-artifacts' }
+    end
+
+    it 'uses the correct artifact dir' do
+      expect(backup.app_files_dir).to eq('/var/gitlab-artifacts')
+    end
+
+    it 'excludes tmp from backup tar' do
+      expect(backup).to receive(:tar).and_return('blabla-tar')
+      expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/gitlab-artifacts -cf - .), 'gzip -c -1'], any_args)
+      backup.dump
+    end
+  end
+end
diff --git a/spec/lib/backup/database_spec.rb b/spec/lib/backup/database_spec.rb
new file mode 100644
index 00000000000..fccd6db0018
--- /dev/null
+++ b/spec/lib/backup/database_spec.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Database do
+  let(:progress) { StringIO.new }
+  let(:output) { progress.string }
+
+  describe '#restore' do
+    let(:cmd) { %W[#{Gem.ruby} -e $stdout.puts(1)] }
+    let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
+
+    subject { described_class.new(progress, filename: data) }
+
+    before do
+      allow(subject).to receive(:pg_restore_cmd).and_return(cmd)
+    end
+
+    context 'with an empty .gz file' do
+      let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
+
+      it 'returns successfully' do
+        expect(subject.restore).to eq([])
+
+        expect(output).to include("Restoring PostgreSQL database")
+        expect(output).to include("[DONE]")
+        expect(output).not_to include("ERRORS")
+      end
+    end
+
+    context 'with a corrupted .gz file' do
+      let(:data) { Rails.root.join("spec/fixtures/big-image.png").to_s }
+
+      it 'raises a backup error' do
+        expect { subject.restore }.to raise_error(Backup::Error)
+      end
+    end
+
+    context 'when the restore command prints errors' do
+      let(:visible_error) { "This is a test error\n" }
+      let(:noise) { "Table projects does not exist\nmust be owner of extension pg_trgm\n" }
+      let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] }
+
+      it 'filters out noise from errors' do
+        expect(subject.restore).to eq([visible_error])
+        expect(output).to include("ERRORS")
+        expect(output).not_to include(noise)
+        expect(output).to include(visible_error)
+      end
+    end
+  end
+end
diff --git a/spec/lib/backup/files_spec.rb b/spec/lib/backup/files_spec.rb
index a7374b82ce0..c2dbaac7f15 100644
--- a/spec/lib/backup/files_spec.rb
+++ b/spec/lib/backup/files_spec.rb
@@ -14,6 +14,8 @@ RSpec.describe Backup::Files do
     allow(File).to receive(:exist?).and_return(true)
     allow(File).to receive(:realpath).with("/var/gitlab-registry").and_return("/var/gitlab-registry")
     allow(File).to receive(:realpath).with("/var/gitlab-registry/..").and_return("/var")
+    allow(File).to receive(:realpath).with("/var/gitlab-pages").and_return("/var/gitlab-pages")
+    allow(File).to receive(:realpath).with("/var/gitlab-pages/..").and_return("/var")
 
     allow_any_instance_of(String).to receive(:color) do |string, _color|
       string
@@ -82,4 +84,48 @@ RSpec.describe Backup::Files do
       end
     end
   end
+
+  describe '#dump' do
+    subject { described_class.new('pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
+
+    before do
+      allow(subject).to receive(:run_pipeline!).and_return(true)
+    end
+
+    it 'raises no errors' do
+      expect { subject.dump }.not_to raise_error
+    end
+
+    it 'excludes tmp dirs from archive' do
+      expect(subject).to receive(:tar).and_return('blabla-tar')
+
+      expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args)
+      subject.dump
+    end
+
+    describe 'with STRATEGY=copy' do
+      before do
+        stub_env('STRATEGY', 'copy')
+      end
+
+      it 'excludes tmp dirs from rsync' do
+        allow(Gitlab.config.backup).to receive(:path) { '/var/gitlab-backup' }
+        allow(File).to receive(:realpath).with("/var/gitlab-backup").and_return("/var/gitlab-backup")
+
+        expect(Gitlab::Popen).to receive(:popen).with(%w(rsync -a --exclude=lost+found --exclude=/@pages.tmp /var/gitlab-pages /var/gitlab-backup)).and_return(['', 0])
+
+        subject.dump
+      end
+    end
+
+    describe '#exclude_dirs' do
+      it 'prepends a leading dot slash to tar excludes' do
+        expect(subject.exclude_dirs(:tar)).to eq(['--exclude=lost+found', '--exclude=./@pages.tmp'])
+      end
+
+      it 'prepends a leading slash to rsync excludes' do
+        expect(subject.exclude_dirs(:rsync)).to eq(['--exclude=lost+found', '--exclude=/@pages.tmp'])
+      end
+    end
+  end
 end
diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb
index 38a5c30506b..feaca6164eb 100644
--- a/spec/lib/backup/manager_spec.rb
+++ b/spec/lib/backup/manager_spec.rb
@@ -416,5 +416,28 @@ RSpec.describe Backup::Manager do
         subject.upload
       end
     end
+
+    context 'with AzureRM provider' do
+      before do
+        stub_backup_setting(
+          upload: {
+            connection: {
+              provider: 'AzureRM',
+              azure_storage_account_name: 'test-access-id',
+              azure_storage_access_key: 'secret'
+            },
+            remote_directory: 'directory',
+            multipart_chunk_size: nil,
+            encryption: nil,
+            encryption_key: nil,
+            storage_class: nil
+          }
+        )
+      end
+
+      it 'loads the provider' do
+        expect { subject.upload }.not_to raise_error
+      end
+    end
   end
 end
diff --git a/spec/lib/backup/pages_spec.rb b/spec/lib/backup/pages_spec.rb
new file mode 100644
index 00000000000..59df4d1adf7
--- /dev/null
+++ b/spec/lib/backup/pages_spec.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Pages do
+  let(:progress) { StringIO.new }
+
+  subject { described_class.new(progress) }
+
+  before do
+    allow(File).to receive(:realpath).with("/var/gitlab-pages").and_return("/var/gitlab-pages")
+    allow(File).to receive(:realpath).with("/var/gitlab-pages/..").and_return("/var")
+  end
+
+  describe '#dump' do
+    it 'uses the correct pages dir' do
+      allow(Gitlab.config.pages).to receive(:path) { '/var/gitlab-pages' }
+
+      expect(subject.app_files_dir).to eq('/var/gitlab-pages')
+    end
+
+    it 'excludes tmp from backup tar' do
+      allow(Gitlab.config.pages).to receive(:path) { '/var/gitlab-pages' }
+
+      expect(subject).to receive(:tar).and_return('blabla-tar')
+      expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args)
+      subject.dump
+    end
+  end
+end
diff --git a/spec/lib/backup/repository_spec.rb b/spec/lib/backup/repository_spec.rb
index c4ad239f9d7..718f38f9452 100644
--- a/spec/lib/backup/repository_spec.rb
+++ b/spec/lib/backup/repository_spec.rb
@@ -47,11 +47,23 @@ RSpec.describe Backup::Repository do
         end
 
         it 'project query raises an error' do
-          allow(Project).to receive(:find_each).and_raise(ActiveRecord::StatementTimeout)
+          allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
 
           expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(ActiveRecord::StatementTimeout)
         end
       end
+
+      it 'avoids N+1 database queries' do
+        control_count = ActiveRecord::QueryRecorder.new do
+          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        end.count
+
+        create_list(:project, 2, :wiki_repo)
+
+        expect do
+          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        end.not_to exceed_query_limit(control_count)
+      end
     end
 
     [4, 10].each do |max_storage_concurrency|
@@ -89,7 +101,7 @@ RSpec.describe Backup::Repository do
           end
 
           it 'project query raises an error' do
-            allow(Project).to receive_message_chain('for_repository_storage.find_each').and_raise(ActiveRecord::StatementTimeout)
+            allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
 
             expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(ActiveRecord::StatementTimeout)
           end
@@ -102,6 +114,18 @@ RSpec.describe Backup::Repository do
             end
           end
         end
+
+        it 'avoids N+1 database queries' do
+          control_count = ActiveRecord::QueryRecorder.new do
+            subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
+          end.count
+
+          create_list(:project, 2, :wiki_repo)
+
+          expect do
+            subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
+          end.not_to exceed_query_limit(control_count)
+        end
       end
     end
   end
diff --git a/spec/lib/backup/uploads_spec.rb b/spec/lib/backup/uploads_spec.rb
index 7c2d715b580..678b670db34 100644
--- a/spec/lib/backup/uploads_spec.rb
+++ b/spec/lib/backup/uploads_spec.rb
@@ -18,4 +18,22 @@ RSpec.describe Backup::Uploads do
       end
     end
   end
+
+  describe '#dump' do
+    before do
+      allow(File).to receive(:realpath).with('/var/uploads').and_return('/var/uploads')
+      allow(File).to receive(:realpath).with('/var/uploads/..').and_return('/var')
+      allow(Gitlab.config.uploads).to receive(:storage_path) { '/var' }
+    end
+
+    it 'uses the correct upload dir' do
+      expect(backup.app_files_dir).to eq('/var/uploads')
+    end
+
+    it 'excludes tmp from backup tar' do
+      expect(backup).to receive(:tar).and_return('blabla-tar')
+      expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/uploads -cf - .), 'gzip -c -1'], any_args)
+      backup.dump
+    end
+  end
 end