Merge branch 'backup-improvements' into 'master'

Reduce disk IO and space usage during backups

This is based on improvements made to the GitLab CI 8.0 backup script.

- Avoid creating many small intermediate files while backing up builds and uploads by using tar and light gzip compression
- Use same backup/restore code for uploads and builds
- Only store a compressed intermediate DB dump

See merge request !1520

1 files changed, 8 insertions, 7 deletions

diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 3be7dd4e52b..386ac9c8372 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -55,6 +55,7 @@ describe 'gitlab:app namespace rake task' do
         expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
+        expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
         expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
       end
@@ -112,14 +113,14 @@ describe 'gitlab:app namespace rake task' do
 
     it 'should set correct permissions on the tar contents' do
       tar_contents, exit_status = Gitlab::Popen.popen(
-        %W{tar -tvf #{@backup_tar} db uploads repositories builds}
+        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz}
       )
       expect(exit_status).to eq(0)
       expect(tar_contents).to match('db/')
-      expect(tar_contents).to match('uploads/')
+      expect(tar_contents).to match('uploads.tar.gz')
       expect(tar_contents).to match('repositories/')
-      expect(tar_contents).to match('builds/')
-      expect(tar_contents).not_to match(/^.{4,9}[rwx].* (db|uploads|repositories|builds)\/$/)
+      expect(tar_contents).to match('builds.tar.gz')
+      expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz)\/$/)
     end
 
     it 'should delete temp directories' do
@@ -160,12 +161,12 @@ describe 'gitlab:app namespace rake task' do
 
     it "does not contain skipped item" do
       tar_contents, _exit_status = Gitlab::Popen.popen(
-        %W{tar -tvf #{@backup_tar} db uploads repositories builds}
+        %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz}
       )
 
       expect(tar_contents).to match('db/')
-      expect(tar_contents).to match('uploads/')
-      expect(tar_contents).to match('builds/')
+      expect(tar_contents).to match('uploads.tar.gz')
+      expect(tar_contents).to match('builds.tar.gz')
       expect(tar_contents).not_to match('repositories/')
     end