Cleanup CI backup => migrate with GitLab

7 files changed, 55 insertions, 346 deletions

diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
new file mode 100644
index 00000000000..4280438e86c
--- /dev/null
+++ b/lib/backup/builds.rb
@@ -0,0 +1,30 @@
+module Backup
+  class Builds
+    attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
+
+    def initialize
+      @app_builds_dir = File.realpath(Rails.root.join('ci/builds'))
+      @backup_dir = GitlabCi.config.backup.path
+      @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds')
+    end
+
+    # Copy builds from builds directory to backup/builds
+    def dump
+      FileUtils.mkdir_p(backup_builds_dir)
+      FileUtils.cp_r(app_builds_dir, backup_dir)
+    end
+
+    def restore
+      backup_existing_builds_dir
+
+      FileUtils.cp_r(backup_builds_dir, app_builds_dir)
+    end
+
+    def backup_existing_builds_dir
+      timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+      if File.exists?(app_builds_dir)
+        FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+      end
+    end
+  end
+end
diff --git a/lib/ci/backup/builds.rb b/lib/ci/backup/builds.rb
deleted file mode 100644
index 832a5ab8fdc..00000000000
--- a/lib/ci/backup/builds.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-module Ci
-  module Backup
-    class Builds
-      attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
-
-      def initialize
-        @app_builds_dir = File.realpath(Rails.root.join('ci/builds'))
-        @backup_dir = GitlabCi.config.backup.path
-        @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds')
-      end
-
-      # Copy builds from builds directory to backup/builds
-      def dump
-        FileUtils.mkdir_p(backup_builds_dir)
-        FileUtils.cp_r(app_builds_dir, backup_dir)
-      end
-
-      def restore
-        backup_existing_builds_dir
-
-        FileUtils.cp_r(backup_builds_dir, app_builds_dir)
-      end
-
-      def backup_existing_builds_dir
-        timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
-        if File.exists?(app_builds_dir)
-          FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
-        end
-      end
-    end
-  end
-end
diff --git a/lib/ci/backup/database.rb b/lib/ci/backup/database.rb
deleted file mode 100644
index 3f2277024e4..00000000000
--- a/lib/ci/backup/database.rb
+++ /dev/null
@@ -1,94 +0,0 @@
-require 'yaml'
-
-module Ci
-  module Backup
-    class Database
-      attr_reader :config, :db_dir
-
-      def initialize
-        @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
-        @db_dir = File.join(GitlabCi.config.backup.path, 'db')
-        FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)
-      end
-
-      def dump
-        success = case config["adapter"]
-                  when /^mysql/ then
-                    $progress.print "Dumping MySQL database #{config['database']} ... "
-                    system('mysqldump', *mysql_args, config['database'], out: db_file_name)
-                  when "postgresql" then
-                    $progress.print "Dumping PostgreSQL database #{config['database']} ... "
-                    pg_env
-                    system('pg_dump', config['database'], out: db_file_name)
-        end
-        report_success(success)
-        abort 'Backup failed' unless success
-      end
-
-      def restore
-        success = case config["adapter"]
-                  when /^mysql/ then
-                    $progress.print "Restoring MySQL database #{config['database']} ... "
-                    system('mysql', *mysql_args, config['database'], in: db_file_name)
-                  when "postgresql" then
-                    $progress.print "Restoring PostgreSQL database #{config['database']} ... "
-                    # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE
-                    # statements like MySQL.
-                    drop_all_tables
-                    drop_all_postgres_sequences
-                    pg_env
-                    system('psql', config['database'], '-f', db_file_name)
-        end
-        report_success(success)
-        abort 'Restore failed' unless success
-      end
-
-      protected
-
-      def db_file_name
-        File.join(db_dir, 'database.sql')
-      end
-
-      def mysql_args
-        args = {
-          'host'      => '--host',
-          'port'      => '--port',
-          'socket'    => '--socket',
-          'username'  => '--user',
-          'encoding'  => '--default-character-set',
-          'password'  => '--password'
-        }
-        args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
-      end
-
-      def pg_env
-        ENV['PGUSER']     = config["username"] if config["username"]
-        ENV['PGHOST']     = config["host"] if config["host"]
-        ENV['PGPORT']     = config["port"].to_s if config["port"]
-        ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
-      end
-
-      def report_success(success)
-        if success
-          $progress.puts '[DONE]'.green
-        else
-          $progress.puts '[FAILED]'.red
-        end
-      end
-
-      def drop_all_tables
-        connection = ActiveRecord::Base.connection
-        connection.tables.each do |table|
-          connection.drop_table(table)
-        end
-      end
-
-      def drop_all_postgres_sequences
-        connection = ActiveRecord::Base.connection
-        connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence|
-          connection.execute("DROP SEQUENCE #{sequence['relname']}")
-        end
-      end
-    end
-  end
-end
diff --git a/lib/ci/backup/manager.rb b/lib/ci/backup/manager.rb
deleted file mode 100644
index 2e9d6df7139..00000000000
--- a/lib/ci/backup/manager.rb
+++ /dev/null
@@ -1,158 +0,0 @@
-module Ci
-  module Backup
-    class Manager
-      def pack
-        # saving additional informations
-        s = {}
-        s[:db_version]         = "#{ActiveRecord::Migrator.current_version}"
-        s[:backup_created_at]  = Time.now
-        s[:gitlab_version]     = GitlabCi::VERSION
-        s[:tar_version]        = tar_version
-        tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz"
-
-        Dir.chdir(GitlabCi.config.backup.path) do
-          File.open("#{GitlabCi.config.backup.path}/backup_information.yml",
-                    "w+") do |file|
-            file << s.to_yaml.gsub(/^---\n/,'')
-          end
-
-          FileUtils.chmod(0700, ["db", "builds"])
-
-          # create archive
-          $progress.print "Creating backup archive: #{tar_file} ... "
-          orig_umask = File.umask(0077)
-          if Kernel.system('tar', '-czf', tar_file, *backup_contents)
-            $progress.puts "done".green
-          else
-            puts "creating archive #{tar_file} failed".red
-            abort 'Backup failed'
-          end
-          File.umask(orig_umask)
-
-          upload(tar_file)
-        end
-      end
-
-      def upload(tar_file)
-        remote_directory = GitlabCi.config.backup.upload.remote_directory
-        $progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
-
-        connection_settings = GitlabCi.config.backup.upload.connection
-        if connection_settings.blank?
-          $progress.puts "skipped".yellow
-          return
-        end
-
-        connection = ::Fog::Storage.new(connection_settings)
-        directory = connection.directories.get(remote_directory)
-
-        if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
-            multipart_chunk_size: GitlabCi.config.backup.upload.multipart_chunk_size)
-          $progress.puts "done".green
-        else
-          puts "uploading backup to #{remote_directory} failed".red
-          abort 'Backup failed'
-        end
-      end
-
-      def cleanup
-        $progress.print "Deleting tmp directories ... "
-        
-        backup_contents.each do |dir|
-          next unless File.exist?(File.join(GitlabCi.config.backup.path, dir))
-
-          if FileUtils.rm_rf(File.join(GitlabCi.config.backup.path, dir))
-            $progress.puts "done".green
-          else
-            puts "deleting tmp directory '#{dir}' failed".red
-            abort 'Backup failed'
-          end
-        end
-      end
-
-      def remove_old
-        # delete backups
-        $progress.print "Deleting old backups ... "
-        keep_time = GitlabCi.config.backup.keep_time.to_i
-
-        if keep_time > 0
-          removed = 0
-          
-          Dir.chdir(GitlabCi.config.backup.path) do
-            file_list = Dir.glob('*_gitlab_ci_backup.tar.gz')
-            file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ }
-            file_list.sort.each do |timestamp|
-              if Time.at(timestamp) < (Time.now - keep_time)
-                if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz))
-                  removed += 1
-                end
-              end
-            end
-          end
-
-          $progress.puts "done. (#{removed} removed)".green
-        else
-          $progress.puts "skipping".yellow
-        end
-      end
-
-      def unpack
-        Dir.chdir(GitlabCi.config.backup.path)
-
-        # check for existing backups in the backup dir
-        file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i }
-        puts "no backups found" if file_list.count == 0
-
-        if file_list.count > 1 && ENV["BACKUP"].nil?
-          puts "Found more than one backup, please specify which one you want to restore:"
-          puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
-          exit 1
-        end
-
-        tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz")
-
-        unless File.exists?(tar_file)
-          puts "The specified backup doesn't exist!"
-          exit 1
-        end
-
-        $progress.print "Unpacking backup ... "
-
-        unless Kernel.system(*%W(tar -xzf #{tar_file}))
-          puts "unpacking backup failed".red
-          exit 1
-        else
-          $progress.puts "done".green
-        end
-
-        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
-
-        # restoring mismatching backups can lead to unexpected problems
-        if settings[:gitlab_version] != GitlabCi::VERSION
-          puts "GitLab CI version mismatch:".red
-          puts "  Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI version in the backup!".red
-          puts "  Please switch to the following version and try again:".red
-          puts "  version: #{settings[:gitlab_version]}".red
-          puts
-          puts "Hint: git checkout v#{settings[:gitlab_version]}"
-          exit 1
-        end
-      end
-
-      def tar_version
-        tar_version = `tar --version`
-        tar_version.force_encoding('locale').split("\n").first
-      end
-
-      private
-
-      def backup_contents
-        ["db", "builds", "backup_information.yml"]
-      end
-
-      def settings
-        @settings ||= YAML.load_file("backup_information.yml")
-      end
-    end
-  end
-end
diff --git a/lib/tasks/ci/backup.rake b/lib/tasks/ci/backup.rake
deleted file mode 100644
index 1cb2e43f875..00000000000
--- a/lib/tasks/ci/backup.rake
+++ /dev/null
@@ -1,62 +0,0 @@
-namespace :ci do
-  namespace :backup do
-    
-    desc "GITLAB | Create a backup of the GitLab CI database"
-    task create: :environment do
-      configure_cron_mode
-
-      $progress.puts "Dumping database ... ".blue
-      Ci::Backup::Database.new.dump
-      $progress.puts "done".green
-
-      $progress.puts "Dumping builds ... ".blue
-      Ci::Backup::Builds.new.dump
-      $progress.puts "done".green
-
-      backup = Ci::Backup::Manager.new
-      backup.pack
-      backup.cleanup
-      backup.remove_old
-    end
-
-    desc "GITLAB | Restore a previously created backup"
-    task restore: :environment do
-      configure_cron_mode
-
-      backup = Ci::Backup::Manager.new
-      backup.unpack
-
-      $progress.puts "Restoring database ... ".blue
-      Ci::Backup::Database.new.restore
-      $progress.puts "done".green
-
-      $progress.puts "Restoring builds ... ".blue
-      Ci::Backup::Builds.new.restore
-      $progress.puts "done".green
-
-      backup.cleanup
-    end
-
-    def configure_cron_mode
-      if ENV['CRON']
-        # We need an object we can say 'puts' and 'print' to; let's use a
-        # StringIO.
-        require 'stringio'
-        $progress = StringIO.new
-      else
-        $progress = $stdout
-      end
-    end
-  end
-
-  # Disable colors for CRON
-  unless STDOUT.isatty
-    module Colored
-      extend self
-
-      def colorize(string, options={})
-        string
-      end
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 4c73f90bbf2..f20c7f71ba5 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -11,6 +11,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:db:create"].invoke
       Rake::Task["gitlab:backup:repo:create"].invoke
       Rake::Task["gitlab:backup:uploads:create"].invoke
+      Rake::Task["gitlab:backup:builds:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -30,6 +31,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db")
       Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
+      Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -73,6 +75,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :builds do
+      task create: :environment do
+        $progress.puts "Dumping builds ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("builds")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Builds.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring builds ... ".blue
+        Backup::Builds.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     namespace :uploads do
       task create: :environment do
         $progress.puts "Dumping uploads ... ".blue
diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 23f322e0a62..32adcbec71f 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -54,6 +54,7 @@ describe 'gitlab:app namespace rake task' do
           and_return({ gitlab_version: gitlab_version })
         expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke)
+        expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
         expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
         expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
       end
@@ -117,6 +118,7 @@ describe 'gitlab:app namespace rake task' do
       expect(tar_contents).to match('db/')
       expect(tar_contents).to match('uploads/')
       expect(tar_contents).to match('repositories/')
+      expect(tar_contents).to match('builds/')
       expect(tar_contents).not_to match(/^.{4,9}[rwx].* (db|uploads|repositories)\/$/)
     end
 
@@ -163,6 +165,7 @@ describe 'gitlab:app namespace rake task' do
 
       expect(tar_contents).to match('db/')
       expect(tar_contents).to match('uploads/')
+      expect(tar_contents).to match('builds/')
       expect(tar_contents).not_to match('repositories/')
     end
 
@@ -173,6 +176,7 @@ describe 'gitlab:app namespace rake task' do
 
       expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke
       expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke
+      expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke
       expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
       expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
     end