From 265ad515c697b9d50c4fd3e4bb360689a369a190 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Sun, 20 Sep 2015 23:20:51 +0200 Subject: Create CI migration task on GitLab side --- lib/ci/migrate/builds.rb | 29 +++++++++++++++++++ lib/ci/migrate/database.rb | 42 +++++++++++++-------------- lib/ci/migrate/manager.rb | 72 ++++++++++++++++++++++++++++++++++++++++++++++ lib/tasks/ci/migrate.rake | 49 +++++++++++++++++++------------ 4 files changed, 152 insertions(+), 40 deletions(-) create mode 100644 lib/ci/migrate/builds.rb create mode 100644 lib/ci/migrate/manager.rb (limited to 'lib') diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb new file mode 100644 index 00000000000..fdc143cfad5 --- /dev/null +++ b/lib/ci/migrate/builds.rb @@ -0,0 +1,29 @@ +module Ci + module Migrate + class Builds + attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir + + def initialize + @app_builds_dir = Settings.gitlab_ci.builds_path + @backup_dir = Gitlab.config.backup.path + @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz') + end + + def restore + backup_existing_builds_dir + + FileUtils.mkdir_p(app_builds_dir, mode: 0700) + unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball) + abort 'Restore failed'.red + end + end + + def backup_existing_builds_dir + timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") + if File.exists?(app_builds_dir) + FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) + end + end + end + end +end diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb index 74f592dcaea..bf9b80f1f62 100644 --- a/lib/ci/migrate/database.rb +++ b/lib/ci/migrate/database.rb @@ -9,32 +9,32 @@ module Ci @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] end - def restore(ci_dump) - puts 'Deleting all CI related data ... ' - truncate_ci_tables + def restore + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close - puts 'Restoring CI data ... ' - case config["adapter"] - when /^mysql/ then - print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysql', *mysql_args, config['database'], in: ci_dump) - when "postgresql" then - puts "Restoring PostgreSQL database #{config['database']} ... " - pg_env - system('psql', config['database'], '-f', ci_dump) - end + restore_pid = case config["adapter"] + when /^mysql/ then + $progress.print "Restoring MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) + when "postgresql" then + $progress.print "Restoring PostgreSQL database #{config['database']} ... " + pg_env + spawn('psql', config['database'], in: decompress_rd) + end + decompress_rd.close + + success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + abort 'Restore failed' unless success end protected - def truncate_ci_tables - c = ActiveRecord::Base.connection - c.tables.select { |t| t.start_with?('ci_') }.each do |table| - puts "Deleting data from #{table}..." - c.execute("DELETE FROM #{table}") - end + def db_file_name + File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end def mysql_args diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb new file mode 100644 index 00000000000..9405397031c --- /dev/null +++ b/lib/ci/migrate/manager.rb @@ -0,0 +1,72 @@ +module Ci + module Migrate + class Manager + VERSION = '8.0.0.pre' + + def cleanup + $progress.print "Deleting tmp directories ... " + + backup_contents.each do |dir| + next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + + if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) + $progress.puts "done".green + else + puts "deleting tmp directory '#{dir}' failed".red + abort 'Backup failed' + end + end + end + + def unpack + Dir.chdir(Gitlab.config.backup.path) + + # check for existing backups in the backup dir + file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i } + puts "no backups found" if file_list.count == 0 + + if file_list.count > 1 && ENV["BACKUP"].nil? + puts "Found more than one backup, please specify which one you want to restore:" + puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" + exit 1 + end + + tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar") + + unless File.exists?(tar_file) + puts "The specified CI backup doesn't exist!" + exit 1 + end + + $progress.print "Unpacking backup ... " + + unless Kernel.system(*%W(tar -xf #{tar_file})) + puts "unpacking backup failed".red + exit 1 + else + $progress.puts "done".green + end + + ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 + + # restoring mismatching backups can lead to unexpected problems + if settings[:gitlab_version] != VERSION + puts "GitLab CI version mismatch:".red + puts " Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red + exit 1 + end + end + + private + + def backup_contents + ["db", "builds", "backup_information.yml"] + end + + def settings + @settings ||= YAML.load_file("backup_information.yml") + end + end + end +end + diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake index e7d41874a11..3507871fcb0 100644 --- a/lib/tasks/ci/migrate.rake +++ b/lib/tasks/ci/migrate.rake @@ -1,40 +1,49 @@ namespace :ci do desc 'GitLab | Import and migrate CI database' task migrate: :environment do + warn_user_is_not_gitlab + configure_cron_mode + unless ENV['force'] == 'yes' - puts "This will truncate all CI tables and restore it from provided backup." - puts "You will lose any previous CI data stored in the database." + puts 'This will remove all CI related data and restore it from the provided backup.' ask_to_continue - puts "" + puts '' end - Rake::Task["ci:migrate:db"].invoke - Rake::Task["ci:migrate:autoincrements"].invoke - Rake::Task["ci:migrate:tags"].invoke - Rake::Task["ci:migrate:services"].invoke + migrate = Ci::Migrate::Manager.new + migrate.unpack + + Rake::Task['ci:migrate:db'].invoke + Rake::Task['ci:migrate:builds'].invoke + Rake::Task['ci:migrate:tags'].invoke + Rake::Task['ci:migrate:services'].invoke + + migrate.cleanup end namespace :migrate do desc 'GitLab | Import CI database' task db: :environment do - if ENV["CI_DUMP"].nil? - puts "No CI SQL dump specified:" - puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql" - exit 1 - end - - ci_dump = ENV["CI_DUMP"] - unless File.exists?(ci_dump) - puts "The specified sql dump doesn't exist!" - exit 1 - end + configure_cron_mode + $progress.puts 'Restoring database ... '.blue + Ci::Migrate::Database.new.restore + $progress.puts 'done'.green + end - ::Ci::Migrate::Database.new.restore(ci_dump) + desc 'GitLab | Import CI builds' + task builds: :environment do + configure_cron_mode + $progress.puts 'Restoring builds ... '.blue + Ci::Migrate::Builds.new.restore + $progress.puts 'done'.green end desc 'GitLab | Migrate CI tags' task tags: :environment do + configure_cron_mode + $progress.puts 'Migrating tags ... '.blue ::Ci::Migrate::Tags.new.restore + $progress.puts 'done'.green end desc 'GitLab | Migrate CI auto-increments' @@ -56,8 +65,10 @@ namespace :ci do desc 'GitLab | Migrate CI services' task services: :environment do + $progress.puts 'Migrating services ... '.blue c = ActiveRecord::Base.connection c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") + $progress.puts 'done'.green end end end -- cgit v1.2.3