From 45a105b127eb5ee4f9ebc99f3cf46d582de3601c Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Sun, 20 Sep 2015 22:12:45 +0200 Subject: Use pure SQL queries to migrate CI tags --- lib/ci/migrate/tags.rb | 58 ++++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb index 125a535e9a9..2bbd253f8de 100644 --- a/lib/ci/migrate/tags.rb +++ b/lib/ci/migrate/tags.rb @@ -4,45 +4,37 @@ module Ci module Migrate class Tags def restore - puts 'Migrating tags for Runners... ' - list_objects('Runner').each do |id| - putc '.' - runner = Ci::Runner.find_by_id(id) - if runner - tags = list_tags('Runner', id) - runner.update_attributes(tag_list: tags) - end - end - puts '' + ActiveRecord::Base.transaction do + puts 'Inserting tags...' + connection.execute( + 'INSERT INTO tags (name) ' + + 'SELECT ci_tags.name FROM ci_tags ' + + 'WHERE (SELECT COUNT(*) FROM tags WHERE tags.name = ci_tags.name)=0' + ) + + puts 'Deleting old records' + connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'" + + puts 'Inserting tags...' + connection.execute( + 'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' + + "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " + + 'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' + + 'JOIN tags ON tags.name = ci_tags.name ' + ) - puts 'Migrating tags for Builds... ' - list_objects('Build').each do |id| - putc '.' - build = Ci::Build.find_by_id(id) - if build - tags = list_tags('Build', id) - build.update_attributes(tag_list: tags) - end + puts 'Resetting counters... ' + connection.execute( + 'UPDATE tags SET ' + + 'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)' + ) end - puts '' end protected - def list_objects(type) - ids = ActiveRecord::Base.connection.select_all( - "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}" - ) - ids.map { |id| id['taggable_id'] } - end - - def list_tags(type, id) - tags = ActiveRecord::Base.connection.select_all( - 'select ci_tags.name from ci_tags ' + - 'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' + - "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'" - ) - tags.map { |tag| tag['name'] } + def connection + ActiveRecord::Base.connection end end end -- cgit v1.2.3 From 265ad515c697b9d50c4fd3e4bb360689a369a190 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Sun, 20 Sep 2015 23:20:51 +0200 Subject: Create CI migration task on GitLab side --- lib/ci/migrate/builds.rb | 29 +++++++++++++++++++ lib/ci/migrate/database.rb | 42 +++++++++++++-------------- lib/ci/migrate/manager.rb | 72 ++++++++++++++++++++++++++++++++++++++++++++++ lib/tasks/ci/migrate.rake | 49 +++++++++++++++++++------------ 4 files changed, 152 insertions(+), 40 deletions(-) create mode 100644 lib/ci/migrate/builds.rb create mode 100644 lib/ci/migrate/manager.rb (limited to 'lib') diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb new file mode 100644 index 00000000000..fdc143cfad5 --- /dev/null +++ b/lib/ci/migrate/builds.rb @@ -0,0 +1,29 @@ +module Ci + module Migrate + class Builds + attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir + + def initialize + @app_builds_dir = Settings.gitlab_ci.builds_path + @backup_dir = Gitlab.config.backup.path + @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz') + end + + def restore + backup_existing_builds_dir + + FileUtils.mkdir_p(app_builds_dir, mode: 0700) + unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball) + abort 'Restore failed'.red + end + end + + def backup_existing_builds_dir + timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") + if File.exists?(app_builds_dir) + FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) + end + end + end + end +end diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb index 74f592dcaea..bf9b80f1f62 100644 --- a/lib/ci/migrate/database.rb +++ b/lib/ci/migrate/database.rb @@ -9,32 +9,32 @@ module Ci @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] end - def restore(ci_dump) - puts 'Deleting all CI related data ... ' - truncate_ci_tables + def restore + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close - puts 'Restoring CI data ... ' - case config["adapter"] - when /^mysql/ then - print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysql', *mysql_args, config['database'], in: ci_dump) - when "postgresql" then - puts "Restoring PostgreSQL database #{config['database']} ... " - pg_env - system('psql', config['database'], '-f', ci_dump) - end + restore_pid = case config["adapter"] + when /^mysql/ then + $progress.print "Restoring MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) + when "postgresql" then + $progress.print "Restoring PostgreSQL database #{config['database']} ... " + pg_env + spawn('psql', config['database'], in: decompress_rd) + end + decompress_rd.close + + success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + abort 'Restore failed' unless success end protected - def truncate_ci_tables - c = ActiveRecord::Base.connection - c.tables.select { |t| t.start_with?('ci_') }.each do |table| - puts "Deleting data from #{table}..." - c.execute("DELETE FROM #{table}") - end + def db_file_name + File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end def mysql_args diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb new file mode 100644 index 00000000000..9405397031c --- /dev/null +++ b/lib/ci/migrate/manager.rb @@ -0,0 +1,72 @@ +module Ci + module Migrate + class Manager + VERSION = '8.0.0.pre' + + def cleanup + $progress.print "Deleting tmp directories ... " + + backup_contents.each do |dir| + next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + + if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) + $progress.puts "done".green + else + puts "deleting tmp directory '#{dir}' failed".red + abort 'Backup failed' + end + end + end + + def unpack + Dir.chdir(Gitlab.config.backup.path) + + # check for existing backups in the backup dir + file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i } + puts "no backups found" if file_list.count == 0 + + if file_list.count > 1 && ENV["BACKUP"].nil? + puts "Found more than one backup, please specify which one you want to restore:" + puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" + exit 1 + end + + tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar") + + unless File.exists?(tar_file) + puts "The specified CI backup doesn't exist!" + exit 1 + end + + $progress.print "Unpacking backup ... " + + unless Kernel.system(*%W(tar -xf #{tar_file})) + puts "unpacking backup failed".red + exit 1 + else + $progress.puts "done".green + end + + ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 + + # restoring mismatching backups can lead to unexpected problems + if settings[:gitlab_version] != VERSION + puts "GitLab CI version mismatch:".red + puts " Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red + exit 1 + end + end + + private + + def backup_contents + ["db", "builds", "backup_information.yml"] + end + + def settings + @settings ||= YAML.load_file("backup_information.yml") + end + end + end +end + diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake index e7d41874a11..3507871fcb0 100644 --- a/lib/tasks/ci/migrate.rake +++ b/lib/tasks/ci/migrate.rake @@ -1,40 +1,49 @@ namespace :ci do desc 'GitLab | Import and migrate CI database' task migrate: :environment do + warn_user_is_not_gitlab + configure_cron_mode + unless ENV['force'] == 'yes' - puts "This will truncate all CI tables and restore it from provided backup." - puts "You will lose any previous CI data stored in the database." + puts 'This will remove all CI related data and restore it from the provided backup.' ask_to_continue - puts "" + puts '' end - Rake::Task["ci:migrate:db"].invoke - Rake::Task["ci:migrate:autoincrements"].invoke - Rake::Task["ci:migrate:tags"].invoke - Rake::Task["ci:migrate:services"].invoke + migrate = Ci::Migrate::Manager.new + migrate.unpack + + Rake::Task['ci:migrate:db'].invoke + Rake::Task['ci:migrate:builds'].invoke + Rake::Task['ci:migrate:tags'].invoke + Rake::Task['ci:migrate:services'].invoke + + migrate.cleanup end namespace :migrate do desc 'GitLab | Import CI database' task db: :environment do - if ENV["CI_DUMP"].nil? - puts "No CI SQL dump specified:" - puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql" - exit 1 - end - - ci_dump = ENV["CI_DUMP"] - unless File.exists?(ci_dump) - puts "The specified sql dump doesn't exist!" - exit 1 - end + configure_cron_mode + $progress.puts 'Restoring database ... '.blue + Ci::Migrate::Database.new.restore + $progress.puts 'done'.green + end - ::Ci::Migrate::Database.new.restore(ci_dump) + desc 'GitLab | Import CI builds' + task builds: :environment do + configure_cron_mode + $progress.puts 'Restoring builds ... '.blue + Ci::Migrate::Builds.new.restore + $progress.puts 'done'.green end desc 'GitLab | Migrate CI tags' task tags: :environment do + configure_cron_mode + $progress.puts 'Migrating tags ... '.blue ::Ci::Migrate::Tags.new.restore + $progress.puts 'done'.green end desc 'GitLab | Migrate CI auto-increments' @@ -56,8 +65,10 @@ namespace :ci do desc 'GitLab | Migrate CI services' task services: :environment do + $progress.puts 'Migrating services ... '.blue c = ActiveRecord::Base.connection c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") + $progress.puts 'done'.green end end end -- cgit v1.2.3 From 500e277a8326b345854a5fc3114c1163826c1b4a Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 21 Sep 2015 00:13:58 +0200 Subject: Change notices during migrating --- lib/ci/migrate/tags.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb index 2bbd253f8de..2e4872f0716 100644 --- a/lib/ci/migrate/tags.rb +++ b/lib/ci/migrate/tags.rb @@ -12,10 +12,10 @@ module Ci 'WHERE (SELECT COUNT(*) FROM tags WHERE tags.name = ci_tags.name)=0' ) - puts 'Deleting old records' + puts 'Deleting old taggings...' connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'" - puts 'Inserting tags...' + puts 'Inserting taggings...' connection.execute( 'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' + "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " + -- cgit v1.2.3 From 098c1982713f3a4075efd6f27ab6326c75fc20ea Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 21 Sep 2015 00:14:07 +0200 Subject: Disable CI for time of migration --- lib/tasks/ci/migrate.rake | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'lib') diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake index 3507871fcb0..1de664c85e1 100644 --- a/lib/tasks/ci/migrate.rake +++ b/lib/tasks/ci/migrate.rake @@ -10,6 +10,10 @@ namespace :ci do puts '' end + # disable CI for time of migration + enable_ci(false) + + # unpack archives migrate = Ci::Migrate::Manager.new migrate.unpack @@ -18,6 +22,9 @@ namespace :ci do Rake::Task['ci:migrate:tags'].invoke Rake::Task['ci:migrate:services'].invoke + # enable CI for time of migration + enable_ci(true) + migrate.cleanup end @@ -71,4 +78,10 @@ namespace :ci do $progress.puts 'done'.green end end + + def enable_ci(enabled) + settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults + settings.ci_enabled = enabled + settings.save! + end end -- cgit v1.2.3 From a37e591e7414f51ca9076e966c724030225528db Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 21 Sep 2015 10:07:53 +0200 Subject: Use INSERT INTO to insert tags --- lib/ci/migrate/tags.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb index 2e4872f0716..97e043ece27 100644 --- a/lib/ci/migrate/tags.rb +++ b/lib/ci/migrate/tags.rb @@ -4,14 +4,15 @@ module Ci module Migrate class Tags def restore - ActiveRecord::Base.transaction do - puts 'Inserting tags...' - connection.execute( - 'INSERT INTO tags (name) ' + - 'SELECT ci_tags.name FROM ci_tags ' + - 'WHERE (SELECT COUNT(*) FROM tags WHERE tags.name = ci_tags.name)=0' - ) + puts 'Inserting tags...' + connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag| + begin + connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})") + rescue ActiveRecord::RecordNotUnique + end + end + ActiveRecord::Base.transaction do puts 'Deleting old taggings...' connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'" -- cgit v1.2.3