diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/ci/api/api.rb | 1 | ||||
| -rw-r--r-- | lib/ci/api/forks.rb | 37 | ||||
| -rw-r--r-- | lib/ci/migrate/builds.rb | 29 | ||||
| -rw-r--r-- | lib/ci/migrate/database.rb | 42 | ||||
| -rw-r--r-- | lib/ci/migrate/manager.rb | 72 | ||||
| -rw-r--r-- | lib/ci/migrate/tags.rb | 55 | ||||
| -rw-r--r-- | lib/gitlab/email/receiver.rb | 5 | ||||
| -rw-r--r-- | lib/gitlab/incoming_email.rb (renamed from lib/gitlab/reply_by_email.rb) | 20 | ||||
| -rw-r--r-- | lib/tasks/ci/migrate.rake | 62 | ||||
| -rw-r--r-- | lib/tasks/gitlab/check.rake | 18 |
10 files changed, 208 insertions, 133 deletions
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 7bb8869d61a..5109c84e0ea 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -36,7 +36,6 @@ module Ci mount Commits mount Runners mount Projects - mount Forks mount Triggers end end diff --git a/lib/ci/api/forks.rb b/lib/ci/api/forks.rb deleted file mode 100644 index 152883a599f..00000000000 --- a/lib/ci/api/forks.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Ci - module API - class Forks < Grape::API - resource :forks do - # Create a fork - # - # Parameters: - # project_id (required) - The ID of a project - # project_token (requires) - Project token - # private_token(required) - User private token - # data (required) - GitLab project data (name_with_namespace, web_url, default_branch, ssh_url_to_repo) - # - # - # Example Request: - # POST /forks - post do - required_attributes! [:project_id, :data, :project_token, :private_token] - project = Ci::Project.find_by!(gitlab_id: params[:project_id]) - authenticate_project_token!(project) - - fork = Ci::CreateProjectService.new.execute( - current_user, - params[:data], - Ci::RoutesHelper.ci_project_url(":project_id"), - project - ) - - if fork - present fork, with: Entities::Project - else - not_found! - end - end - end - end - end -end diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb new file mode 100644 index 00000000000..c4f62e55295 --- /dev/null +++ b/lib/ci/migrate/builds.rb @@ -0,0 +1,29 @@ +module Ci + module Migrate + class Builds + attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir + + def initialize + @app_builds_dir = Settings.gitlab_ci.builds_path + @backup_dir = Gitlab.config.backup.path + @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz') + end + + def restore + backup_existing_builds_dir + + FileUtils.mkdir_p(app_builds_dir, mode: 0700) + unless system('tar', '-C', app_builds_dir, '-zxf', backup_builds_tarball) + abort 'Restore failed'.red + end + end + + def backup_existing_builds_dir + timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") + if File.exists?(app_builds_dir) + FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) + end + end + end + end +end diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb index 74f592dcaea..bf9b80f1f62 100644 --- a/lib/ci/migrate/database.rb +++ b/lib/ci/migrate/database.rb @@ -9,32 +9,32 @@ module Ci @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] end - def restore(ci_dump) - puts 'Deleting all CI related data ... ' - truncate_ci_tables + def restore + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close - puts 'Restoring CI data ... ' - case config["adapter"] - when /^mysql/ then - print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysql', *mysql_args, config['database'], in: ci_dump) - when "postgresql" then - puts "Restoring PostgreSQL database #{config['database']} ... " - pg_env - system('psql', config['database'], '-f', ci_dump) - end + restore_pid = case config["adapter"] + when /^mysql/ then + $progress.print "Restoring MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) + when "postgresql" then + $progress.print "Restoring PostgreSQL database #{config['database']} ... " + pg_env + spawn('psql', config['database'], in: decompress_rd) + end + decompress_rd.close + + success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + abort 'Restore failed' unless success end protected - def truncate_ci_tables - c = ActiveRecord::Base.connection - c.tables.select { |t| t.start_with?('ci_') }.each do |table| - puts "Deleting data from #{table}..." - c.execute("DELETE FROM #{table}") - end + def db_file_name + File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end def mysql_args diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb new file mode 100644 index 00000000000..e5e4fb784eb --- /dev/null +++ b/lib/ci/migrate/manager.rb @@ -0,0 +1,72 @@ +module Ci + module Migrate + class Manager + CI_IMPORT_PREFIX = '8.0' # Only allow imports from CI 8.0.x + + def cleanup + $progress.print "Deleting tmp directories ... " + + backup_contents.each do |dir| + next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + + if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) + $progress.puts "done".green + else + puts "deleting tmp directory '#{dir}' failed".red + abort 'Backup failed' + end + end + end + + def unpack + Dir.chdir(Gitlab.config.backup.path) + + # check for existing backups in the backup dir + file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i } + puts "no backups found" if file_list.count == 0 + + if file_list.count > 1 && ENV["BACKUP"].nil? + puts "Found more than one backup, please specify which one you want to restore:" + puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" + exit 1 + end + + tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar") + + unless File.exists?(tar_file) + puts "The specified CI backup doesn't exist!" + exit 1 + end + + $progress.print "Unpacking backup ... " + + unless Kernel.system(*%W(tar -xf #{tar_file})) + puts "unpacking backup failed".red + exit 1 + else + $progress.puts "done".green + end + + ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 + + # restoring mismatching backups can lead to unexpected problems + if !settings[:gitlab_version].start_with?(CI_IMPORT_PREFIX) + puts "GitLab CI version mismatch:".red + puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red + exit 1 + end + end + + private + + def backup_contents + ["db", "builds", "backup_information.yml"] + end + + def settings + @settings ||= YAML.load_file("backup_information.yml") + end + end + end +end + diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb index 125a535e9a9..97e043ece27 100644 --- a/lib/ci/migrate/tags.rb +++ b/lib/ci/migrate/tags.rb @@ -4,45 +4,38 @@ module Ci module Migrate class Tags def restore - puts 'Migrating tags for Runners... ' - list_objects('Runner').each do |id| - putc '.' - runner = Ci::Runner.find_by_id(id) - if runner - tags = list_tags('Runner', id) - runner.update_attributes(tag_list: tags) + puts 'Inserting tags...' + connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag| + begin + connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})") + rescue ActiveRecord::RecordNotUnique end end - puts '' - puts 'Migrating tags for Builds... ' - list_objects('Build').each do |id| - putc '.' - build = Ci::Build.find_by_id(id) - if build - tags = list_tags('Build', id) - build.update_attributes(tag_list: tags) - end + ActiveRecord::Base.transaction do + puts 'Deleting old taggings...' + connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'" + + puts 'Inserting taggings...' + connection.execute( + 'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' + + "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " + + 'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' + + 'JOIN tags ON tags.name = ci_tags.name ' + ) + + puts 'Resetting counters... ' + connection.execute( + 'UPDATE tags SET ' + + 'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)' + ) end - puts '' end protected - def list_objects(type) - ids = ActiveRecord::Base.connection.select_all( - "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}" - ) - ids.map { |id| id['taggable_id'] } - end - - def list_tags(type, id) - tags = ActiveRecord::Base.connection.select_all( - 'select ci_tags.name from ci_tags ' + - 'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' + - "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'" - ) - tags.map { |tag| tag['name'] } + def connection + ActiveRecord::Base.connection end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 355fbd27898..2b252b32887 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -65,7 +65,7 @@ module Gitlab def reply_key reply_key = nil message.to.each do |address| - reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address) + reply_key = Gitlab::IncomingEmail.key_from_address(address) break if reply_key end @@ -98,7 +98,8 @@ module Gitlab note: reply, noteable_type: sent_notification.noteable_type, noteable_id: sent_notification.noteable_id, - commit_id: sent_notification.commit_id + commit_id: sent_notification.commit_id, + line_code: sent_notification.line_code ).execute end end diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/incoming_email.rb index c3fe6778f06..856ccc71084 100644 --- a/lib/gitlab/reply_by_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,5 +1,5 @@ module Gitlab - module ReplyByEmail + module IncomingEmail class << self def enabled? config.enabled && address_formatted_correctly? @@ -7,20 +7,14 @@ module Gitlab def address_formatted_correctly? config.address && - config.address.include?("%{reply_key}") + config.address.include?("%{key}") end - def reply_key - return nil unless enabled? - - SecureRandom.hex(16) - end - - def reply_address(reply_key) - config.address.gsub('%{reply_key}', reply_key) + def reply_address(key) + config.address.gsub('%{key}', key) end - def reply_key_from_address(address) + def key_from_address(address) regex = address_regex return unless regex @@ -33,7 +27,7 @@ module Gitlab private def config - Gitlab.config.reply_by_email + Gitlab.config.incoming_email end def address_regex @@ -41,7 +35,7 @@ module Gitlab return nil unless wildcard_address regex = Regexp.escape(wildcard_address) - regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.+)") + regex = regex.gsub(Regexp.escape('%{key}'), "(.+)") Regexp.new(regex).freeze end end diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake index e7d41874a11..1de664c85e1 100644 --- a/lib/tasks/ci/migrate.rake +++ b/lib/tasks/ci/migrate.rake @@ -1,40 +1,56 @@ namespace :ci do desc 'GitLab | Import and migrate CI database' task migrate: :environment do + warn_user_is_not_gitlab + configure_cron_mode + unless ENV['force'] == 'yes' - puts "This will truncate all CI tables and restore it from provided backup." - puts "You will lose any previous CI data stored in the database." + puts 'This will remove all CI related data and restore it from the provided backup.' ask_to_continue - puts "" + puts '' end - Rake::Task["ci:migrate:db"].invoke - Rake::Task["ci:migrate:autoincrements"].invoke - Rake::Task["ci:migrate:tags"].invoke - Rake::Task["ci:migrate:services"].invoke + # disable CI for time of migration + enable_ci(false) + + # unpack archives + migrate = Ci::Migrate::Manager.new + migrate.unpack + + Rake::Task['ci:migrate:db'].invoke + Rake::Task['ci:migrate:builds'].invoke + Rake::Task['ci:migrate:tags'].invoke + Rake::Task['ci:migrate:services'].invoke + + # enable CI for time of migration + enable_ci(true) + + migrate.cleanup end namespace :migrate do desc 'GitLab | Import CI database' task db: :environment do - if ENV["CI_DUMP"].nil? - puts "No CI SQL dump specified:" - puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql" - exit 1 - end - - ci_dump = ENV["CI_DUMP"] - unless File.exists?(ci_dump) - puts "The specified sql dump doesn't exist!" - exit 1 - end + configure_cron_mode + $progress.puts 'Restoring database ... '.blue + Ci::Migrate::Database.new.restore + $progress.puts 'done'.green + end - ::Ci::Migrate::Database.new.restore(ci_dump) + desc 'GitLab | Import CI builds' + task builds: :environment do + configure_cron_mode + $progress.puts 'Restoring builds ... '.blue + Ci::Migrate::Builds.new.restore + $progress.puts 'done'.green end desc 'GitLab | Migrate CI tags' task tags: :environment do + configure_cron_mode + $progress.puts 'Migrating tags ... '.blue ::Ci::Migrate::Tags.new.restore + $progress.puts 'done'.green end desc 'GitLab | Migrate CI auto-increments' @@ -56,8 +72,16 @@ namespace :ci do desc 'GitLab | Migrate CI services' task services: :environment do + $progress.puts 'Migrating services ... '.blue c = ActiveRecord::Base.connection c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") + $progress.puts 'done'.green end end + + def enable_ci(enabled) + settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults + settings.ci_enabled = enabled + settings.save! + end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index b8eb13a4fea..c8222f8ce11 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -2,7 +2,7 @@ namespace :gitlab do desc "GitLab | Check the configuration of GitLab and its environment" task check: %w{gitlab:gitlab_shell:check gitlab:sidekiq:check - gitlab:reply_by_email:check + gitlab:incoming_email:check gitlab:ldap:check gitlab:app:check} @@ -634,13 +634,13 @@ namespace :gitlab do end - namespace :reply_by_email do + namespace :incoming_email do desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab start_checking "Reply by email" - if Gitlab.config.reply_by_email.enabled + if Gitlab.config.incoming_email.enabled check_address_formatted_correctly check_mail_room_config_exists check_imap_authentication @@ -665,12 +665,12 @@ namespace :gitlab do def check_address_formatted_correctly print "Address formatted correctly? ... " - if Gitlab::ReplyByEmail.address_formatted_correctly? + if Gitlab::IncomingEmail.address_formatted_correctly? puts "yes".green else puts "no".red try_fixing_it( - "Make sure that the address in config/gitlab.yml includes the '%{reply_key}' placeholder." + "Make sure that the address in config/gitlab.yml includes the '%{key}' placeholder." ) fix_and_rerun end @@ -689,7 +689,7 @@ namespace :gitlab do "Enable mail_room in the init.d configuration." ) for_more_information( - "doc/reply_by_email/README.md" + "doc/incoming_email/README.md" ) fix_and_rerun end @@ -708,7 +708,7 @@ namespace :gitlab do "Enable mail_room in your Procfile." ) for_more_information( - "doc/reply_by_email/README.md" + "doc/incoming_email/README.md" ) fix_and_rerun end @@ -753,7 +753,7 @@ namespace :gitlab do "Check that the information in config/mail_room.yml is correct" ) for_more_information( - "doc/reply_by_email/README.md" + "doc/incoming_email/README.md" ) fix_and_rerun end @@ -789,7 +789,7 @@ namespace :gitlab do "Check that the information in config/mail_room.yml is correct" ) for_more_information( - "doc/reply_by_email/README.md" + "doc/incoming_email/README.md" ) fix_and_rerun end |