diff options
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/gitlab/backup.rake | 109 | ||||
-rw-r--r-- | lib/tasks/gitlab/cleanup.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/db.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/docs/compile_deprecations.rake | 39 | ||||
-rw-r--r-- | lib/tasks/gitlab/docs/redirect.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/gitaly.rake | 38 | ||||
-rw-r--r-- | lib/tasks/gitlab/seed/group_seed.rake | 2 |
7 files changed, 121 insertions, 75 deletions
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index cc10d73f76a..0bca63a64f5 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -9,14 +9,9 @@ namespace :gitlab do task create: :gitlab_environment do warn_user_is_not_gitlab - Rake::Task['gitlab:backup:db:create'].invoke - Rake::Task['gitlab:backup:repo:create'].invoke - Rake::Task['gitlab:backup:uploads:create'].invoke - Rake::Task['gitlab:backup:builds:create'].invoke - Rake::Task['gitlab:backup:artifacts:create'].invoke - Rake::Task['gitlab:backup:pages:create'].invoke - Rake::Task['gitlab:backup:lfs:create'].invoke - Rake::Task['gitlab:backup:registry:create'].invoke + %w(db repo uploads builds artifacts pages lfs terraform_state registry packages).each do |type| + Rake::Task["gitlab:backup:#{type}:create"].invoke + end backup = Backup::Manager.new(progress) backup.write_info @@ -83,7 +78,9 @@ namespace :gitlab do Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') + Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') + Rake::Task['gitlab:backup:packages:restore'].invoke unless backup.skipped?('packages') Rake::Task['gitlab:shell:setup'].invoke Rake::Task['cache:clear'].invoke @@ -133,8 +130,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("db") puts_time "[SKIPPED]".color(:cyan) else - Backup::Database.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Database.new(progress).dump + puts_time "done".color(:green) + rescue Backup::DatabaseBackupError => e + progress.puts "#{e.message}" + end end end @@ -166,8 +167,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("builds") puts_time "[SKIPPED]".color(:cyan) else - Backup::Builds.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Builds.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -185,8 +190,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("uploads") puts_time "[SKIPPED]".color(:cyan) else - Backup::Uploads.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Uploads.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -204,8 +213,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") puts_time "[SKIPPED]".color(:cyan) else - Backup::Artifacts.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Artifacts.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -223,8 +236,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("pages") puts_time "[SKIPPED]".color(:cyan) else - Backup::Pages.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Pages.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -242,8 +259,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("lfs") puts_time "[SKIPPED]".color(:cyan) else - Backup::Lfs.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Lfs.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end end @@ -254,6 +275,25 @@ namespace :gitlab do end end + namespace :terraform_state do + task create: :gitlab_environment do + puts_time "Dumping terraform states ... ".color(:blue) + + if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state") + puts_time "[SKIPPED]".color(:cyan) + else + Backup::TerraformState.new(progress).dump + puts_time "done".color(:green) + end + end + + task restore: :gitlab_environment do + puts_time "Restoring terraform states ... ".color(:blue) + Backup::TerraformState.new(progress).restore + puts_time "done".color(:green) + end + end + namespace :registry do task create: :gitlab_environment do puts_time "Dumping container registry images ... ".color(:blue) @@ -262,8 +302,12 @@ namespace :gitlab do if ENV["SKIP"] && ENV["SKIP"].include?("registry") puts_time "[SKIPPED]".color(:cyan) else - Backup::Registry.new(progress).dump - puts_time "done".color(:green) + begin + Backup::Registry.new(progress).dump + puts_time "done".color(:green) + rescue Backup::FileBackupError => e + progress.puts "#{e.message}" + end end else puts_time "[DISABLED]".color(:cyan) @@ -282,6 +326,25 @@ namespace :gitlab do end end + namespace :packages do + task create: :gitlab_environment do + puts_time "Dumping packages ... ".color(:blue) + + if ENV['SKIP'] && ENV['SKIP'].include?('packages') + puts_time "[SKIPPED]".color(:cyan) + else + Backup::Packages.new(progress).dump + puts_time "done".color(:green) + end + end + + task restore: :gitlab_environment do + puts_time "Restoring packages ...".color(:blue) + Backup::Packages.new(progress).restore + puts_time "done".color(:green) + end + end + def puts_time(msg) progress.puts "#{Time.now} -- #{msg}" Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") @@ -302,7 +365,7 @@ namespace :gitlab do if Feature.enabled?(:gitaly_backup, default_enabled: :yaml) max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence - Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency) + Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) else Backup::GitalyRpcBackup.new(progress) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 8f033a41e3d..f908a7606fa 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -100,15 +100,13 @@ namespace :gitlab do namespace :sessions do desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys" task active_sessions_lookup_keys: :gitlab_environment do - use_redis_session_store = Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) - redis_store_class = use_redis_session_store ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState session_key_pattern = "#{Gitlab::Redis::Sessions::USER_SESSIONS_LOOKUP_NAMESPACE}:*" last_save_check = Time.at(0) wait_time = 10.seconds cursor = 0 total_users_scanned = 0 - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| begin cursor, keys = redis.scan(cursor, match: session_key_pattern) total_users_scanned += keys.count diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 9e733fc3a0f..efb0e1ef1e1 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -170,7 +170,7 @@ namespace :gitlab do # the `ActiveRecord::Base.connection` might be switched to another one # This is due to `if should_reconnect`: # https://github.com/rails/rails/blob/a81aeb63a007ede2fe606c50539417dada9030c7/activerecord/lib/active_record/railties/databases.rake#L622 - ActiveRecord::Base.establish_connection :main + ActiveRecord::Base.establish_connection :main # rubocop: disable Database/EstablishConnection Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end diff --git a/lib/tasks/gitlab/docs/compile_deprecations.rake b/lib/tasks/gitlab/docs/compile_deprecations.rake index dc9788cb0b2..4ac68a9f850 100644 --- a/lib/tasks/gitlab/docs/compile_deprecations.rake +++ b/lib/tasks/gitlab/docs/compile_deprecations.rake @@ -4,19 +4,19 @@ namespace :gitlab do namespace :docs do desc "Generate deprecation list from individual files" task :compile_deprecations do - require_relative '../../../../tooling/deprecations/docs' - - File.write(Deprecations::Docs.path, Deprecations::Docs.render) - - puts "Deprecations compiled to #{Deprecations::Docs.path}" + require_relative '../../../../tooling/docs/deprecation_handling' + path = Rails.root.join("doc/update/deprecations.md") + File.write(path, Docs::DeprecationHandling.new('deprecation').render) + puts "Deprecations compiled to #{path}" end desc "Check that the deprecation doc is up to date" task :check_deprecations do - require_relative '../../../../tooling/deprecations/docs' + require_relative '../../../../tooling/docs/deprecation_handling' + path = Rails.root.join("doc/update/deprecations.md") - contents = Deprecations::Docs.render - doc = File.read(Deprecations::Docs.path) + contents = Docs::DeprecationHandling.new('deprecation').render + doc = File.read(path) if doc == contents puts "Deprecations doc is up to date." @@ -25,5 +25,28 @@ namespace :gitlab do abort end end + + desc "Generate removal list from individual files" + task :compile_removals do + require_relative '../../../../tooling/docs/deprecation_handling' + path = Rails.root.join("doc/update/removals.md") + File.write(path, Docs::DeprecationHandling.new('removal').render) + puts "Removals compiled to #{path}" + end + + desc "Check that the removal doc is up to date" + task :check_removals do + require_relative '../../../../tooling/docs/deprecation_handling' + path = Rails.root.join("doc/update/removals.md") + contents = Docs::DeprecationHandling.new('removal').render + doc = File.read(path) + + if doc == contents + puts "Removals doc is up to date." + else + format_output('Removals doc is outdated! You (or your technical writer) can update it by running `bin/rake gitlab:docs:compile_removals`.') + abort + end + end end end diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake index 123a4775605..e7ece9e0fdd 100644 --- a/lib/tasks/gitlab/docs/redirect.rake +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -51,7 +51,7 @@ namespace :gitlab do post.puts "remove_date: '#{date}'" post.puts '---' post.puts - post.puts "This file was moved to [another location](#{new_path})." + post.puts "This document was moved to [another location](#{new_path})." post.puts post.puts "<!-- This redirect file can be deleted after <#{date}>. -->" post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->" diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index b01a7902bf2..18c68615637 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -2,41 +2,6 @@ namespace :gitlab do namespace :gitaly do - desc 'Installs gitaly for running tests within gitlab-development-kit' - task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| - inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT')) - - if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk - Rake::Task["gitlab:gitaly:install"].invoke(*args) - - next - end - - gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly')) - - # Our test setup expects a git repo, so clone rather than copy - clone_repo(gdk_gitaly_dir, args.dir, clone_opts: %w[--depth 1]) unless Dir.exist?(args.dir) - - # We assume the GDK gitaly already compiled binaries - build_dir = File.join(gdk_gitaly_dir, '_build') - FileUtils.cp_r(build_dir, args.dir) - - # We assume the GDK gitaly already ran bundle install - bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle') - FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby')) - - # For completeness we copy this for gitaly's make target - ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle') - FileUtils.cp_r(ruby_bundle_file, args.dir) - - gitaly_binary = File.join(build_dir, 'bin', 'gitaly') - warn_gitaly_out_of_date!(gitaly_binary, Gitlab::GitalyClient.expected_server_version) - rescue Errno::ENOENT => e - puts "Could not copy files, did you run `gdk update`? Error: #{e.message}" - - raise - end - desc 'GitLab | Gitaly | Clone and checkout gitaly' task :clone, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab @@ -60,9 +25,6 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") storage_paths = { 'default' => args.storage_path } Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths) - # In CI we run scripts/gitaly-test-build - next if ENV['CI'].present? - Dir.chdir(args.dir) do Bundler.with_original_env do env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil } diff --git a/lib/tasks/gitlab/seed/group_seed.rake b/lib/tasks/gitlab/seed/group_seed.rake index a9a350fb6c3..491cf782985 100644 --- a/lib/tasks/gitlab/seed/group_seed.rake +++ b/lib/tasks/gitlab/seed/group_seed.rake @@ -125,7 +125,7 @@ class GroupSeeder name: FFaker::Name.name, email: FFaker::Internet.email, confirmed_at: DateTime.now, - password: Devise.friendly_token + password: Gitlab::Password.test_default ) end |