From c085f88fc6a293732cb9f1f96884c0795ff86bd3 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 7 Jun 2017 15:38:12 +0200 Subject: fix backup task to ignore errors per project --- lib/backup/repository.rb | 55 +++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 6b29600a751..7723f9979b3 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -14,7 +14,7 @@ module Backup # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace - if project.empty_repo? + if empty_repo?(project) $progress.puts "[SKIPPED]".color(:cyan) else in_path(path_to_project_repo) do |dir| @@ -23,10 +23,7 @@ module Backup output, status = Gitlab::Popen.popen(cmd) unless status.zero? - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(project, cmd.join(' '), output) end end @@ -36,10 +33,7 @@ module Backup if status.zero? $progress.puts "[DONE]".color(:green) else - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(project, cmd.join(' '), output) end end @@ -49,7 +43,7 @@ module Backup if File.exist?(path_to_wiki_repo) $progress.print " * #{wiki.path_with_namespace} ... " - if wiki.repository.empty? + if empty_wiki_repo?(wiki) $progress.puts " [SKIPPED]".color(:cyan) else cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) @@ -57,10 +51,7 @@ module Backup if status.zero? $progress.puts " [DONE]".color(:green) else - puts " [FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(wiki, cmd.join(' '), output) end end end @@ -96,10 +87,7 @@ module Backup if status.zero? $progress.puts "[DONE]".color(:green) else - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end in_path(path_to_tars(project)) do |dir| @@ -107,10 +95,7 @@ module Backup output, status = Gitlab::Popen.popen(cmd) unless status.zero? - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end end @@ -131,10 +116,7 @@ module Backup if status.zero? $progress.puts " [DONE]".color(:green) else - puts " [FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end end end @@ -201,6 +183,27 @@ module Backup private + def progress_warn(project, cmd, output) + $progress.puts "[WARNING] Executing #{cmd}".color(:orange) + $progress.puts "Ignoring error on #{project.path_with_namespace} - #{output}".color(:orange) + end + + def empty_repo?(project) + project.empty_repo? + rescue => e + $progress.puts "Ignoring error on #{project.full_path} repository - #{e.message}".color(:orange) + + false + end + + def empty_wiki_repo?(wiki) + wiki.repository.empty? + rescue => e + $progress.puts "Ignoring error on #{wiki.path_with_namespace} repository - #{e.message}".color(:orange) + + false + end + def repository_storage_paths_args Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } end -- cgit v1.2.3 From 8acb0f55961500658377a1f941584dddb7952570 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 7 Jun 2017 16:50:21 +0200 Subject: refactor code and spec --- lib/backup/repository.rb | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 7723f9979b3..8e6235c2014 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -7,7 +7,7 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - $progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.path_with_namespace} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -15,7 +15,7 @@ module Backup FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace if empty_repo?(project) - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else in_path(path_to_project_repo) do |dir| FileUtils.mkdir_p(path_to_tars(project)) @@ -31,7 +31,7 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts "[DONE]".color(:green) + progress.puts "[DONE]".color(:green) else progress_warn(project, cmd.join(' '), output) end @@ -42,14 +42,14 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_repo) - $progress.print " * #{wiki.path_with_namespace} ... " + progress.print " * #{wiki.path_with_namespace} ... " if empty_wiki_repo?(wiki) - $progress.puts " [SKIPPED]".color(:cyan) + progress.puts " [SKIPPED]".color(:cyan) else cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else progress_warn(wiki, cmd.join(' '), output) end @@ -71,7 +71,7 @@ module Backup end Project.find_each(batch_size: 1000) do |project| - $progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.path_with_namespace} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -85,7 +85,7 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts "[DONE]".color(:green) + progress.puts "[DONE]".color(:green) else progress_warn(project, cmd.join(' '), output) end @@ -104,7 +104,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - $progress.print " * #{wiki.path_with_namespace} ... " + progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then @@ -114,19 +114,19 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else progress_warn(project, cmd.join(' '), output) end end end - $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) + progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd}" @@ -184,22 +184,14 @@ module Backup private def progress_warn(project, cmd, output) - $progress.puts "[WARNING] Executing #{cmd}".color(:orange) - $progress.puts "Ignoring error on #{project.path_with_namespace} - #{output}".color(:orange) + progress.puts "[WARNING] Executing #{cmd}".color(:orange) + progress.puts "Ignoring error on #{project.path_with_namespace} - #{output}".color(:orange) end - def empty_repo?(project) - project.empty_repo? + def empty_repo?(project_or_wiki) + project_or_wiki.repository.empty_repo? rescue => e - $progress.puts "Ignoring error on #{project.full_path} repository - #{e.message}".color(:orange) - - false - end - - def empty_wiki_repo?(wiki) - wiki.repository.empty? - rescue => e - $progress.puts "Ignoring error on #{wiki.path_with_namespace} repository - #{e.message}".color(:orange) + progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.path_with_namespace} - #{e.message}".color(:orange) false end @@ -207,5 +199,9 @@ module Backup def repository_storage_paths_args Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } end + + def progress + progress + end end end -- cgit v1.2.3 From a5b920dcf0a5ff6cbef718ea278207916425034e Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 7 Jun 2017 16:54:12 +0200 Subject: fix wiki --- lib/backup/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 8e6235c2014..e93827211ae 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -43,7 +43,7 @@ module Backup if File.exist?(path_to_wiki_repo) progress.print " * #{wiki.path_with_namespace} ... " - if empty_wiki_repo?(wiki) + if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) -- cgit v1.2.3 From 35a78da27f94e7b77c8487913beceb23db3b66bc Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 7 Jun 2017 17:00:32 +0200 Subject: fix typo --- lib/backup/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index e93827211ae..a1685c77916 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -201,7 +201,7 @@ module Backup end def progress - progress + $progress end end end -- cgit v1.2.3