1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
# frozen_string_literal: true
require 'yaml'
module Backup
# Backup and restores repositories by querying the database
class Repositories < Task
extend ::Gitlab::Utils::Override
# @param [IO] progress IO interface to output progress
# @param [Object] :strategy Fetches backups from gitaly
# @param [Array<String>] :storages Filter by specified storage names. Empty means all storages.
# @param [Array<String>] :paths Filter by specified project paths. Empty means all projects, groups, and snippets.
# @param [Array<String>] :skip_paths Skip specified project paths. Empty means all projects, groups, and snippets.
def initialize(progress, strategy:, storages: [], paths: [], skip_paths: [])
super(progress)
@strategy = strategy
@storages = storages
@paths = paths
@skip_paths = skip_paths
end
override :dump
def dump(destination_path, backup_id)
strategy.start(:create, destination_path, backup_id: backup_id)
enqueue_consecutive
ensure
strategy.finish!
end
override :restore
def restore(destination_path, backup_id)
strategy.start(:restore, destination_path, remove_all_repositories: remove_all_repositories, backup_id: backup_id)
enqueue_consecutive
ensure
strategy.finish!
cleanup_snippets_without_repositories
restore_object_pools
end
private
attr_reader :strategy, :storages, :paths, :skip_paths
def remove_all_repositories
return if paths.present?
storages.presence || Gitlab.config.repositories.storages.keys
end
def enqueue_consecutive
enqueue_consecutive_projects
enqueue_consecutive_snippets
end
def enqueue_consecutive_projects
project_relation.find_each(batch_size: 1000) do |project|
enqueue_project(project)
end
end
def enqueue_consecutive_snippets
snippet_relation.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) }
end
def enqueue_project(project)
strategy.enqueue(project, Gitlab::GlRepository::PROJECT)
strategy.enqueue(project, Gitlab::GlRepository::WIKI)
return unless project.design_management_repository
strategy.enqueue(project.design_management_repository, Gitlab::GlRepository::DESIGN)
end
def enqueue_snippet(snippet)
strategy.enqueue(snippet, Gitlab::GlRepository::SNIPPET)
end
def project_relation
scope = Project.includes(:route, :group, :namespace)
scope = scope.id_in(ProjectRepository.for_repository_storage(storages).select(:project_id)) if storages.any?
if paths.any?
scope = scope.where_full_path_in(paths).or(
Project.where(namespace_id: Namespace.where_full_path_in(paths).self_and_descendants)
)
end
scope = scope.and(skipped_path_relation) if skip_paths.any?
scope
end
def snippet_relation
scope = Snippet.all
scope = scope.id_in(SnippetRepository.for_repository_storage(storages).select(:snippet_id)) if storages.any?
if paths.any?
scope = scope.joins(:project).merge(
Project.where_full_path_in(paths).or(
Project.where(namespace_id: Namespace.where_full_path_in(paths).self_and_descendants)
)
)
end
if skip_paths.any?
scope = scope.where(project: skipped_path_relation)
scope = scope.or(Snippet.where(project: nil)) if !paths.any? && !storages.any?
end
scope
end
def skipped_path_relation
Project.where.not(id: Project.where_full_path_in(skip_paths).or(
Project.where(namespace_id: Namespace.where_full_path_in(skip_paths).self_and_descendants)
))
end
def restore_object_pools
PoolRepository.includes(:source_project).find_each do |pool|
progress.puts " - Object pool #{pool.disk_path}..."
unless pool.source_project
progress.puts " - Object pool #{pool.disk_path}... " + "[SKIPPED]".color(:cyan)
next
end
pool.state = 'none'
pool.save
pool.schedule
end
end
# Snippets without a repository should be removed because they failed to import
# due to having invalid repositories
def cleanup_snippets_without_repositories
invalid_snippets = []
snippet_relation.find_each(batch_size: 1000).each do |snippet|
response = Snippets::RepositoryValidationService.new(nil, snippet).execute
next if response.success?
snippet.repository.remove
progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}")
invalid_snippets << snippet.id
end
Snippet.id_in(invalid_snippets).delete_all
end
end
end
Backup::Repositories.prepend_mod_with('Backup::Repositories')
|
