1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
|
# frozen_string_literal: true
# This file contains environment settings for gitaly when it's running
# as part of the gitlab-ce/ee test suite.
#
# Please be careful when modifying this file. Your changes must work
# both for local development rspec runs, and in CI.
require 'securerandom'
require 'socket'
require 'logger'
require 'fileutils'
require_relative '../../../lib/gitlab/utils'
module GitalySetup
extend self
REPOS_STORAGE = 'default'
LOGGER = begin
default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse
Logger.new($stdout, level: level, formatter: ->(_, _, _, msg) { msg })
end
def expand_path(path)
File.expand_path(path, File.join(__dir__, '../../..'))
end
def tmp_tests_gitaly_dir
expand_path('tmp/tests/gitaly')
end
def runtime_dir
expand_path('tmp/run')
end
def tmp_tests_gitaly_bin_dir
File.join(tmp_tests_gitaly_dir, '_build', 'bin')
end
def tmp_tests_gitlab_shell_dir
expand_path('tmp/tests/gitlab-shell')
end
def rails_gitlab_shell_secret
expand_path('.gitlab_shell_secret')
end
def gitlab_shell_secret_file
File.join(tmp_tests_gitlab_shell_dir, '.gitlab_shell_secret')
end
def env
{
# Git hooks can't run during tests as the internal API is not running.
'GITALY_TESTING_NO_GIT_HOOKS' => "1",
'GITALY_TESTING_ENABLE_ALL_FEATURE_FLAGS' => "true"
}
end
def config_path(service)
case service
when :gitaly
File.join(tmp_tests_gitaly_dir, 'config.toml')
when :gitaly2
File.join(tmp_tests_gitaly_dir, 'gitaly2.config.toml')
when :praefect
File.join(tmp_tests_gitaly_dir, 'praefect.config.toml')
end
end
def repos_path(storage = REPOS_STORAGE)
Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
end
def service_cmd(service, toml = nil)
toml ||= config_path(service)
case service
when :gitaly, :gitaly2
[File.join(tmp_tests_gitaly_bin_dir, 'gitaly'), toml]
when :praefect
[File.join(tmp_tests_gitaly_bin_dir, 'praefect'), '-config', toml]
end
end
def run_command(cmd, env: {})
system(env, *cmd, exception: true, chdir: tmp_tests_gitaly_dir)
end
def build_gitaly
run_command(%w[make all WITH_BUNDLED_GIT=YesPlease], env: env.merge('GIT_VERSION' => nil))
end
def start_gitaly(toml = nil)
start(:gitaly, toml)
end
def start_gitaly2
start(:gitaly2)
end
def start_praefect
if praefect_with_db?
LOGGER.debug 'Starting Praefect with database election strategy'
start(:praefect, File.join(tmp_tests_gitaly_dir, 'praefect-db.config.toml'))
else
LOGGER.debug 'Starting Praefect with in-memory election strategy'
start(:praefect)
end
end
def start(service, toml = nil)
toml ||= config_path(service)
args = service_cmd(service, toml)
# Ensure that tmp/run exists
FileUtils.mkdir_p(runtime_dir)
# Ensure user configuration does not affect Git
# Context: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58776#note_547613780
env = self.env.merge('HOME' => nil, 'XDG_CONFIG_HOME' => nil)
pid = spawn(env, *args, [:out, :err] => "log/#{service}-test.log")
begin
try_connect!(service, toml)
rescue StandardError
Process.kill('TERM', pid)
raise
end
pid
end
# Taken from Gitlab::Shell.generate_and_link_secret_token
def ensure_gitlab_shell_secret!
secret_file = rails_gitlab_shell_secret
shell_link = gitlab_shell_secret_file
unless File.size?(secret_file)
File.write(secret_file, SecureRandom.hex(16))
end
unless File.exist?(shell_link)
FileUtils.ln_s(secret_file, shell_link)
end
end
def connect_proc(toml)
# This code needs to work in an environment where we cannot use bundler,
# so we cannot easily use the toml-rb gem. This ad-hoc parser should be
# good enough.
config_text = File.read(toml)
config_text.lines.each do |line|
match_data = line.match(/^\s*(socket_path|listen_addr)\s*=\s*"([^"]*)"$/)
next unless match_data
case match_data[1]
when 'socket_path'
return -> { UNIXSocket.new(match_data[2]) }
when 'listen_addr'
addr, port = match_data[2].split(':')
return -> { TCPSocket.new(addr, port.to_i) }
end
end
raise "failed to find socket_path or listen_addr in #{toml}"
end
def try_connect!(service, toml)
LOGGER.debug "Trying to connect to #{service}: "
timeout = 20
delay = 0.1
connect = connect_proc(toml)
Integer(timeout / delay).times do
connect.call
LOGGER.debug " OK\n"
return
rescue Errno::ENOENT, Errno::ECONNREFUSED
LOGGER.debug '.'
sleep delay
end
LOGGER.warn " FAILED to connect to #{service}\n"
raise "could not connect to #{service}"
end
def gitaly_socket_path
Gitlab::GitalyClient.address(REPOS_STORAGE).delete_prefix('unix:')
end
def gitaly_dir
socket_path = gitaly_socket_path
socket_path = File.expand_path(gitaly_socket_path) if expand_path_for_socket?
File.dirname(socket_path)
end
# Linux fails with "bind: invalid argument" if a UNIX socket path exceeds 108 characters:
# https://github.com/golang/go/issues/6895. We use absolute paths in CI to ensure
# that changes in the current working directory don't affect GRPC reconnections.
def expand_path_for_socket?
!!ENV['CI']
end
def setup_gitaly
unless ENV['CI']
# In CI Gitaly is built in the setup-test-env job and saved in the
# artifacts. So when tests are started, there's no need to build Gitaly.
build_gitaly
end
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
runtime_dir: runtime_dir,
prometheus_listen_addr: 'localhost:9236'
}
)
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
runtime_dir: runtime_dir,
gitaly_socket: "gitaly2.socket",
config_filename: "gitaly2.config.toml"
}
)
# In CI we need to pre-generate both config files.
# For local testing we'll create the correct file on-demand.
if ENV['CI'] || !praefect_with_db?
Gitlab::SetupHelper::Praefect.create_configuration(
gitaly_dir,
{ 'praefect' => repos_path },
force: true
)
end
if ENV['CI'] || praefect_with_db?
Gitlab::SetupHelper::Praefect.create_configuration(
gitaly_dir,
{ 'praefect' => repos_path },
force: true,
options: {
per_repository: true,
config_filename: 'praefect-db.config.toml',
pghost: ENV['CI'] ? 'postgres' : ENV.fetch('PGHOST'),
pgport: ENV['CI'] ? 5432 : ENV.fetch('PGPORT').to_i,
pguser: ENV['CI'] ? 'postgres' : ENV.fetch('USER')
}
)
end
# In CI no database is running when Gitaly is set up
# so scripts/gitaly-test-spawn will take care of it instead.
setup_praefect unless ENV['CI']
end
def setup_praefect
return unless praefect_with_db?
migrate_cmd = service_cmd(:praefect, File.join(tmp_tests_gitaly_dir, 'praefect-db.config.toml')) + ['sql-migrate']
system(env, *migrate_cmd, [:out, :err] => 'log/praefect-test.log')
end
def socket_path(service)
File.join(tmp_tests_gitaly_dir, "#{service}.socket")
end
def praefect_socket_path
"unix:" + socket_path(:praefect)
end
def stop(pid)
Process.kill('KILL', pid)
rescue Errno::ESRCH
# The process can already be gone if the test run was INTerrupted.
end
def spawn_gitaly(toml = nil)
pids = []
if toml
pids << start_gitaly(toml)
else
pids << start_gitaly
pids << start_gitaly2
pids << start_praefect
end
Kernel.at_exit do
# In CI, this function is called by scripts/gitaly-test-spawn, triggered
# in a before_script. Gitaly needs to remain running until the container
# is stopped.
next if ENV['CI']
# In Workhorse tests (locally or in CI), this function is called by
# scripts/gitaly-test-spawn during `make test`. Gitaly needs to remain
# running until `make test` cleans it up.
next if ENV['GITALY_PID_FILE']
pids.each { |pid| stop(pid) }
end
rescue StandardError
raise gitaly_failure_message
end
def gitaly_failure_message
message = "gitaly spawn failed\n\n"
message += "- The `gitaly` binary does not exist: #{gitaly_binary}\n" unless File.exist?(gitaly_binary)
message += "- The `praefect` binary does not exist: #{praefect_binary}\n" unless File.exist?(praefect_binary)
message += "- No `git` binaries exist\n" if git_binaries.empty?
message += "\nCheck log/gitaly-test.log & log/praefect-test.log for errors.\n"
unless ENV['CI']
message += "\nIf binaries are missing, try running `make -C tmp/tests/gitaly all WITH_BUNDLED_GIT=YesPlease`.\n"
message += "\nOtherwise, try running `rm -rf #{tmp_tests_gitaly_dir}`."
end
message
end
def git_binaries
Dir.glob(File.join(tmp_tests_gitaly_dir, "_build", "bin", "gitaly-git-v*"))
end
def gitaly_binary
File.join(tmp_tests_gitaly_dir, "_build", "bin", "gitaly")
end
def praefect_binary
File.join(tmp_tests_gitaly_dir, "_build", "bin", "praefect")
end
def praefect_with_db?
Gitlab::Utils.to_boolean(ENV['GITALY_PRAEFECT_WITH_DB'], default: false)
end
end
|