1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
|
package main
import (
"encoding/base64"
"errors"
"fmt"
"io"
"math/rand"
"net/url"
"os"
"strings"
"time"
"github.com/namsral/flag"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/labkit/errortracking"
cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/logging"
"gitlab.com/gitlab-org/gitlab-pages/internal/request"
"gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig"
"gitlab.com/gitlab-org/gitlab-pages/internal/validateargs"
"gitlab.com/gitlab-org/gitlab-pages/metrics"
)
// VERSION stores the information about the semantic version of application
var VERSION = "dev"
// REVISION stores the information about the git revision of application
var REVISION = "HEAD"
func init() {
flag.Var(&listenHTTP, "listen-http", "The address(es) to listen on for HTTP requests")
flag.Var(&listenHTTPS, "listen-https", "The address(es) to listen on for HTTPS requests")
flag.Var(&listenProxy, "listen-proxy", "The address(es) to listen on for proxy requests")
flag.Var(&ListenHTTPSProxyv2, "listen-https-proxyv2", "The address(es) to listen on for HTTPS PROXYv2 requests (https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt)")
flag.Var(&header, "header", "The additional http header(s) that should be send to the client")
}
var (
pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages")
pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages")
redirectHTTP = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS")
useHTTP2 = flag.Bool("use-http2", true, "Enable HTTP2 support")
pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored")
pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages")
artifactsServer = flag.String("artifacts-server", "", "API URL to proxy artifact requests to, e.g.: 'https://gitlab.com/api/v4'")
artifactsServerTimeout = flag.Int("artifacts-server-timeout", 10, "Timeout (in seconds) for a proxied request to the artifacts server")
pagesStatus = flag.String("pages-status", "", "The url path for a status page, e.g., /@status")
metricsAddress = flag.String("metrics-address", "", "The address to listen on for metrics requests")
sentryDSN = flag.String("sentry-dsn", "", "The address for sending sentry crash reporting to")
sentryEnvironment = flag.String("sentry-environment", "", "The environment for sentry crash reporting")
daemonUID = flag.Uint("daemon-uid", 0, "Drop privileges to this user")
daemonGID = flag.Uint("daemon-gid", 0, "Drop privileges to this group")
daemonInplaceChroot = flag.Bool("daemon-inplace-chroot", false, "Fall back to a non-bind-mount chroot of -pages-root when daemonizing")
logFormat = flag.String("log-format", "text", "The log output format: 'text' or 'json'")
logVerbose = flag.Bool("log-verbose", false, "Verbose logging")
_ = flag.String("admin-secret-path", "", "DEPRECATED")
_ = flag.String("admin-unix-listener", "", "DEPRECATED")
_ = flag.String("admin-https-listener", "", "DEPRECATED")
_ = flag.String("admin-https-cert", "", "DEPRECATED")
_ = flag.String("admin-https-key", "", "DEPRECATED")
secret = flag.String("auth-secret", "", "Cookie store hash key, should be at least 32 bytes long")
gitLabAuthServer = flag.String("auth-server", "", "DEPRECATED, use gitlab-server instead. GitLab server, for example https://www.gitlab.com")
gitLabServer = flag.String("gitlab-server", "", "GitLab server, for example https://www.gitlab.com")
internalGitLabServer = flag.String("internal-gitlab-server", "", "Internal GitLab server used for API requests, useful if you want to send that traffic over an internal load balancer, example value https://www.gitlab.com (defaults to value of gitlab-server)")
gitLabAPISecretKey = flag.String("api-secret-key", "", "File with secret key used to authenticate with the GitLab API")
gitlabClientHTTPTimeout = flag.Duration("gitlab-client-http-timeout", 10*time.Second, "GitLab API HTTP client connection timeout in seconds (default: 10s)")
gitlabClientJWTExpiry = flag.Duration("gitlab-client-jwt-expiry", 30*time.Second, "JWT Token expiry time in seconds (default: 30s)")
// TODO: implement functionality for disk, auto and gitlab https://gitlab.com/gitlab-org/gitlab/-/issues/217912
domainConfigSource = flag.String("domain-config-source", "disk", "Domain configuration source 'disk', 'auto' or 'gitlab' (default: 'disk')")
clientID = flag.String("auth-client-id", "", "GitLab application Client ID")
clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret")
redirectURI = flag.String("auth-redirect-uri", "", "GitLab application redirect URI")
maxConns = flag.Uint("max-conns", 5000, "Limit on the number of concurrent connections to the HTTP, HTTPS or proxy listeners")
insecureCiphers = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4")
tlsMinVersion = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min"))
tlsMaxVersion = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max"))
disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests")
// zip serving settings
zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval (default: 60s)")
zipCacheCleanup = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval (default: 30s)")
zipCacheRefresh = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval (default: 30s)")
zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout (default: 30s)")
// See init()
listenHTTP MultiStringFlag
listenHTTPS MultiStringFlag
listenProxy MultiStringFlag
ListenHTTPSProxyv2 MultiStringFlag
header MultiStringFlag
)
var (
errArtifactSchemaUnsupported = errors.New("artifacts-server scheme must be either http:// or https://")
errArtifactsServerTimeoutValue = errors.New("artifacts-server-timeout must be greater than or equal to 1")
errSecretNotDefined = errors.New("auth-secret must be defined if authentication is supported")
errClientIDNotDefined = errors.New("auth-client-id must be defined if authentication is supported")
errClientSecretNotDefined = errors.New("auth-client-secret must be defined if authentication is supported")
errGitLabServerNotDefined = errors.New("gitlab-server must be defined if authentication is supported")
errRedirectURINotDefined = errors.New("auth-redirect-uri must be defined if authentication is supported")
)
func gitlabServerFromFlags() string {
if *gitLabServer != "" {
return *gitLabServer
}
if *gitLabAuthServer != "" {
log.Warn("auth-server parameter is deprecated, use gitlab-server instead")
return *gitLabAuthServer
}
u, err := url.Parse(*artifactsServer)
if err != nil {
return ""
}
u.Path = ""
return u.String()
}
func internalGitLabServerFromFlags() string {
if *internalGitLabServer != "" {
return *internalGitLabServer
}
return gitlabServerFromFlags()
}
func setArtifactsServer(artifactsServer string, artifactsServerTimeout int, config *appConfig) {
u, err := url.Parse(artifactsServer)
if err != nil {
log.Fatal(err)
}
// url.Parse ensures that the Scheme attribute is always lower case.
if u.Scheme != request.SchemeHTTP && u.Scheme != request.SchemeHTTPS {
errortracking.Capture(err)
log.Fatal(errArtifactSchemaUnsupported)
}
if artifactsServerTimeout < 1 {
errortracking.Capture(err)
log.Fatal(errArtifactsServerTimeoutValue)
}
config.ArtifactsServerTimeout = artifactsServerTimeout
config.ArtifactsServer = artifactsServer
}
func setGitLabAPISecretKey(secretFile string, config *appConfig) {
encoded := readFile(secretFile)
decoded := make([]byte, base64.StdEncoding.DecodedLen(len(encoded)))
secretLength, err := base64.StdEncoding.Decode(decoded, encoded)
if err != nil {
log.WithError(err).Fatal("Failed to decode GitLab API secret")
}
if secretLength != 32 {
log.WithError(fmt.Errorf("Expected 32 bytes GitLab API secret but got %d bytes", secretLength)).Fatal("Failed to decode GitLab API secret")
}
config.GitLabAPISecretKey = decoded
}
func configFromFlags() appConfig {
var config appConfig
config.Domain = strings.ToLower(*pagesDomain)
config.RedirectHTTP = *redirectHTTP
config.HTTP2 = *useHTTP2
config.DisableCrossOriginRequests = *disableCrossOriginRequests
config.StatusPath = *pagesStatus
config.LogFormat = *logFormat
config.LogVerbose = *logVerbose
config.MaxConns = int(*maxConns)
config.InsecureCiphers = *insecureCiphers
// tlsMinVersion and tlsMaxVersion are validated in appMain
config.TLSMinVersion = tlsconfig.AllTLSVersions[*tlsMinVersion]
config.TLSMaxVersion = tlsconfig.AllTLSVersions[*tlsMaxVersion]
config.CustomHeaders = header
for _, file := range []struct {
contents *[]byte
path string
}{
{&config.RootCertificate, *pagesRootCert},
{&config.RootKey, *pagesRootKey},
} {
if file.path != "" {
*file.contents = readFile(file.path)
}
}
if *gitLabAPISecretKey != "" {
setGitLabAPISecretKey(*gitLabAPISecretKey, &config)
}
if *artifactsServer != "" {
setArtifactsServer(*artifactsServer, *artifactsServerTimeout, &config)
}
config.GitLabServer = gitlabServerFromFlags()
config.InternalGitLabServer = internalGitLabServerFromFlags()
config.GitlabClientHTTPTimeout = *gitlabClientHTTPTimeout
config.GitlabJWTTokenExpiration = *gitlabClientJWTExpiry
config.DomainConfigurationSource = *domainConfigSource
config.StoreSecret = *secret
config.ClientID = *clientID
config.ClientSecret = *clientSecret
config.RedirectURI = *redirectURI
config.SentryDSN = *sentryDSN
config.SentryEnvironment = *sentryEnvironment
checkAuthenticationConfig(config)
cfg.DefaultConfig.SetZip(&cfg.ZipServing{
ExpirationInterval: *zipCacheExpiration,
CleanupInterval: *zipCacheCleanup,
RefreshInterval: *zipCacheRefresh,
OpenTimeout: *zipOpenTimeout,
})
return config
}
func checkAuthenticationConfig(config appConfig) {
if config.StoreSecret == "" && config.ClientID == "" &&
config.ClientSecret == "" && config.RedirectURI == "" {
return
}
assertAuthConfig(config)
}
func assertAuthConfig(config appConfig) {
if config.StoreSecret == "" {
log.Fatal(errSecretNotDefined)
}
if config.ClientID == "" {
log.Fatal(errClientIDNotDefined)
}
if config.ClientSecret == "" {
log.Fatal(errClientSecretNotDefined)
}
if config.GitLabServer == "" {
log.Fatal(errGitLabServerNotDefined)
}
if config.RedirectURI == "" {
log.Fatal(errRedirectURINotDefined)
}
}
func initErrorReporting(sentryDSN, sentryEnvironment string) {
errortracking.Initialize(
errortracking.WithSentryDSN(sentryDSN),
errortracking.WithVersion(fmt.Sprintf("%s-%s", VERSION, REVISION)),
errortracking.WithLoggerName("gitlab-pages"),
errortracking.WithSentryEnvironment(sentryEnvironment))
}
func loadConfig() appConfig {
if err := validateargs.NotAllowed(os.Args[1:]); err != nil {
log.WithError(err).Fatal("Using invalid arguments, use -config=gitlab-pages-config file instead")
}
if err := validateargs.Deprecated(os.Args[1:]); err != nil {
log.WithError(err).Warn("Using deprecated arguments")
}
config := configFromFlags()
if config.SentryDSN != "" {
initErrorReporting(config.SentryDSN, config.SentryEnvironment)
}
log.WithFields(log.Fields{
"artifacts-server": *artifactsServer,
"artifacts-server-timeout": *artifactsServerTimeout,
"daemon-gid": *daemonGID,
"daemon-uid": *daemonUID,
"daemon-inplace-chroot": *daemonInplaceChroot,
"default-config-filename": flag.DefaultConfigFlagname,
"disable-cross-origin-requests": *disableCrossOriginRequests,
"domain": config.Domain,
"insecure-ciphers": config.InsecureCiphers,
"listen-http": strings.Join(listenHTTP, ","),
"listen-https": strings.Join(listenHTTPS, ","),
"listen-proxy": strings.Join(listenProxy, ","),
"listen-https-proxyv2": strings.Join(ListenHTTPSProxyv2, ","),
"log-format": *logFormat,
"metrics-address": *metricsAddress,
"pages-domain": *pagesDomain,
"pages-root": *pagesRoot,
"pages-status": *pagesStatus,
"redirect-http": config.RedirectHTTP,
"root-cert": *pagesRootKey,
"root-key": *pagesRootCert,
"status_path": config.StatusPath,
"tls-min-version": *tlsMinVersion,
"tls-max-version": *tlsMaxVersion,
"use-http-2": config.HTTP2,
"gitlab-server": config.GitLabServer,
"internal-gitlab-server": config.InternalGitLabServer,
"api-secret-key": *gitLabAPISecretKey,
"domain-config-source": config.DomainConfigurationSource,
"auth-redirect-uri": config.RedirectURI,
}).Debug("Start daemon with configuration")
return config
}
func appMain() {
var showVersion = flag.Bool("version", false, "Show version")
// read from -config=/path/to/gitlab-pages-config
flag.String(flag.DefaultConfigFlagname, "", "path to config file")
flag.Parse()
if err := tlsconfig.ValidateTLSVersions(*tlsMinVersion, *tlsMaxVersion); err != nil {
fatal(err, "invalid TLS version")
}
printVersion(*showVersion, VERSION)
err := logging.ConfigureLogging(*logFormat, *logVerbose)
if err != nil {
log.WithError(err).Fatal("Failed to initialize logging")
}
log.WithFields(log.Fields{
"version": VERSION,
"revision": REVISION,
}).Print("GitLab Pages Daemon")
log.Printf("URL: https://gitlab.com/gitlab-org/gitlab-pages")
if err := os.Chdir(*pagesRoot); err != nil {
fatal(err, "could not change directory into pagesRoot")
}
config := loadConfig()
for _, cs := range [][]io.Closer{
createAppListeners(&config),
createMetricsListener(&config),
} {
defer closeAll(cs)
}
if *daemonUID != 0 || *daemonGID != 0 {
if err := daemonize(config, *daemonUID, *daemonGID, *daemonInplaceChroot); err != nil {
errortracking.Capture(err)
fatal(err, "could not create pages daemon")
}
return
}
runApp(config)
}
func closeAll(cs []io.Closer) {
for _, c := range cs {
c.Close()
}
}
// createAppListeners returns net.Listener and *os.File instances. The
// caller must ensure they don't get closed or garbage-collected (which
// implies closing) too soon.
func createAppListeners(config *appConfig) []io.Closer {
var closers []io.Closer
for _, addr := range listenHTTP.Split() {
l, f := createSocket(addr)
closers = append(closers, l, f)
log.WithFields(log.Fields{
"listener": addr,
}).Debug("Set up HTTP listener")
config.ListenHTTP = append(config.ListenHTTP, f.Fd())
}
for _, addr := range listenHTTPS.Split() {
l, f := createSocket(addr)
closers = append(closers, l, f)
log.WithFields(log.Fields{
"listener": addr,
}).Debug("Set up HTTPS listener")
config.ListenHTTPS = append(config.ListenHTTPS, f.Fd())
}
for _, addr := range listenProxy.Split() {
l, f := createSocket(addr)
closers = append(closers, l, f)
log.WithFields(log.Fields{
"listener": addr,
}).Debug("Set up proxy listener")
config.ListenProxy = append(config.ListenProxy, f.Fd())
}
for _, addr := range ListenHTTPSProxyv2.Split() {
l, f := createSocket(addr)
closers = append(closers, l, f)
log.WithFields(log.Fields{
"listener": addr,
}).Debug("Set up https proxyv2 listener")
config.ListenHTTPSProxyv2 = append(config.ListenHTTPSProxyv2, f.Fd())
}
return closers
}
// createMetricsListener returns net.Listener and *os.File instances. The
// caller must ensure they don't get closed or garbage-collected (which
// implies closing) too soon.
func createMetricsListener(config *appConfig) []io.Closer {
addr := *metricsAddress
if addr == "" {
return nil
}
l, f := createSocket(addr)
config.ListenMetrics = f.Fd()
log.WithFields(log.Fields{
"listener": addr,
}).Debug("Set up metrics listener")
return []io.Closer{l, f}
}
func printVersion(showVersion bool, version string) {
if showVersion {
fmt.Fprintf(os.Stdout, "%s\n", version)
os.Exit(0)
}
}
func main() {
log.SetOutput(os.Stderr)
rand.Seed(time.Now().UnixNano())
metrics.MustRegister()
daemonMain()
appMain()
}
|