Move configuration parsing to Config package

Changelog: changed
Signed-off-by: Balasankar "Balu" C <balasankarc@autistici.org>

1 files changed, 36 insertions, 306 deletions

diff --git a/main.go b/main.go
index 6f758244..94cc597f 100644
--- a/main.go
+++ b/main.go
@@ -1,23 +1,18 @@
 package main
 
 import (
-	"encoding/base64"
 	"fmt"
 	"io"
 	"math/rand"
-	"net/url"
 	"os"
-	"strings"
 	"time"
 
-	"github.com/namsral/flag"
 	log "github.com/sirupsen/logrus"
 
 	"gitlab.com/gitlab-org/labkit/errortracking"
 
+	cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config"
 	"gitlab.com/gitlab-org/gitlab-pages/internal/logging"
-	"gitlab.com/gitlab-org/gitlab-pages/internal/request"
-	"gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig"
 	"gitlab.com/gitlab-org/gitlab-pages/internal/validateargs"
 	"gitlab.com/gitlab-org/gitlab-pages/metrics"
 )
@@ -28,229 +23,6 @@ var VERSION = "dev"
 // REVISION stores the information about the git revision of application
 var REVISION = "HEAD"
 
-func init() {
-	// TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507
-	flag.Var(&listenHTTP, "listen-http", "The address(es) to listen on for HTTP requests")
-	flag.Var(&listenHTTPS, "listen-https", "The address(es) to listen on for HTTPS requests")
-	flag.Var(&listenProxy, "listen-proxy", "The address(es) to listen on for proxy requests")
-	flag.Var(&listenHTTPSProxyv2, "listen-https-proxyv2", "The address(es) to listen on for HTTPS PROXYv2 requests (https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt)")
-	flag.Var(&header, "header", "The additional http header(s) that should be send to the client")
-}
-
-var (
-	// TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507
-	pagesRootCert           = flag.String("root-cert", "", "The default path to file certificate to serve static pages")
-	pagesRootKey            = flag.String("root-key", "", "The default path to file certificate to serve static pages")
-	redirectHTTP            = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS")
-	useHTTP2                = flag.Bool("use-http2", true, "Enable HTTP2 support")
-	pagesRoot               = flag.String("pages-root", "shared/pages", "The directory where pages are stored")
-	pagesDomain             = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages")
-	artifactsServer         = flag.String("artifacts-server", "", "API URL to proxy artifact requests to, e.g.: 'https://gitlab.com/api/v4'")
-	artifactsServerTimeout  = flag.Int("artifacts-server-timeout", 10, "Timeout (in seconds) for a proxied request to the artifacts server")
-	pagesStatus             = flag.String("pages-status", "", "The url path for a status page, e.g., /@status")
-	metricsAddress          = flag.String("metrics-address", "", "The address to listen on for metrics requests")
-	sentryDSN               = flag.String("sentry-dsn", "", "The address for sending sentry crash reporting to")
-	sentryEnvironment       = flag.String("sentry-environment", "", "The environment for sentry crash reporting")
-	daemonUID               = flag.Uint("daemon-uid", 0, "Drop privileges to this user")
-	daemonGID               = flag.Uint("daemon-gid", 0, "Drop privileges to this group")
-	daemonInplaceChroot     = flag.Bool("daemon-inplace-chroot", false, "Fall back to a non-bind-mount chroot of -pages-root when daemonizing")
-	propagateCorrelationID  = flag.Bool("propagate-correlation-id", false, "Reuse existing Correlation-ID from the incoming request header `X-Request-ID` if present")
-	logFormat               = flag.String("log-format", "text", "The log output format: 'text' or 'json'")
-	logVerbose              = flag.Bool("log-verbose", false, "Verbose logging")
-	_                       = flag.String("admin-secret-path", "", "DEPRECATED")
-	_                       = flag.String("admin-unix-listener", "", "DEPRECATED")
-	_                       = flag.String("admin-https-listener", "", "DEPRECATED")
-	_                       = flag.String("admin-https-cert", "", "DEPRECATED")
-	_                       = flag.String("admin-https-key", "", "DEPRECATED")
-	secret                  = flag.String("auth-secret", "", "Cookie store hash key, should be at least 32 bytes long")
-	gitLabAuthServer        = flag.String("auth-server", "", "DEPRECATED, use gitlab-server instead. GitLab server, for example https://www.gitlab.com")
-	gitLabServer            = flag.String("gitlab-server", "", "GitLab server, for example https://www.gitlab.com")
-	internalGitLabServer    = flag.String("internal-gitlab-server", "", "Internal GitLab server used for API requests, useful if you want to send that traffic over an internal load balancer, example value https://www.gitlab.com (defaults to value of gitlab-server)")
-	gitLabAPISecretKey      = flag.String("api-secret-key", "", "File with secret key used to authenticate with the GitLab API")
-	gitlabClientHTTPTimeout = flag.Duration("gitlab-client-http-timeout", 10*time.Second, "GitLab API HTTP client connection timeout in seconds (default: 10s)")
-	gitlabClientJWTExpiry   = flag.Duration("gitlab-client-jwt-expiry", 30*time.Second, "JWT Token expiry time in seconds (default: 30s)")
-	// TODO: implement functionality for disk, auto and gitlab https://gitlab.com/gitlab-org/gitlab/-/issues/217912
-	domainConfigSource = flag.String("domain-config-source", "auto", "Domain configuration source 'disk', 'auto' or 'gitlab' (default: 'auto'). DEPRECATED: gitlab-pages will use the API-based configuration starting from 14.0 see https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382")
-	clientID           = flag.String("auth-client-id", "", "GitLab application Client ID")
-	clientSecret       = flag.String("auth-client-secret", "", "GitLab application Client Secret")
-	redirectURI        = flag.String("auth-redirect-uri", "", "GitLab application redirect URI")
-	authScope          = flag.String("auth-scope", "api", "Scope to be used for authentication (must match GitLab Pages OAuth application settings)")
-	maxConns           = flag.Uint("max-conns", 5000, "Limit on the number of concurrent connections to the HTTP, HTTPS or proxy listeners")
-	insecureCiphers    = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4")
-	tlsMinVersion      = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min"))
-	tlsMaxVersion      = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max"))
-	// TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507
-	zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval")
-	zipCacheCleanup    = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval")
-	zipCacheRefresh    = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval")
-	zipOpenTimeout     = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout")
-
-	disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests")
-
-	// See init()
-	listenHTTP         = MultiStringFlag{separator: ","}
-	listenHTTPS        = MultiStringFlag{separator: ","}
-	listenProxy        = MultiStringFlag{separator: ","}
-	listenHTTPSProxyv2 = MultiStringFlag{separator: ","}
-
-	header = MultiStringFlag{separator: ";;"}
-)
-
-func gitlabServerFromFlags() string {
-	if *gitLabServer != "" {
-		return *gitLabServer
-	}
-
-	if *gitLabAuthServer != "" {
-		log.Warn("auth-server parameter is deprecated, use gitlab-server instead")
-		return *gitLabAuthServer
-	}
-
-	u, err := url.Parse(*artifactsServer)
-	if err != nil {
-		return ""
-	}
-
-	u.Path = ""
-	return u.String()
-}
-
-func internalGitLabServerFromFlags() string {
-	if *internalGitLabServer != "" {
-		return *internalGitLabServer
-	}
-
-	return gitlabServerFromFlags()
-}
-
-func setArtifactsServer(artifactsServer string, artifactsServerTimeout int, config *appConfig) {
-	u, err := url.Parse(artifactsServer)
-	if err != nil {
-		log.Fatal(err)
-	}
-	// url.Parse ensures that the Scheme attribute is always lower case.
-	if u.Scheme != request.SchemeHTTP && u.Scheme != request.SchemeHTTPS {
-		errortracking.Capture(err)
-		log.Fatal("artifacts-server scheme must be either http:// or https://")
-	}
-
-	if artifactsServerTimeout < 1 {
-		errortracking.Capture(err)
-		log.Fatal("artifacts-server-timeout must be greater than or equal to 1")
-	}
-
-	config.ArtifactsServerTimeout = artifactsServerTimeout
-	config.ArtifactsServer = artifactsServer
-}
-
-func setGitLabAPISecretKey(secretFile string, config *appConfig) {
-	encoded := readFile(secretFile)
-
-	decoded := make([]byte, base64.StdEncoding.DecodedLen(len(encoded)))
-	secretLength, err := base64.StdEncoding.Decode(decoded, encoded)
-	if err != nil {
-		log.WithError(err).Fatal("Failed to decode GitLab API secret")
-	}
-
-	if secretLength != 32 {
-		log.WithError(fmt.Errorf("expected 32 bytes GitLab API secret but got %d bytes", secretLength)).Fatal("Failed to decode GitLab API secret")
-	}
-
-	config.GitLabAPISecretKey = decoded
-}
-
-func configFromFlags() appConfig {
-	var config appConfig
-
-	config.PagesRoot = *pagesRoot
-
-	config.Domain = strings.ToLower(*pagesDomain)
-	config.RedirectHTTP = *redirectHTTP
-	config.HTTP2 = *useHTTP2
-	config.DisableCrossOriginRequests = *disableCrossOriginRequests
-	config.PropagateCorrelationID = *propagateCorrelationID
-	config.StatusPath = *pagesStatus
-	config.LogFormat = *logFormat
-	config.LogVerbose = *logVerbose
-	config.MaxConns = int(*maxConns)
-	config.InsecureCiphers = *insecureCiphers
-	// tlsMinVersion and tlsMaxVersion are validated in appMain
-	config.TLSMinVersion = tlsconfig.AllTLSVersions[*tlsMinVersion]
-	config.TLSMaxVersion = tlsconfig.AllTLSVersions[*tlsMaxVersion]
-	config.CustomHeaders = header.Split()
-
-	for _, file := range []struct {
-		contents *[]byte
-		path     string
-	}{
-		{&config.RootCertificate, *pagesRootCert},
-		{&config.RootKey, *pagesRootKey},
-	} {
-		if file.path != "" {
-			*file.contents = readFile(file.path)
-		}
-	}
-
-	if *gitLabAPISecretKey != "" {
-		setGitLabAPISecretKey(*gitLabAPISecretKey, &config)
-	}
-
-	if *artifactsServer != "" {
-		setArtifactsServer(*artifactsServer, *artifactsServerTimeout, &config)
-	}
-
-	config.GitLabServer = gitlabServerFromFlags()
-	config.InternalGitLabServer = internalGitLabServerFromFlags()
-	config.GitlabClientHTTPTimeout = *gitlabClientHTTPTimeout
-	config.GitlabJWTTokenExpiration = *gitlabClientJWTExpiry
-	config.DomainConfigurationSource = *domainConfigSource
-	config.StoreSecret = *secret
-	config.ClientID = *clientID
-	config.ClientSecret = *clientSecret
-	config.RedirectURI = *redirectURI
-	config.AuthScope = *authScope
-	config.SentryDSN = *sentryDSN
-	config.SentryEnvironment = *sentryEnvironment
-
-	config.ZipCacheExpiry = *zipCacheExpiration
-	config.ZipCacheCleanup = *zipCacheCleanup
-	config.ZipCacheRefresh = *zipCacheRefresh
-	config.ZipeOpenTimeout = *zipOpenTimeout
-
-	checkAuthenticationConfig(config)
-
-	return config
-}
-
-func checkAuthenticationConfig(config appConfig) {
-	if config.StoreSecret == "" && config.ClientID == "" &&
-		config.ClientSecret == "" && config.RedirectURI == "" {
-		return
-	}
-	assertAuthConfig(config)
-}
-
-func assertAuthConfig(config appConfig) {
-	if config.StoreSecret == "" {
-		log.Fatal("auth-secret must be defined if authentication is supported")
-	}
-	if config.ClientID == "" {
-		log.Fatal("auth-client-id must be defined if authentication is supported")
-	}
-	if config.ClientSecret == "" {
-		log.Fatal("auth-client-secret must be defined if authentication is supported")
-	}
-	if config.GitLabServer == "" {
-		log.Fatal("gitlab-server must be defined if authentication is supported")
-	}
-	if config.RedirectURI == "" {
-		log.Fatal("auth-redirect-uri must be defined if authentication is supported")
-	}
-	if config.AuthScope == "" {
-		log.Fatal("auth-scope must be defined if authentication is supported")
-	}
-}
-
 func initErrorReporting(sentryDSN, sentryEnvironment string) {
 	errortracking.Initialize(
 		errortracking.WithSentryDSN(sentryDSN),
@@ -259,7 +31,7 @@ func initErrorReporting(sentryDSN, sentryEnvironment string) {
 		errortracking.WithSentryEnvironment(sentryEnvironment))
 }
 
-func loadConfig() appConfig {
+func appMain() {
 	if err := validateargs.NotAllowed(os.Args[1:]); err != nil {
 		log.WithError(err).Fatal("Using invalid arguments, use -config=gitlab-pages-config file instead")
 	}
@@ -268,93 +40,40 @@ func loadConfig() appConfig {
 		log.WithError(err).Warn("Using deprecated arguments")
 	}
 
-	config := configFromFlags()
-	if config.SentryDSN != "" {
-		initErrorReporting(config.SentryDSN, config.SentryEnvironment)
-	}
+	config := cfg.LoadConfig()
 
-	log.WithFields(log.Fields{
-		"artifacts-server":              *artifactsServer,
-		"artifacts-server-timeout":      *artifactsServerTimeout,
-		"daemon-gid":                    *daemonGID,
-		"daemon-uid":                    *daemonUID,
-		"daemon-inplace-chroot":         *daemonInplaceChroot,
-		"default-config-filename":       flag.DefaultConfigFlagname,
-		"disable-cross-origin-requests": *disableCrossOriginRequests,
-		"domain":                        config.Domain,
-		"insecure-ciphers":              config.InsecureCiphers,
-		"listen-http":                   listenHTTP,
-		"listen-https":                  listenHTTPS,
-		"listen-proxy":                  listenProxy,
-		"listen-https-proxyv2":          listenHTTPSProxyv2,
-		"log-format":                    *logFormat,
-		"metrics-address":               *metricsAddress,
-		"pages-domain":                  *pagesDomain,
-		"pages-root":                    *pagesRoot,
-		"pages-status":                  *pagesStatus,
-		"propagate-correlation-id":      *propagateCorrelationID,
-		"redirect-http":                 config.RedirectHTTP,
-		"root-cert":                     *pagesRootKey,
-		"root-key":                      *pagesRootCert,
-		"status_path":                   config.StatusPath,
-		"tls-min-version":               *tlsMinVersion,
-		"tls-max-version":               *tlsMaxVersion,
-		"use-http-2":                    config.HTTP2,
-		"gitlab-server":                 config.GitLabServer,
-		"internal-gitlab-server":        config.InternalGitLabServer,
-		"api-secret-key":                *gitLabAPISecretKey,
-		"domain-config-source":          config.DomainConfigurationSource,
-		"auth-redirect-uri":             config.RedirectURI,
-		"auth-scope":                    config.AuthScope,
-		"zip-cache-expiration":          config.ZipCacheExpiry,
-		"zip-cache-cleanup":             config.ZipCacheCleanup,
-		"zip-cache-refresh":             config.ZipCacheRefresh,
-		"zip-open-timeout":              config.ZipeOpenTimeout,
-	}).Debug("Start daemon with configuration")
-
-	return config
-}
+	printVersion(config.General.ShowVersion, VERSION)
 
-func appMain() {
-	var showVersion = flag.Bool("version", false, "Show version")
-
-	// read from -config=/path/to/gitlab-pages-config
-	flag.String(flag.DefaultConfigFlagname, "", "path to config file")
-
-	flag.Parse()
-
-	if err := tlsconfig.ValidateTLSVersions(*tlsMinVersion, *tlsMaxVersion); err != nil {
-		fatal(err, "invalid TLS version")
+	if config.Sentry.DSN != "" {
+		initErrorReporting(config.Sentry.DSN, config.Sentry.Environment)
 	}
 
-	printVersion(*showVersion, VERSION)
-
-	err := logging.ConfigureLogging(*logFormat, *logVerbose)
+	err := logging.ConfigureLogging(config.Log.Format, config.Log.Verbose)
 	if err != nil {
 		log.WithError(err).Fatal("Failed to initialize logging")
 	}
 
+	cfg.LogConfig(config)
+
 	log.WithFields(log.Fields{
 		"version":  VERSION,
 		"revision": REVISION,
 	}).Print("GitLab Pages Daemon")
 	log.Printf("URL: https://gitlab.com/gitlab-org/gitlab-pages")
 
-	if err := os.Chdir(*pagesRoot); err != nil {
+	if err := os.Chdir(config.General.RootDir); err != nil {
 		fatal(err, "could not change directory into pagesRoot")
 	}
 
-	config := loadConfig()
-
 	for _, cs := range [][]io.Closer{
-		createAppListeners(&config),
-		createMetricsListener(&config),
+		createAppListeners(config),
+		createMetricsListener(config),
 	} {
 		defer closeAll(cs)
 	}
 
-	if *daemonUID != 0 || *daemonGID != 0 {
-		if err := daemonize(config, *daemonUID, *daemonGID, *daemonInplaceChroot, config.PagesRoot); err != nil {
+	if config.Daemon.UID != 0 || config.Daemon.GID != 0 {
+		if err := daemonize(config); err != nil {
 			errortracking.Capture(err)
 			fatal(err, "could not create pages daemon")
 		}
@@ -374,10 +93,14 @@ func closeAll(cs []io.Closer) {
 // createAppListeners returns net.Listener and *os.File instances. The
 // caller must ensure they don't get closed or garbage-collected (which
 // implies closing) too soon.
-func createAppListeners(config *appConfig) []io.Closer {
+func createAppListeners(config *cfg.Config) []io.Closer {
 	var closers []io.Closer
+	var httpListeners []uintptr
+	var httpsListeners []uintptr
+	var proxyListeners []uintptr
+	var httpsProxyv2Listeners []uintptr
 
-	for _, addr := range listenHTTP.Split() {
+	for _, addr := range config.ListenHTTPStrings.Split() {
 		l, f := createSocket(addr)
 		closers = append(closers, l, f)
 
@@ -385,10 +108,10 @@ func createAppListeners(config *appConfig) []io.Closer {
 			"listener": addr,
 		}).Debug("Set up HTTP listener")
 
-		config.ListenHTTP = append(config.ListenHTTP, f.Fd())
+		httpListeners = append(httpListeners, f.Fd())
 	}
 
-	for _, addr := range listenHTTPS.Split() {
+	for _, addr := range config.ListenHTTPSStrings.Split() {
 		l, f := createSocket(addr)
 		closers = append(closers, l, f)
 
@@ -396,10 +119,10 @@ func createAppListeners(config *appConfig) []io.Closer {
 			"listener": addr,
 		}).Debug("Set up HTTPS listener")
 
-		config.ListenHTTPS = append(config.ListenHTTPS, f.Fd())
+		httpsListeners = append(httpsListeners, f.Fd())
 	}
 
-	for _, addr := range listenProxy.Split() {
+	for _, addr := range config.ListenProxyStrings.Split() {
 		l, f := createSocket(addr)
 		closers = append(closers, l, f)
 
@@ -407,10 +130,10 @@ func createAppListeners(config *appConfig) []io.Closer {
 			"listener": addr,
 		}).Debug("Set up proxy listener")
 
-		config.ListenProxy = append(config.ListenProxy, f.Fd())
+		proxyListeners = append(proxyListeners, f.Fd())
 	}
 
-	for _, addr := range listenHTTPSProxyv2.Split() {
+	for _, addr := range config.ListenHTTPSProxyv2Strings.Split() {
 		l, f := createSocket(addr)
 		closers = append(closers, l, f)
 
@@ -418,7 +141,14 @@ func createAppListeners(config *appConfig) []io.Closer {
 			"listener": addr,
 		}).Debug("Set up https proxyv2 listener")
 
-		config.ListenHTTPSProxyv2 = append(config.ListenHTTPSProxyv2, f.Fd())
+		httpsProxyv2Listeners = append(httpsProxyv2Listeners, f.Fd())
+	}
+
+	config.Listeners = &cfg.Listeners{
+		HTTP:         httpListeners,
+		HTTPS:        httpsListeners,
+		Proxy:        proxyListeners,
+		HTTPSProxyv2: httpsProxyv2Listeners,
 	}
 
 	return closers
@@ -427,8 +157,8 @@ func createAppListeners(config *appConfig) []io.Closer {
 // createMetricsListener returns net.Listener and *os.File instances. The
 // caller must ensure they don't get closed or garbage-collected (which
 // implies closing) too soon.
-func createMetricsListener(config *appConfig) []io.Closer {
-	addr := *metricsAddress
+func createMetricsListener(config *cfg.Config) []io.Closer {
+	addr := config.General.MetricsAddress
 	if addr == "" {
 		return nil
 	}