Adding auth to praefectjc-praefect-tls

4 files changed, 87 insertions, 9 deletions

diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index d6f4466a3..cfb92c279 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"flag"
 	"fmt"
@@ -15,6 +16,9 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 
 	"gitlab.com/gitlab-org/gitaly/internal/log"
 	"gitlab.com/gitlab-org/gitaly/internal/praefect"
@@ -45,7 +49,7 @@ func main() {
 		logger.Fatal(err)
 	}
 
-	listeners, err := getListeners(conf.SocketPath, conf.ListenAddr)
+	listeners, err := getInsecureListeners(conf.SocketPath, conf.ListenAddr)
 	if err != nil {
 		logger.Fatalf("%s", err)
 	}
@@ -106,10 +110,29 @@ func run(listeners []net.Listener, conf config.Config) error {
 
 	signal.Notify(termCh, signals...)
 
+	servers := []*praefect.Server{srv}
+
 	for _, l := range listeners {
 		go func(lis net.Listener) { serverErrors <- srv.Start(lis) }(l)
 	}
 
+	if conf.TLSListenAddr != "" {
+		cert, err := tls.LoadX509KeyPair(conf.TLS.CertificatePath, conf.TLS.KeyPath)
+		if err != nil {
+			logger.Fatal(err)
+		}
+
+		secureSrv := praefect.NewServer(coordinator, repl, []grpc.ServerOption{grpc.Creds(credentials.NewServerTLSFromCert(&cert))}, logger)
+
+		secureListener, err := getSecureListener(conf.TLSListenAddr)
+		if err != nil {
+			logger.Fatal(err)
+		}
+		go func() { serverErrors <- secureSrv.Start(secureListener) }()
+
+		servers = append(servers, secureSrv)
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -131,9 +154,22 @@ func run(listeners []net.Listener, conf config.Config) error {
 		cancel() // cancels the replicator job processing
 
 		ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
-		if shutdownErr := srv.Shutdown(ctx); shutdownErr != nil {
-			logger.Warnf("error received during shutting down: %v", shutdownErr)
-			return shutdownErr
+
+		g, ctx := errgroup.WithContext(ctx)
+
+		for _, srv := range servers {
+			g.Go(func() error {
+				if shutdownErr := srv.Shutdown(ctx); shutdownErr != nil {
+					logger.Warnf("error received during shutting down: %v", shutdownErr)
+					return shutdownErr
+				}
+
+				return nil
+			})
+		}
+
+		if err := g.Wait(); err != nil {
+			return err
 		}
 	case err := <-serverErrors:
 		return err
@@ -142,7 +178,7 @@ func run(listeners []net.Listener, conf config.Config) error {
 	return nil
 }
 
-func getListeners(socketPath, listenAddr string) ([]net.Listener, error) {
+func getInsecureListeners(socketPath, listenAddr string) ([]net.Listener, error) {
 	var listeners []net.Listener
 
 	if socketPath != "" {
@@ -174,6 +210,20 @@ func getListeners(socketPath, listenAddr string) ([]net.Listener, error) {
 	return listeners, nil
 }
 
+func getSecureListener(tlsListenAddr string) (net.Listener, error) {
+	if tlsListenAddr != "" {
+		l, err := net.Listen("tcp", tlsListenAddr)
+		if err != nil {
+			return nil, err
+		}
+
+		logger.WithField("address", tlsListenAddr).Info("listening at tcp address")
+		return l, nil
+	}
+
+	return nil, errors.New("listen address empty")
+}
+
 // registerServerVersionPromGauge registers a label with the current server version
 // making it easy to see what versions of Gitaly are running across a cluster
 func registerServerVersionPromGauge() {
diff --git a/config.praefect.toml.example b/config.praefect.toml.example
index 368f9fed6..5384dced7 100644
--- a/config.praefect.toml.example
+++ b/config.praefect.toml.example
@@ -7,7 +7,8 @@ listen_addr = "127.0.0.1:2305"
 # socket_path = "/home/git/gitlab/tmp/sockets/private/praefect.socket"
 # # Optional: export metrics via Prometheus
 # prometheus_listen_addr = "127.0.01:10101"
-
+# listen_addr = "127.0.0.1:9999"
+# tls_listen_addr = "127.0.0.1:8888"
 # # You can optionally configure Praefect to output JSON-formatted log messages to stdout
 # [logging]
 #   format = "json"
@@ -15,7 +16,13 @@ listen_addr = "127.0.0.1:2305"
 # # One of, in order: debug, info, warn, errror, fatal, panic
 # # Defaults to "info"
 #   level = "warn"
-
+# [tls]
+#   certificate_path = '/home/git/cert.cert'
+#   key_path = '/home/git/key.pem'
+#
+# [auth]
+#   token = 'abc123secret'
+#
 # # One or more Gitaly servers need to be configured to be managed. The names
 # of each server are used to link multiple nodes, or `gitaly_server`s together
 # as shard. listen_addr should be unique for all nodes.
@@ -25,11 +32,15 @@ listen_addr = "127.0.0.1:2305"
   storage = "praefect-git-0"
   address = "tcp://praefect-git-0.internal"
   primary = true
+  token = 'token1'
 
 [[node]]
   storage = "praefect-git-1"
   address = "tcp://praefect-git-1.internal"
+  token = 'token2'
 
 [[node]]
   storage = "praefect-git-2"
   address = "tcp://praefect-git-2.internal"
+  token = 'token3'
+
diff --git a/internal/praefect/config/config.go b/internal/praefect/config/config.go
index 633f57d1f..cfd4d7391 100644
--- a/internal/praefect/config/config.go
+++ b/internal/praefect/config/config.go
@@ -12,13 +12,27 @@ import (
 
 // Config is a container for everything found in the TOML config file
 type Config struct {
-	ListenAddr string `toml:"listen_addr"`
-	SocketPath string `toml:"socket_path"`
+	ListenAddr    string `toml:"listen_addr"`
+	TLSListenAddr string `toml:"tls_listen_addr"`
+	SocketPath    string `toml:"socket_path"`
 
 	Nodes []*models.Node `toml:"node"`
 
 	Logging              config.Logging `toml:"logging"`
 	PrometheusListenAddr string         `toml:"prometheus_listen_addr"`
+	Auth                 Auth           `toml:"auth"`
+	TLS                  TLS            `toml:"tls"`
+}
+
+// TLS contains details about the tls configuration
+type TLS struct {
+	CertificatePath string `toml:"certificate_path"`
+	KeyPath         string `toml:"key_path"`
+}
+
+// Auth contains details about the authentication configuration
+type Auth struct {
+	Token string `toml:"token"`
 }
 
 // FromFile loads the config for the passed file path
diff --git a/internal/praefect/server.go b/internal/praefect/server.go
index 3a7c0fabe..74e863a23 100644
--- a/internal/praefect/server.go
+++ b/internal/praefect/server.go
@@ -13,6 +13,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/internal/middleware/metadatahandler"
 	"gitlab.com/gitlab-org/gitaly/internal/middleware/panichandler"
 	"gitlab.com/gitlab-org/gitaly/internal/praefect/grpc-proxy/proxy"
+	"gitlab.com/gitlab-org/gitaly/internal/server/auth"
 	grpccorrelation "gitlab.com/gitlab-org/labkit/correlation/grpc"
 	grpctracing "gitlab.com/gitlab-org/labkit/tracing/grpc"
 	"google.golang.org/grpc"
@@ -35,6 +36,7 @@ func NewServer(c *Coordinator, repl ReplMgr, grpcOpts []grpc.ServerOption, l *lo
 			grpc_prometheus.StreamServerInterceptor,
 			cancelhandler.Stream, // Should be below LogHandler
 			grpctracing.StreamServerTracingInterceptor(),
+			auth.StreamServerInterceptor(),
 			// Panic handler should remain last so that application panics will be
 			// converted to errors and logged
 			panichandler.StreamPanicHandler,
@@ -45,6 +47,7 @@ func NewServer(c *Coordinator, repl ReplMgr, grpcOpts []grpc.ServerOption, l *lo
 			grpc_prometheus.UnaryServerInterceptor,
 			cancelhandler.Unary, // Should be below LogHandler
 			grpctracing.UnaryServerTracingInterceptor(),
+			auth.UnaryServerInterceptor(),
 			// Panic handler should remain last so that application panics will be
 			// converted to errors and logged
 			panichandler.UnaryPanicHandler,