Merge branch 'zj-socket-listener-preafect' into 'master'

Support socket paths for praefect

Closes #1523

See merge request gitlab-org/gitaly!1115

7 files changed, 95 insertions, 18 deletions

diff --git a/changelogs/unreleased/zj-socket-listener-preafect.yml b/changelogs/unreleased/zj-socket-listener-preafect.yml
new file mode 100644
index 000000000..b030e133b
--- /dev/null
+++ b/changelogs/unreleased/zj-socket-listener-preafect.yml
@@ -0,0 +1,5 @@
+---
+title: Support socket paths for praefect
+merge_request: 1115
+author:
+type: added
diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index a6d99e080..0661081e7 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -34,13 +34,12 @@ func main() {
 		logger.Fatal(err)
 	}
 
-	l, err := net.Listen("tcp", conf.ListenAddr)
+	listeners, err := getListeners(conf.SocketPath, conf.ListenAddr)
 	if err != nil {
 		logger.Fatalf("%s", err)
 	}
 
-	logger.WithField("address", conf.ListenAddr).Info("listening at tcp address")
-	logger.Fatalf("%v", run(l, conf))
+	logger.Fatalf("%v", run(listeners, conf))
 }
 
 func configure() (config.Config, error) {
@@ -75,7 +74,7 @@ func configure() (config.Config, error) {
 	return conf, nil
 }
 
-func run(l net.Listener, conf config.Config) error {
+func run(listeners []net.Listener, conf config.Config) error {
 	srv := praefect.NewServer(nil, logger)
 
 	signals := []os.Signal{syscall.SIGTERM, syscall.SIGINT}
@@ -83,7 +82,10 @@ func run(l net.Listener, conf config.Config) error {
 	signal.Notify(termCh, signals...)
 
 	serverErrors := make(chan error, 1)
-	go func() { serverErrors <- srv.Start(l) }()
+
+	for _, l := range listeners {
+		go func(lis net.Listener) { serverErrors <- srv.Start(lis) }(l)
+	}
 
 	for _, gitaly := range conf.GitalyServers {
 		srv.RegisterNode(gitaly.Name, gitaly.ListenAddr)
@@ -106,3 +108,33 @@ func run(l net.Listener, conf config.Config) error {
 
 	return err
 }
+
+func getListeners(socketPath, listenAddr string) ([]net.Listener, error) {
+	var listeners []net.Listener
+
+	if socketPath != "" {
+		if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+			return nil, err
+		}
+		l, err := net.Listen("unix", socketPath)
+		if err != nil {
+			return nil, err
+		}
+
+		listeners = append(listeners, l)
+
+		logger.WithField("address", socketPath).Info("listening on unix socket")
+	}
+
+	if listenAddr != "" {
+		l, err := net.Listen("tcp", listenAddr)
+		if err != nil {
+			return nil, err
+		}
+
+		listeners = append(listeners, l)
+		logger.WithField("address", listenAddr).Info("listening at tcp address")
+	}
+
+	return listeners, nil
+}
diff --git a/config.praefect.toml.example b/config.praefect.toml.example
index 9f041a3f3..b6b8662a9 100644
--- a/config.praefect.toml.example
+++ b/config.praefect.toml.example
@@ -1,10 +1,9 @@
 # Example Praefect configuration file
 
 # # TCP address to listen on
-listen_addr = "127.0.0.1:2305"
-# socket_path = "/home/git/gitlab/tmp/sockets/private/praefect.socket"
-#
-prometheus_listen_addr = "127.0.0.1:10101"
+# listen_addr = "127.0.0.1:2305"
+socket_path = "/home/git/gitlab/tmp/sockets/private/praefect.socket"
+# prometheus_listen_addr = "127.0.0.1:10101"
 
 # # You can optionally configure Praefect to output JSON-formatted log messages to stdout
 # [logging]
diff --git a/doc/README.md b/doc/README.md
index 8a56249f0..65217c61c 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -21,3 +21,5 @@ Gitaly does not replicate any data. If a Gitaly server goes down, any of its
 clients can't read or write to the repositories stored on that server. This
 means that Gitaly is not highly available. How this will be solved is described
 [in the HA design document](doc/design_ha.md)
+
+For configuration please read [praefects configuration documentation](doc/configuration/praefect.md).
diff --git a/doc/configuration/praefect.md b/doc/configuration/praefect.md
new file mode 100644
index 000000000..83f554662
--- /dev/null
+++ b/doc/configuration/praefect.md
@@ -0,0 +1,32 @@
+# Configuring Praefect
+
+This document describes how to configure the praefect server.
+
+Praefect is configured via a [TOML](https://github.com/toml-lang/toml)
+configuration file. The TOML file contents and location depends on how you 
+installed GitLab. See: https://docs.gitlab.com/ce/administration/gitaly/
+
+The configuration file is passed as an argument to the `praefect`
+executable. This is usually done by either omnibus-gitlab or your init
+script.
+
+```
+gitaly -config /path/to/config.toml
+```
+
+## Format
+
+```toml
+listen_addr = "127.0.0.1:2305"
+socket_path = "/path/to/praefect.socket"
+
+[logging]
+format = "json"
+level = "info"
+
+[[gitaly_server]]
+name = "default"
+listen_addr = "tcp://localhost:9999"
+```
+
+An example [config toml](config.praefect.toml) is stored in this repository.
diff --git a/internal/praefect/config/config.go b/internal/praefect/config/config.go
index 0aecae847..096c946c0 100644
--- a/internal/praefect/config/config.go
+++ b/internal/praefect/config/config.go
@@ -10,10 +10,12 @@ import (
 
 // Config is a container for everything found in the TOML config file
 type Config struct {
-	ListenAddr           string          `toml:"listen_addr" split_words:"true"`
-	GitalyServers        []*GitalyServer `toml:"gitaly_server", split_words:"true"`
-	Logging              config.Logging  `toml:"logging"`
-	PrometheusListenAddr string          `toml:"prometheus_listen_addr", split_words:"true"`
+	ListenAddr    string          `toml:"listen_addr" split_words:"true"`
+	SocketPath    string          `toml:"socket_path" split_words:"true"`
+	GitalyServers []*GitalyServer `toml:"gitaly_server", split_words:"true"`
+
+	Logging              config.Logging `toml:"logging"`
+	PrometheusListenAddr string         `toml:"prometheus_listen_addr", split_words:"true"`
 }
 
 // GitalyServer allows configuring the servers that RPCs are proxied to
@@ -36,7 +38,7 @@ func FromFile(filePath string) (Config, error) {
 }
 
 var (
-	errNoListenAddr        = errors.New("no listen address configured")
+	errNoListener          = errors.New("no listen address or socket path configured")
 	errNoGitalyServers     = errors.New("no gitaly backends configured")
 	errDuplicateGitalyAddr = errors.New("gitaly listen addresses are not unique")
 	errGitalyWithoutName   = errors.New("all gitaly servers must have a name")
@@ -44,8 +46,8 @@ var (
 
 // Validate establishes if the config is valid
 func (c Config) Validate() error {
-	if c.ListenAddr == "" {
-		return errNoListenAddr
+	if c.ListenAddr == "" && c.SocketPath == "" {
+		return errNoListener
 	}
 
 	if len(c.GitalyServers) == 0 {
diff --git a/internal/praefect/config/config_test.go b/internal/praefect/config/config_test.go
index d896879b3..306650771 100644
--- a/internal/praefect/config/config_test.go
+++ b/internal/praefect/config/config_test.go
@@ -15,9 +15,14 @@ func TestConfigValidation(t *testing.T) {
 		err    error
 	}{
 		{
-			desc:   "No ListenAddr",
+			desc:   "No ListenAddr or SocketPath",
 			config: Config{ListenAddr: "", GitalyServers: gitalySrvs},
-			err:    errNoListenAddr,
+			err:    errNoListener,
+		},
+		{
+			desc:   "Only a SocketPath",
+			config: Config{SocketPath: "/tmp/praefect.socket", GitalyServers: gitalySrvs},
+			err:    nil,
 		},
 		{
 			desc:   "No servers",