Add Postgres config support to Praefect

11 files changed, 304 insertions, 2 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aedc47ad2..922b59778 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -188,3 +188,18 @@ license_management:
     - sudo apt-get install -y libicu-dev libgit2-dev cmake
   variables:
     LICENSE_FINDER_CLI_OPTS: '--aggregate-paths=. ruby'
+
+praefect_sql_connect:
+  <<: *go_test_definition
+  services:
+    - postgres:9.6
+  variables:
+    POSTGRES_DB: praefect_test
+    POSTGRES_USER: praefect
+    POSTGRES_PASSWORD: sql-password
+  script:
+    - make
+    # Sanity check: direct ping with psql
+    - PGPASSWORD=$POSTGRES_PASSWORD psql -h postgres -U $POSTGRES_USER -d $POSTGRES_DB -c 'select now()'
+    - ruby -rerb -e 'ERB.new(ARGF.read).run' _support/config.praefect.toml.ci-sql-test.erb > config.praefect.toml
+    - ./praefect -config config.praefect.toml sql-ping
diff --git a/NOTICE b/NOTICE
index a00b82a2d..01e9f6095 100644
--- a/NOTICE
+++ b/NOTICE
@@ -904,6 +904,16 @@ The above copyright notice and this permission notice shall be included in all c
 
 THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+LICENSE.md - gitlab.com/gitlab-org/gitaly/vendor/github.com/lib/pq
+Copyright (c) 2011-2013, 'pq' Contributors
+Portions Copyright (C) 2011 Blake Mizerany
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 LICENSE - gitlab.com/gitlab-org/gitaly/vendor/github.com/libgit2/git2go
 The MIT License
 
diff --git a/README.md b/README.md
index 018c7f973..c156914b4 100644
--- a/README.md
+++ b/README.md
@@ -107,6 +107,35 @@ use library code from the
 [gitlab.com/gitlab-org/gitaly/client](https://gitlab.com/gitlab-org/gitaly/tree/master/client)
 package.
 
+## High Availability
+
+We are working on a high-availability (HA) solution for Gitaly based on
+asynchronous replication. A Gitaly server would be made highly available
+by assigning one or more standby servers ("secondaries") to it, each of
+which contains a full copy of all the repository data on the primary
+Gitaly server.
+
+To implement this we are building a new GitLab component called
+Praefect, which is hosted alongside the rest of Gitaly in this
+repository. As we currently envision it, Praefect will have four
+responsibilities:
+
+-   route RPC traffic to the primary Gitaly server
+-   inspect RPC traffic and mark repositories as dirty if the RPC is a
+    "mutator"
+-   ensure dirty repositories have their changes replicated to the
+    secondary Gitaly servers
+-   in the event of a failure on the primary, demote it to secondary and
+    elect a new primary
+
+Praefect has internal state: it needs to be able to "remember" which
+repositories are in need of replication, and which Gitaly server is the
+primary. [We will use Postgres to store Praefect's internal state](doc/proposals/praefect-queue-storage.md).
+
+As of December 2019 we are busy rolling out the Postgres integration in
+Praefect. The minimum supported Postgres version is 9.6, just like the
+rest of GitLab.
+
 ## Further reading
 
 More about the project and its processes is [detailed in the docs](doc/README.md).
diff --git a/_support/config.praefect.toml.ci-sql-test.erb b/_support/config.praefect.toml.ci-sql-test.erb
new file mode 100755
index 000000000..3aabeed6f
--- /dev/null
+++ b/_support/config.praefect.toml.ci-sql-test.erb
@@ -0,0 +1,59 @@
+# Example Praefect configuration file
+
+# # TCP address to listen on
+listen_addr = "127.0.0.1:2305"
+
+# # Praefect can listen on a socket when placed on the same machine as all clients
+# socket_path = "/home/git/gitlab/tmp/sockets/private/praefect.socket"
+# # Optional: export metrics via Prometheus
+# prometheus_listen_addr = "127.0.01:10101"
+# # You can optionally configure Praefect to output JSON-formatted log messages to stdout
+# [logging]
+#   format = "json"
+# # Optional: Set log level to only log entries with that severity or above
+# # One of, in order: debug, info, warn, errror, fatal, panic
+# # Defaults to "info"
+#   level = "warn"
+# [sentry]
+#   sentry_environment = ""
+#   sentry_dsn = ""
+#
+#  Optional: authenticate Gitaly requests using a shared secret. This token works the same way as a gitaly token
+# [auth]
+#   token = 'abc123secret'
+#
+# # One or more Gitaly servers need to be configured to be managed. The names
+# of each server are used to link multiple nodes, or `gitaly_server`s together
+# as shard. listen_addr should be unique for all nodes.
+# Requires the protocol to be defined, e.g. tcp://host.tld:1234
+
+[[virtual_storage]]
+name = "praefect"
+
+[[virtual_storage.node]]
+  storage = "praefect-git-0"
+  address = "tcp://praefect-git-0.internal"
+  primary = true
+  token = 'token1'
+
+[[virtual_storage.node]]
+  storage = "praefect-git-1"
+  address = "tcp://praefect-git-1.internal"
+  token = 'token2'
+
+[[virtual_storage.node]]
+  storage = "praefect-git-2"
+  address = "tcp://praefect-git-2.internal"
+  token = 'token3'
+
+[database]
+# In CI this magical hostname 'postgres' points to our dedicated CI postgres instance.
+host = 'postgres'
+
+# POSTGRES_* variables are defined in gitlab-ci.yml
+dbname = '<%= ENV['POSTGRES_DB'] %>'
+user = '<%= ENV['POSTGRES_USER'] %>'
+password = '<%= ENV['POSTGRES_PASSWORD'] %>'
+
+# No SSL in CI
+sslmode = 'disable'
diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index b667273d7..d9a454850 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/sirupsen/logrus"
 	"gitlab.com/gitlab-org/gitaly/internal/bootstrap"
 	"gitlab.com/gitlab-org/gitaly/internal/bootstrap/starter"
 	"gitlab.com/gitlab-org/gitaly/internal/config/sentry"
@@ -30,6 +31,8 @@ var (
 	errNoConfigFile = errors.New("the config flag must be passed")
 )
 
+const progname = "praefect"
+
 func main() {
 	flag.Parse()
 
@@ -44,6 +47,12 @@ func main() {
 		logger.Fatal(err)
 	}
 
+	if args := flag.Args(); len(args) > 0 {
+		os.Exit(subCommand(conf, args[0], args[1:]))
+	}
+
+	logger.WithField("version", praefect.GetVersionString()).Info("Starting Praefect")
+
 	starterConfigs, err := getStarterConfigs(conf.SocketPath, conf.ListenAddr)
 	if err != nil {
 		logger.Fatalf("%s", err)
@@ -86,8 +95,6 @@ func configure() (config.Config, error) {
 		}()
 	}
 
-	logger.WithField("version", praefect.GetVersionString()).Info("Starting Praefect")
-
 	sentry.ConfigureSentry(version.GetVersion(), conf.Sentry)
 
 	return conf, nil
@@ -118,6 +125,8 @@ func run(cfgs []starter.Config, conf config.Config) error {
 		serverErrors = make(chan error, 1)
 	)
 
+	testSQLConnection(logger, conf)
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -169,3 +178,36 @@ func getStarterConfigs(socketPath, listenAddr string) ([]starter.Config, error)
 
 	return cfgs, nil
 }
+
+// Test Postgres connection, for diagnostic purposes only while we roll
+// out Postgres support. https://gitlab.com/gitlab-org/gitaly/issues/1755
+func testSQLConnection(logger *logrus.Entry, conf config.Config) {
+	if err := datastore.CheckPostgresVersion(conf); err != nil {
+		logger.WithError(err).Error("SQL connection check failed")
+	} else {
+		logger.Info("SQL connection check successful")
+	}
+}
+
+// subCommand returns an exit code, to be fed into os.Exit.
+func subCommand(conf config.Config, arg0 string, argRest []string) int {
+	switch arg0 {
+	case "sql-ping":
+		return sqlPing(conf)
+	default:
+		fmt.Printf("%s: unknown subcommand: %q\n", progname, arg0)
+		return 1
+	}
+}
+
+func sqlPing(conf config.Config) int {
+	const subCmd = progname + " sql-ping"
+
+	if err := datastore.CheckPostgresVersion(conf); err != nil {
+		fmt.Printf("%s: fail: %v\n", subCmd, err)
+		return 1
+	}
+
+	fmt.Printf("%s: OK\n", subCmd)
+	return 0
+}
diff --git a/go.mod b/go.mod
index 1b444e5cc..3dfe1487c 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/kelseyhightower/envconfig v1.3.0
+	github.com/lib/pq v1.2.0
 	github.com/libgit2/git2go v0.0.0-20190104134018-ecaeb7a21d47
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/procfs v0.0.3 // indirect
diff --git a/go.sum b/go.sum
index aa4208afb..ffc6b5322 100644
--- a/go.sum
+++ b/go.sum
@@ -103,6 +103,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/labstack/echo/v4 v4.1.10/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/libgit2/git2go v0.0.0-20190104134018-ecaeb7a21d47 h1:HDt7WT3kpXSHq4mlOuLzgXH9LeOK1qlhyFdKIAzxxeM=
 github.com/libgit2/git2go v0.0.0-20190104134018-ecaeb7a21d47/go.mod h1:4bKN42efkbNYMZlvDfxGDxzl066GhpvIircZDsm8Y+Y=
 github.com/lightstep/lightstep-tracer-go v0.15.6 h1:D0GGa7afJ7GcQvu5as6ssLEEKYXvRgKI5d5cevtz8r4=
diff --git a/internal/praefect/config/config.go b/internal/praefect/config/config.go
index 7c2c3daf6..1d1d51a6f 100644
--- a/internal/praefect/config/config.go
+++ b/internal/praefect/config/config.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/BurntSushi/toml"
 
@@ -28,6 +29,7 @@ type Config struct {
 	PrometheusListenAddr string            `toml:"prometheus_listen_addr"`
 	Prometheus           prometheus.Config `toml:"prometheus"`
 	Auth                 auth.Config       `toml:"auth"`
+	DB                   `toml:"database"`
 }
 
 // VirtualStorage represents a set of nodes for a storage
@@ -140,3 +142,46 @@ func (c Config) Validate() error {
 
 	return nil
 }
+
+// DB holds Postgres client configuration data.
+type DB struct {
+	Host        string `toml:"host"`
+	Port        int    `toml:"port"`
+	User        string `toml:"user"`
+	Password    string `toml:"password"`
+	DBName      string `toml:"dbname"`
+	SSLMode     string `toml:"sslmode"`
+	SSLCert     string `toml:"sslcert"`
+	SSLKey      string `toml:"sslkey"`
+	SSLRootCert string `toml:"sslrootcert"`
+}
+
+// ToPQString returns a connection string that can be passed to github.com/lib/pq.
+func (db DB) ToPQString() string {
+	var fields []string
+	if db.Port > 0 {
+		fields = append(fields, fmt.Sprintf("port=%d", db.Port))
+	}
+
+	for _, kv := range []struct{ key, value string }{
+		{"host", db.Host},
+		{"user", db.User},
+		{"password", db.Password},
+		{"dbname", db.DBName},
+		{"sslmode", db.SSLMode},
+		{"sslcert", db.SSLCert},
+		{"sslkey", db.SSLKey},
+		{"sslrootcert", db.SSLRootCert},
+	} {
+		if len(kv.value) == 0 {
+			continue
+		}
+
+		kv.value = strings.ReplaceAll(kv.value, "'", `\'`)
+		kv.value = strings.ReplaceAll(kv.value, " ", `\ `)
+
+		fields = append(fields, kv.key+"="+kv.value)
+	}
+
+	return strings.Join(fields, " ")
+}
diff --git a/internal/praefect/config/config_test.go b/internal/praefect/config/config_test.go
index 63d2ed06d..81f62b863 100644
--- a/internal/praefect/config/config_test.go
+++ b/internal/praefect/config/config_test.go
@@ -168,6 +168,17 @@ func TestConfigParsing(t *testing.T) {
 				Prometheus: gitaly_prometheus.Config{
 					GRPCLatencyBuckets: []float64{0.1, 0.2, 0.3},
 				},
+				DB: DB{
+					Host:        "1.2.3.4",
+					Port:        5432,
+					User:        "praefect",
+					Password:    "db-secret",
+					DBName:      "praefect_production",
+					SSLMode:     "require",
+					SSLCert:     "/path/to/cert",
+					SSLKey:      "/path/to/key",
+					SSLRootCert: "/path/to/root-cert",
+				},
 			},
 		},
 		//TODO: Remove this test, as well as the fixture in testdata/single-virtual-storage.config.toml
@@ -215,3 +226,41 @@ func TestConfigParsing(t *testing.T) {
 		})
 	}
 }
+
+func TestToPQString(t *testing.T) {
+	testCases := []struct {
+		desc string
+		in   DB
+		out  string
+	}{
+		{desc: "empty", in: DB{}, out: ""},
+		{
+			desc: "basic example",
+			in: DB{
+				Host:        "1.2.3.4",
+				Port:        2345,
+				User:        "praefect-user",
+				Password:    "secret",
+				DBName:      "praefect_production",
+				SSLMode:     "require",
+				SSLCert:     "/path/to/cert",
+				SSLKey:      "/path/to/key",
+				SSLRootCert: "/path/to/root-cert",
+			},
+			out: `port=2345 host=1.2.3.4 user=praefect-user password=secret dbname=praefect_production sslmode=require sslcert=/path/to/cert sslkey=/path/to/key sslrootcert=/path/to/root-cert`,
+		},
+		{
+			desc: "with spaces and quotes",
+			in: DB{
+				Password: "secret foo'bar",
+			},
+			out: `password=secret\ foo\'bar`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			require.Equal(t, tc.out, tc.in.ToPQString())
+		})
+	}
+}
diff --git a/internal/praefect/config/testdata/config.toml b/internal/praefect/config/testdata/config.toml
index e57aa6314..0fc25ea9d 100644
--- a/internal/praefect/config/testdata/config.toml
+++ b/internal/praefect/config/testdata/config.toml
@@ -28,3 +28,14 @@ name = "praefect"
 
 [prometheus]
   grpc_latency_buckets = [0.1, 0.2, 0.3] 
+
+[database]
+host = "1.2.3.4"
+port = 5432
+user = "praefect"
+password = "db-secret"
+dbname = "praefect_production"
+sslmode = "require"
+sslcert = "/path/to/cert"
+sslkey = "/path/to/key"
+sslrootcert = "/path/to/root-cert"
diff --git a/internal/praefect/datastore/db.go b/internal/praefect/datastore/db.go
new file mode 100644
index 000000000..2771326f6
--- /dev/null
+++ b/internal/praefect/datastore/db.go
@@ -0,0 +1,39 @@
+package datastore
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	// Blank import to enable integration of github.com/lib/pq into database/sql
+	_ "github.com/lib/pq"
+
+	"gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+)
+
+// CheckPostgresVersion checks the server version of the Postgres DB
+// specified in conf. This is a diagnostic for the Praefect Postgres
+// rollout. https://gitlab.com/gitlab-org/gitaly/issues/1755
+func CheckPostgresVersion(conf config.Config) error {
+	db, err := sql.Open("postgres", conf.DB.ToPQString())
+	if err != nil {
+		return fmt.Errorf("sql open: %v", err)
+	}
+	defer db.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	var serverVersion int
+	if err := db.QueryRowContext(ctx, "SHOW server_version_num").Scan(&serverVersion); err != nil {
+		return fmt.Errorf("get postgres server version: %v", err)
+	}
+
+	const minimumServerVersion = 90600 // Postgres 9.6
+	if serverVersion < minimumServerVersion {
+		return fmt.Errorf("postgres server version too old: %d", serverVersion)
+	}
+
+	return nil
+}