Zero downtime deployment

21 files changed, 1623 insertions, 108 deletions

diff --git a/.gitignore b/.gitignore
index 08ae671a8..24aefa7e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,8 @@
 /gitaly
 cmd/gitaly-ssh/gitaly-ssh
 /gitaly-ssh
+cmd/gitaly-wrapper/gitaly-wrapper
+/gitaly-wrapper
 **/testdata/gitaly-libexec/
 /*.deb
 /_support/package/bin
@@ -17,3 +19,4 @@ cmd/gitaly-ssh/gitaly-ssh
 git-env
 /gitaly-debug
 /praefect
+gitaly.pid
diff --git a/NOTICE b/NOTICE
index f8cbd6d4d..f420323ea 100644
--- a/NOTICE
+++ b/NOTICE
@@ -151,6 +151,19 @@ This Source Code Form is subject to the terms of the Mozilla Public License,
 v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
 one at http://mozilla.org/MPL/2.0/.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+LICENSE - gitlab.com/gitlab-org/gitaly/vendor/github.com/cloudflare/tableflip
+Copyright (c) 2017-2018, Cloudflare. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 LICENSE - gitlab.com/gitlab-org/gitaly/vendor/github.com/davecgh/go-spew
 ISC License
 
diff --git a/changelogs/unreleased/zero-downtime-deployment.yml b/changelogs/unreleased/zero-downtime-deployment.yml
new file mode 100644
index 000000000..592d3560b
--- /dev/null
+++ b/changelogs/unreleased/zero-downtime-deployment.yml
@@ -0,0 +1,5 @@
+---
+title: Zero downtime deployment
+merge_request: 1133
+author:
+type: added
diff --git a/cmd/gitaly-wrapper/main.go b/cmd/gitaly-wrapper/main.go
new file mode 100644
index 000000000..646661b47
--- /dev/null
+++ b/cmd/gitaly-wrapper/main.go
@@ -0,0 +1,147 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"gitlab.com/gitlab-org/gitaly/internal/config"
+)
+
+const (
+	envJSONLogging = "WRAPPER_JSON_LOGGING"
+)
+
+func main() {
+	if jsonLogging() {
+		logrus.SetFormatter(&logrus.JSONFormatter{})
+	}
+
+	if len(os.Args) < 2 {
+		logrus.Fatalf("usage: %s forking_binary [args]", os.Args[0])
+	}
+
+	gitalyBin, gitalyArgs := os.Args[1], os.Args[2:]
+
+	log := logrus.WithField("wrapper", os.Getpid())
+	log.Info("Wrapper started")
+
+	if pidFile() == "" {
+		log.Fatalf("missing pid file ENV variable %q", config.EnvPidFile)
+	}
+
+	log.WithField("pid_file", pidFile()).Info("finding gitaly")
+	gitaly, err := findGitaly()
+	if err != nil {
+		log.WithError(err).Fatal("find gitaly")
+	}
+
+	if gitaly != nil {
+		log.Info("adopting a process")
+	} else {
+		log.Info("spawning a process")
+
+		proc, err := spawnGitaly(gitalyBin, gitalyArgs)
+		if err != nil {
+			log.WithError(err).Fatal("spawn gitaly")
+		}
+
+		gitaly = proc
+	}
+
+	log = log.WithField("gitaly", gitaly.Pid)
+	log.Info("monitoring gitaly")
+
+	forwardSignals(gitaly, log)
+
+	// wait
+	for isAlive(gitaly) {
+		time.Sleep(1 * time.Second)
+	}
+
+	log.Error("wrapper for gitaly shutting down")
+}
+
+func findGitaly() (*os.Process, error) {
+	pid, err := getPid()
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	// os.FindProcess on unix do not return an error if the process does not exist
+	gitaly, err := os.FindProcess(pid)
+	if err != nil {
+		return nil, err
+	}
+
+	if isAlive(gitaly) {
+		return gitaly, nil
+	}
+
+	return nil, nil
+}
+
+func spawnGitaly(bin string, args []string) (*os.Process, error) {
+	cmd := exec.Command(bin, args...)
+	cmd.Env = append(os.Environ(), fmt.Sprintf("%s=true", config.EnvUpgradesEnabled))
+
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	// This cmd.Wait() is crucial. Without it we cannot detect if the command we just spawned has crashed.
+	go cmd.Wait()
+
+	return cmd.Process, nil
+}
+
+func forwardSignals(gitaly *os.Process, log *logrus.Entry) {
+	sigs := make(chan os.Signal, 1)
+	go func() {
+		for sig := range sigs {
+			log.WithField("signal", sig).Warning("forwarding signal")
+
+			if err := gitaly.Signal(sig); err != nil {
+				log.WithField("signal", sig).WithError(err).Error("can't forward the signal")
+			}
+
+		}
+	}()
+
+	signal.Notify(sigs)
+}
+
+func getPid() (int, error) {
+	data, err := ioutil.ReadFile(pidFile())
+	if err != nil {
+		return 0, err
+	}
+
+	return strconv.Atoi(string(data))
+}
+
+func isAlive(p *os.Process) bool {
+	// After p exits, and after it gets reaped, this p.Signal will fail. It is crucial that p gets reaped.
+	// If p was spawned by the current process, it will get reaped from a goroutine that does cmd.Wait().
+	// If p was spawned by someone else we rely on them to reap it, or on p to become an orphan.
+	// In the orphan case p should get reaped by the OS (PID 1).
+	return p.Signal(syscall.Signal(0)) == nil
+}
+
+func pidFile() string {
+	return os.Getenv(config.EnvPidFile)
+}
+
+func jsonLogging() bool {
+	return os.Getenv(envJSONLogging) == "true"
+}
diff --git a/cmd/gitaly/bootstrap.go b/cmd/gitaly/bootstrap.go
new file mode 100644
index 000000000..d3b464052
--- /dev/null
+++ b/cmd/gitaly/bootstrap.go
@@ -0,0 +1,205 @@
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/cloudflare/tableflip"
+	log "github.com/sirupsen/logrus"
+	"gitlab.com/gitlab-org/gitaly/internal/config"
+	"gitlab.com/gitlab-org/gitaly/internal/connectioncounter"
+	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
+	"gitlab.com/gitlab-org/gitaly/internal/server"
+	"google.golang.org/grpc"
+)
+
+type bootstrap struct {
+	*tableflip.Upgrader
+
+	insecureListeners []net.Listener
+	secureListeners   []net.Listener
+
+	serversErrors chan error
+}
+
+// newBootstrap performs tableflip initialization
+//
+// first boot:
+// * gitaly starts as usual, we will refer to it as p1
+// * newBootstrap will build a tableflip.Upgrader, we will refer to it as upg
+// * sockets and files must be opened with upg.Fds
+// * p1 will trap SIGHUP and invoke upg.Upgrade()
+// * when ready to accept incoming connections p1 will call upg.Ready()
+// * upg.Exit() channel will be closed when an upgrades completed successfully and the process must terminate
+//
+// graceful upgrade:
+// * user replaces gitaly binary and/or config file
+// * user sends SIGHUP to p1
+// * p1 will fork and exec the new gitaly, we will refer to it as p2
+// * from now on p1 will ignore other SIGHUP
+// * if p2 terminates with a non-zero exit code, SIGHUP handling will be restored
+// * p2 will follow the "first boot" sequence but upg.Fds will provide sockets and files from p1, when available
+// * when p2 invokes upg.Ready() all the shared file descriptors not claimed by p2 will be closed
+// * upg.Exit() channel in p1 will be closed now and p1 can gracefully terminate already accepted connections
+// * upgrades cannot starts again if p1 and p2 are both running, an hard termination should be scheduled to overcome
+//   freezes during a graceful shutdown
+func newBootstrap(pidFile string, upgradesEnabled bool) (*bootstrap, error) {
+	// PIDFile is optional, if provided tableflip will keep it updated
+	upg, err := tableflip.New(tableflip.Options{PIDFile: pidFile})
+	if err != nil {
+		return nil, err
+	}
+
+	if upgradesEnabled {
+		go func() {
+			sig := make(chan os.Signal, 1)
+			signal.Notify(sig, syscall.SIGHUP)
+
+			for range sig {
+				err := upg.Upgrade()
+				if err != nil {
+					log.WithError(err).Error("Upgrade failed")
+					continue
+				}
+
+				log.Info("Upgrade succeeded")
+			}
+		}()
+	}
+
+	return &bootstrap{Upgrader: upg}, nil
+}
+
+func (b *bootstrap) listen() error {
+	if socketPath := config.Config.SocketPath; socketPath != "" {
+		l, err := b.createUnixListener(socketPath)
+		if err != nil {
+			return err
+		}
+
+		log.WithField("address", socketPath).Info("listening on unix socket")
+		b.insecureListeners = append(b.insecureListeners, l)
+	}
+
+	if addr := config.Config.ListenAddr; addr != "" {
+		l, err := b.Fds.Listen("tcp", addr)
+		if err != nil {
+			return err
+		}
+
+		log.WithField("address", addr).Info("listening at tcp address")
+		b.insecureListeners = append(b.insecureListeners, connectioncounter.New("tcp", l))
+	}
+
+	if addr := config.Config.TLSListenAddr; addr != "" {
+		tlsListener, err := b.Fds.Listen("tcp", addr)
+		if err != nil {
+			return err
+		}
+
+		b.secureListeners = append(b.secureListeners, connectioncounter.New("tls", tlsListener))
+	}
+
+	b.serversErrors = make(chan error, len(b.insecureListeners)+len(b.secureListeners))
+
+	return nil
+}
+
+func (b *bootstrap) prometheusListener() (net.Listener, error) {
+	log.WithField("address", config.Config.PrometheusListenAddr).Info("starting prometheus listener")
+
+	return b.Fds.Listen("tcp", config.Config.PrometheusListenAddr)
+}
+
+func (b *bootstrap) run() {
+	signals := []os.Signal{syscall.SIGTERM, syscall.SIGINT}
+	done := make(chan os.Signal, len(signals))
+	signal.Notify(done, signals...)
+
+	ruby, err := rubyserver.Start()
+	if err != nil {
+		log.WithError(err).Error("start ruby server")
+		return
+	}
+	defer ruby.Stop()
+
+	if len(b.insecureListeners) > 0 {
+		insecureServer := server.NewInsecure(ruby)
+		defer insecureServer.Stop()
+
+		serve(insecureServer, b.insecureListeners, b.Exit(), b.serversErrors)
+	}
+
+	if len(b.secureListeners) > 0 {
+		secureServer := server.NewSecure(ruby)
+		defer secureServer.Stop()
+
+		serve(secureServer, b.secureListeners, b.Exit(), b.serversErrors)
+	}
+
+	if err := b.Ready(); err != nil {
+		log.WithError(err).Error("incomplete bootstrap")
+		return
+	}
+
+	select {
+	case <-b.Exit():
+		// this is the old process and a graceful upgrade is in progress
+		// the new process signaled its readiness and we started a graceful stop
+		// however no further upgrades can be started until this process is running
+		// we set a grace period and then we force a termination.
+		b.waitGracePeriod(done)
+
+		err = fmt.Errorf("graceful upgrade")
+	case s := <-done:
+		err = fmt.Errorf("received signal %q", s)
+	case err = <-b.serversErrors:
+	}
+
+	log.WithError(err).Error("terminating")
+}
+
+func (b *bootstrap) waitGracePeriod(kill <-chan os.Signal) {
+	log.WithField("graceful_restart_timeout", config.Config.GracefulRestartTimeout).Warn("starting grace period")
+
+	select {
+	case <-time.After(config.Config.GracefulRestartTimeout):
+		log.Error("old process stuck on termination. Grace period expired.")
+	case <-kill:
+		log.Error("force shutdown")
+	case <-b.serversErrors:
+		log.Info("graceful stop completed")
+	}
+}
+
+func (b *bootstrap) createUnixListener(socketPath string) (net.Listener, error) {
+	if !b.HasParent() {
+		// During an update the unix socket exists and if we delete it tableflip will not create a new one
+		if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+			return nil, err
+		}
+	}
+
+	l, err := b.Fds.Listen("unix", socketPath)
+	return connectioncounter.New("unix", l), err
+}
+
+func serve(server *grpc.Server, listeners []net.Listener, done <-chan struct{}, errors chan<- error) {
+	go func() {
+		<-done
+
+		server.GracefulStop()
+	}()
+
+	for _, listener := range listeners {
+		// Must pass the listener as a function argument because there is a race
+		// between 'go' and 'for'.
+		go func(l net.Listener) {
+			errors <- server.Serve(l)
+		}(listener)
+	}
+}
diff --git a/cmd/gitaly/bootstrap_test.go b/cmd/gitaly/bootstrap_test.go
new file mode 100644
index 000000000..f6fdf6f90
--- /dev/null
+++ b/cmd/gitaly/bootstrap_test.go
@@ -0,0 +1,141 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// b is global because tableflip do not allow to init more than one Upgrader per process
+var b *bootstrap
+var socketPath = path.Join(os.TempDir(), "test-unix-socket")
+
+// TestMain helps testing bootstrap.
+// When invoked directly it behaves like a normal go test, but if a test performs an upgrade the children will
+// avoid the test suite and start a pid HTTP server on socketPath
+func TestMain(m *testing.M) {
+	var err error
+	b, err = newBootstrap("", true)
+	if err != nil {
+		panic(err)
+	}
+
+	if !b.HasParent() {
+		// Execute test suite if there is no parent.
+		os.Exit(m.Run())
+	}
+
+	// this is a test suite that triggered an upgrade, we are in the children here
+	l, err := b.createUnixListener(socketPath)
+	if err != nil {
+		panic(err)
+	}
+
+	if err := b.Ready(); err != nil {
+		panic(err)
+	}
+
+	done := make(chan struct{})
+	srv := startPidServer(done, l)
+
+	select {
+	case <-done:
+	//no op
+	case <-time.After(2 * time.Minute):
+		srv.Close()
+		panic("safeguard against zombie process")
+	}
+}
+
+func TestCreateUnixListener(t *testing.T) {
+	// simulate a dangling socket
+	if err := os.Remove(socketPath); err != nil {
+		require.True(t, os.IsNotExist(err), "cannot delete dangling socket: %v", err)
+	}
+
+	file, err := os.OpenFile(socketPath, os.O_CREATE, 0755)
+	require.NoError(t, err)
+	require.NoError(t, file.Close())
+
+	require.NoError(t, ioutil.WriteFile(socketPath, nil, 0755))
+
+	l, err := b.createUnixListener(socketPath)
+	require.NoError(t, err)
+
+	done := make(chan struct{})
+	srv := startPidServer(done, l)
+	defer srv.Close()
+
+	require.NoError(t, b.Ready(), "not ready")
+
+	myPid, err := askPid()
+	require.NoError(t, err)
+	require.Equal(t, os.Getpid(), myPid)
+
+	// we trigger an upgrade and wait for children readiness
+	require.NoError(t, b.Upgrade(), "upgrade failed")
+	<-b.Exit()
+	require.NoError(t, srv.Close())
+	<-done
+
+	childPid, err := askPid()
+	require.NoError(t, err)
+	require.NotEqual(t, os.Getpid(), childPid, "this request must be handled by the children")
+}
+
+func askPid() (int, error) {
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", socketPath)
+			},
+		},
+	}
+
+	response, err := client.Get("http://unix")
+	if err != nil {
+		return 0, err
+	}
+	defer response.Body.Close()
+
+	pid, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return 0, err
+	}
+
+	return strconv.Atoi(string(pid))
+}
+
+// startPidServer starts an HTTP server that returns the current PID, if running on a children it will kill itself after serving
+// the first client
+func startPidServer(done chan<- struct{}, l net.Listener) *http.Server {
+	mux := http.NewServeMux()
+	srv := &http.Server{Handler: mux}
+
+	mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
+		io.WriteString(w, fmt.Sprint(os.Getpid()))
+
+		if b.HasParent() {
+			time.AfterFunc(1*time.Second, func() { srv.Close() })
+		}
+	})
+
+	go func() {
+		if err := srv.Serve(l); err != http.ErrServerClosed {
+			fmt.Printf("Serve error: %v", err)
+		}
+		close(done)
+	}()
+
+	return srv
+}
diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go
index 3be1e38e8..10149365f 100644
--- a/cmd/gitaly/main.go
+++ b/cmd/gitaly/main.go
@@ -3,20 +3,15 @@ package main
 import (
 	"flag"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
-	"os/signal"
-	"syscall"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"gitlab.com/gitlab-org/gitaly/internal/config"
-	"gitlab.com/gitlab-org/gitaly/internal/connectioncounter"
 	"gitlab.com/gitlab-org/gitaly/internal/git"
 	"gitlab.com/gitlab-org/gitaly/internal/linguist"
-	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
 	"gitlab.com/gitlab-org/gitaly/internal/server"
 	"gitlab.com/gitlab-org/gitaly/internal/tempdir"
 	"gitlab.com/gitlab-org/gitaly/internal/version"
@@ -81,6 +76,14 @@ func main() {
 	flag.Usage = flagUsage
 	flag.Parse()
 
+	// gitaly-wrapper is supposed to set config.EnvUpgradesEnabled in order to enable graceful upgrades
+	_, isWrapped := os.LookupEnv(config.EnvUpgradesEnabled)
+	b, err := newBootstrap(os.Getenv(config.EnvPidFile), isWrapped)
+	if err != nil {
+		log.WithError(err).Fatal("init bootstrap")
+	}
+	defer b.Stop()
+
 	// If invoked with -version
 	if *flagVersion {
 		fmt.Println(version.GetVersionString())
@@ -108,103 +111,30 @@ func main() {
 
 	tempdir.StartCleaning()
 
-	var insecureListeners []net.Listener
-	var secureListeners []net.Listener
-
-	if socketPath := config.Config.SocketPath; socketPath != "" {
-		l, err := createUnixListener(socketPath)
-		if err != nil {
-			log.WithError(err).Fatal("configure unix listener")
-		}
-		log.WithField("address", socketPath).Info("listening on unix socket")
-		insecureListeners = append(insecureListeners, l)
-	}
-
-	if addr := config.Config.ListenAddr; addr != "" {
-		l, err := net.Listen("tcp", addr)
-		if err != nil {
-			log.WithError(err).Fatal("configure tcp listener")
-		}
-
-		log.WithField("address", addr).Info("listening at tcp address")
-		insecureListeners = append(insecureListeners, connectioncounter.New("tcp", l))
+	if err = b.listen(); err != nil {
+		log.WithError(err).Fatal("bootstrap failed")
 	}
 
-	if addr := config.Config.TLSListenAddr; addr != "" {
-		tlsListener, err := net.Listen("tcp", addr)
+	if config.Config.PrometheusListenAddr != "" {
+		l, err := b.prometheusListener()
 		if err != nil {
-			log.WithError(err).Fatal("configure tls listener")
+			log.WithError(err).Fatal("configure prometheus listener")
 		}
 
-		secureListeners = append(secureListeners, connectioncounter.New("tls", tlsListener))
-	}
-
-	if config.Config.PrometheusListenAddr != "" {
-		log.WithField("address", config.Config.PrometheusListenAddr).Info("Starting prometheus listener")
 		promMux := http.NewServeMux()
 		promMux.Handle("/metrics", promhttp.Handler())
 
 		server.AddPprofHandlers(promMux)
 
 		go func() {
-			http.ListenAndServe(config.Config.PrometheusListenAddr, promMux)
+			err = http.Serve(l, promMux)
+			if err != nil {
+				log.WithError(err).Fatal("Unable to serve prometheus")
+			}
 		}()
 	}
 
-	log.WithError(run(insecureListeners, secureListeners)).Fatal("shutting down")
-}
-
-func createUnixListener(socketPath string) (net.Listener, error) {
-	if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
-		return nil, err
-	}
-	l, err := net.Listen("unix", socketPath)
-	return connectioncounter.New("unix", l), err
-}
-
-// Inside here we can use deferred functions. This is needed because
-// log.Fatal bypasses deferred functions.
-func run(insecureListeners, secureListeners []net.Listener) error {
-	signals := []os.Signal{syscall.SIGTERM, syscall.SIGINT}
-	termCh := make(chan os.Signal, len(signals))
-	signal.Notify(termCh, signals...)
-
-	ruby, err := rubyserver.Start()
-	if err != nil {
-		return err
-	}
-	defer ruby.Stop()
-
-	serverErrors := make(chan error, len(insecureListeners)+len(secureListeners))
-	if len(insecureListeners) > 0 {
-		insecureServer := server.NewInsecure(ruby)
-		defer insecureServer.Stop()
-
-		for _, listener := range insecureListeners {
-			// Must pass the listener as a function argument because there is a race
-			// between 'go' and 'for'.
-			go func(l net.Listener) {
-				serverErrors <- insecureServer.Serve(l)
-			}(listener)
-		}
-	}
-
-	if len(secureListeners) > 0 {
-		secureServer := server.NewSecure(ruby)
-		defer secureServer.Stop()
-
-		for _, listener := range secureListeners {
-			go func(l net.Listener) {
-				serverErrors <- secureServer.Serve(l)
-			}(listener)
-		}
-	}
-
-	select {
-	case s := <-termCh:
-		err = fmt.Errorf("received signal %q", s)
-	case err = <-serverErrors:
-	}
+	b.run()
 
-	return err
+	log.Fatal("shutting down")
 }
diff --git a/internal/config/config.go b/internal/config/config.go
index 14bebbf3f..4f625783e 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -7,32 +7,42 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/BurntSushi/toml"
 	"github.com/kelseyhightower/envconfig"
 	log "github.com/sirupsen/logrus"
 )
 
+const (
+	// EnvPidFile is the name of the environment variable containing the pid file path
+	EnvPidFile = "GITALY_PID_FILE"
+	// EnvUpgradesEnabled is an environment variable that when defined gitaly must enable graceful upgrades on SIGHUP
+	EnvUpgradesEnabled = "GITALY_UPGRADES_ENABLED"
+)
+
 var (
 	// Config stores the global configuration
 	Config config
 )
 
 type config struct {
-	SocketPath           string        `toml:"socket_path" split_words:"true"`
-	ListenAddr           string        `toml:"listen_addr" split_words:"true"`
-	TLSListenAddr        string        `toml:"tls_listen_addr" split_words:"true"`
-	PrometheusListenAddr string        `toml:"prometheus_listen_addr" split_words:"true"`
-	BinDir               string        `toml:"bin_dir"`
-	Git                  Git           `toml:"git" envconfig:"git"`
-	Storages             []Storage     `toml:"storage" envconfig:"storage"`
-	Logging              Logging       `toml:"logging" envconfig:"logging"`
-	Prometheus           Prometheus    `toml:"prometheus"`
-	Auth                 Auth          `toml:"auth"`
-	TLS                  TLS           `toml:"tls"`
-	Ruby                 Ruby          `toml:"gitaly-ruby"`
-	GitlabShell          GitlabShell   `toml:"gitlab-shell"`
-	Concurrency          []Concurrency `toml:"concurrency"`
+	SocketPath                 string        `toml:"socket_path" split_words:"true"`
+	ListenAddr                 string        `toml:"listen_addr" split_words:"true"`
+	TLSListenAddr              string        `toml:"tls_listen_addr" split_words:"true"`
+	PrometheusListenAddr       string        `toml:"prometheus_listen_addr" split_words:"true"`
+	BinDir                     string        `toml:"bin_dir"`
+	Git                        Git           `toml:"git" envconfig:"git"`
+	Storages                   []Storage     `toml:"storage" envconfig:"storage"`
+	Logging                    Logging       `toml:"logging" envconfig:"logging"`
+	Prometheus                 Prometheus    `toml:"prometheus"`
+	Auth                       Auth          `toml:"auth"`
+	TLS                        TLS           `toml:"tls"`
+	Ruby                       Ruby          `toml:"gitaly-ruby"`
+	GitlabShell                GitlabShell   `toml:"gitlab-shell"`
+	Concurrency                []Concurrency `toml:"concurrency"`
+	GracefulRestartTimeout     time.Duration
+	GracefulRestartTimeoutToml duration `toml:"graceful_restart_timeout"`
 }
 
 // TLS configuration
@@ -100,6 +110,8 @@ func Load(file io.Reader) error {
 		return fmt.Errorf("envconfig: %v", err)
 	}
 
+	Config.setDefaults()
+
 	return nil
 }
 
@@ -121,6 +133,13 @@ func Validate() error {
 	return nil
 }
 
+func (c *config) setDefaults() {
+	c.GracefulRestartTimeout = c.GracefulRestartTimeoutToml.Duration
+	if c.GracefulRestartTimeout == 0 {
+		c.GracefulRestartTimeout = 1 * time.Minute
+	}
+}
+
 func validateListeners() error {
 	if len(Config.SocketPath) == 0 && len(Config.ListenAddr) == 0 {
 		return fmt.Errorf("invalid listener config: at least one of socket_path and listen_addr must be set")
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 6a8bcb5d0..333abe62d 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -10,6 +10,7 @@ import (
 	"path"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -41,7 +42,10 @@ func TestLoadEmptyConfig(t *testing.T) {
 	err := Load(tmpFile)
 	assert.NoError(t, err)
 
-	assert.Equal(t, config{}, Config)
+	defaultConf := config{}
+	defaultConf.setDefaults()
+
+	assert.Equal(t, defaultConf, Config)
 }
 
 func TestLoadStorage(t *testing.T) {
@@ -53,11 +57,14 @@ path = "/tmp"`)
 	assert.NoError(t, err)
 
 	if assert.Equal(t, 1, len(Config.Storages), "Expected one (1) storage") {
-		assert.Equal(t, config{
+		expectedConf := config{
 			Storages: []Storage{
 				{Name: "default", Path: "/tmp"},
 			},
-		}, Config)
+		}
+		expectedConf.setDefaults()
+
+		assert.Equal(t, expectedConf, Config)
 	}
 }
 
@@ -74,12 +81,15 @@ path="/tmp/repos2"`)
 	assert.NoError(t, err)
 
 	if assert.Equal(t, 2, len(Config.Storages), "Expected one (1) storage") {
-		assert.Equal(t, config{
+		expectedConf := config{
 			Storages: []Storage{
 				{Name: "default", Path: "/tmp/repos1"},
 				{Name: "other", Path: "/tmp/repos2"},
 			},
-		}, Config)
+		}
+		expectedConf.setDefaults()
+
+		assert.Equal(t, expectedConf, Config)
 	}
 }
 
@@ -490,3 +500,31 @@ func TestValidateListeners(t *testing.T) {
 		})
 	}
 }
+
+func TestLoadGracefulRestartTimeout(t *testing.T) {
+	tests := []struct {
+		name     string
+		config   string
+		expected time.Duration
+	}{
+		{
+			name:     "default value",
+			expected: 1 * time.Minute,
+		},
+		{
+			name:     "8m03s",
+			config:   `graceful_restart_timeout = "8m03s"`,
+			expected: 8*time.Minute + 3*time.Second,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			tmpFile := configFileReader(test.config)
+
+			err := Load(tmpFile)
+			assert.NoError(t, err)
+
+			assert.Equal(t, test.expected, Config.GracefulRestartTimeout)
+		})
+	}
+}
diff --git a/vendor/github.com/cloudflare/tableflip/LICENSE b/vendor/github.com/cloudflare/tableflip/LICENSE
new file mode 100644
index 000000000..6eea1cc21
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/LICENSE
@@ -0,0 +1,11 @@
+Copyright (c) 2017-2018, Cloudflare. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/vendor/github.com/cloudflare/tableflip/README.md b/vendor/github.com/cloudflare/tableflip/README.md
new file mode 100644
index 000000000..7b90a1971
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/README.md
@@ -0,0 +1,62 @@
+# Graceful process restarts in Go
+
+[![](https://godoc.org/github.com/cloudflare/tableflip?status.svg)](https://godoc.org/github.com/cloudflare/tableflip)
+
+It is sometimes useful to update the running code and / or configuration of a
+network service, without disrupting existing connections. Usually, this is
+achieved by starting a new process, somehow transferring clients to it and
+then exiting the old process.
+
+There are [many ways to implement graceful upgrades](https://blog.cloudflare.com/graceful-upgrades-in-go/).
+They vary wildly in the trade-offs they make, and how much control they afford the user. This library
+has the following goals:
+
+* No old code keeps running after a successful upgrade
+* The new process has a grace period for performing initialisation
+* Crashing during initialisation is OK
+* Only a single upgrade is ever run in parallel
+
+`tableflip` does not work on Windows.
+
+It's easy to get started:
+
+```Go
+upg, err := tableflip.New(tableflip.Options{})
+if err != nil {
+	panic(err)
+}
+defer upg.Stop()
+
+go func() {
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGHUP)
+	for range sig {
+		err := upg.Upgrade()
+		if err != nil {
+			log.Println("Upgrade failed:", err)
+			continue
+		}
+
+		log.Println("Upgrade succeeded")
+	}
+}()
+
+ln, err := upg.Fds.Listen("tcp", "localhost:8080")
+if err != nil {
+	log.Fatalln("Can't listen:", err)
+}
+
+var server http.Server
+go server.Serve(ln)
+
+if err := upg.Ready(); err != nil {
+	panic(err)
+}
+<-upg.Exit()
+
+time.AfterFunc(30*time.Second, func() {
+	os.Exit(1)
+})
+
+_ = server.Shutdown(context.Background())
+```
diff --git a/vendor/github.com/cloudflare/tableflip/child.go b/vendor/github.com/cloudflare/tableflip/child.go
new file mode 100644
index 000000000..d3aeecfe2
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/child.go
@@ -0,0 +1,113 @@
+package tableflip
+
+import (
+	"encoding/gob"
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+type child struct {
+	*env
+	proc           process
+	readyR, namesW *os.File
+	ready          <-chan *os.File
+	result         <-chan error
+	exited         <-chan struct{}
+}
+
+func startChild(env *env, passedFiles map[fileName]*file) (*child, error) {
+	// These pipes are used for communication between parent and child
+	// readyW is passed to the child, readyR stays with the parent
+	readyR, readyW, err := os.Pipe()
+	if err != nil {
+		return nil, errors.Wrap(err, "pipe failed")
+	}
+
+	namesR, namesW, err := os.Pipe()
+	if err != nil {
+		readyR.Close()
+		readyW.Close()
+		return nil, errors.Wrap(err, "pipe failed")
+	}
+
+	// Copy passed fds and append the notification pipe
+	fds := []*os.File{readyW, namesR}
+	var fdNames [][]string
+	for name, file := range passedFiles {
+		nameSlice := make([]string, len(name))
+		copy(nameSlice, name[:])
+		fdNames = append(fdNames, nameSlice)
+		fds = append(fds, file.File)
+	}
+
+	// Copy environment and append the notification env vars
+	environ := append([]string(nil), env.environ()...)
+	environ = append(environ,
+		fmt.Sprintf("%s=yes", sentinelEnvVar))
+
+	proc, err := env.newProc(os.Args[0], os.Args[1:], fds, environ)
+	if err != nil {
+		readyR.Close()
+		readyW.Close()
+		namesR.Close()
+		namesW.Close()
+		return nil, errors.Wrapf(err, "can't start process %s", os.Args[0])
+	}
+
+	exited := make(chan struct{})
+	result := make(chan error, 1)
+	ready := make(chan *os.File, 1)
+
+	c := &child{
+		env,
+		proc,
+		readyR,
+		namesW,
+		ready,
+		result,
+		exited,
+	}
+	go c.writeNames(fdNames)
+	go c.waitExit(result, exited)
+	go c.waitReady(ready)
+	return c, nil
+}
+
+func (c *child) String() string {
+	return c.proc.String()
+}
+
+func (c *child) Kill() {
+	c.proc.Signal(os.Kill)
+}
+
+func (c *child) waitExit(result chan<- error, exited chan<- struct{}) {
+	result <- c.proc.Wait()
+	close(exited)
+	// Unblock waitReady and writeNames
+	c.readyR.Close()
+	c.namesW.Close()
+}
+
+func (c *child) waitReady(ready chan<- *os.File) {
+	var b [1]byte
+	if n, _ := c.readyR.Read(b[:]); n > 0 && b[0] == notifyReady {
+		// We know that writeNames has exited by this point.
+		// Closing the FD now signals to the child that the parent
+		// has exited.
+		ready <- c.namesW
+	}
+	c.readyR.Close()
+}
+
+func (c *child) writeNames(names [][]string) {
+	enc := gob.NewEncoder(c.namesW)
+	if names == nil {
+		// Gob panics on nil
+		_ = enc.Encode([][]string{})
+		return
+	}
+	_ = enc.Encode(names)
+}
diff --git a/vendor/github.com/cloudflare/tableflip/doc.go b/vendor/github.com/cloudflare/tableflip/doc.go
new file mode 100644
index 000000000..bd493ffbf
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/doc.go
@@ -0,0 +1,41 @@
+// Package tableflip implements zero downtime upgrades.
+//
+// An upgrade spawns a new copy of argv[0] and passes
+// file descriptors of used listening sockets to the new process. The old process exits
+// once the new process signals readiness. Thus new code can use sockets allocated
+// in the old process. This is similar to the approach used by nginx, but
+// as a library.
+//
+// At any point in time there are one or two processes, with at most one of them
+// in non-ready state. A successful upgrade fully replaces all old configuration
+// and code.
+//
+// To use this library with systemd you need to use the PIDFile option in the service
+// file.
+//
+//    [Unit]
+//    Description=Service using tableflip
+//
+//    [Service]
+//    ExecStart=/path/to/binary -some-flag /path/to/pid-file
+//    ExecReload=/bin/kill -HUP $MAINPID
+//    PIDFile=/path/to/pid-file
+//
+// Then pass /path/to/pid-file to New. You can use systemd-run to
+// test your implementation:
+//
+//    systemd-run --user -p PIDFile=/path/to/pid-file /path/to/binary
+//
+// systemd-run will print a unit name, which you can use with systemctl to
+// inspect the service.
+//
+// NOTES:
+//
+// Requires at least Go 1.9, since there is a race condition on the
+// pipes used for communication between parent and child.
+//
+// If you're seeing "can't start process: no such file or directory",
+// you're probably using "go run main.go", for graceful reloads to work,
+// you'll need use "go build main.go".
+//
+package tableflip
diff --git a/vendor/github.com/cloudflare/tableflip/dup_file.go b/vendor/github.com/cloudflare/tableflip/dup_file.go
new file mode 100644
index 000000000..43a52c4f8
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/dup_file.go
@@ -0,0 +1,12 @@
+// +build go1.12
+
+package tableflip
+
+import (
+	"os"
+)
+
+func dupFile(fh *os.File, name fileName) (*file, error) {
+	// os.File implements syscall.Conn from go 1.12
+	return dupConn(fh, name)
+}
diff --git a/vendor/github.com/cloudflare/tableflip/dup_file_legacy.go b/vendor/github.com/cloudflare/tableflip/dup_file_legacy.go
new file mode 100644
index 000000000..b045837ce
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/dup_file_legacy.go
@@ -0,0 +1,11 @@
+// +build !go1.12
+
+package tableflip
+
+import (
+	"os"
+)
+
+func dupFile(fh *os.File, name fileName) (*file, error) {
+	return dupFd(fh.Fd(), name)
+}
diff --git a/vendor/github.com/cloudflare/tableflip/env.go b/vendor/github.com/cloudflare/tableflip/env.go
new file mode 100644
index 000000000..3a8d3e7b7
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/env.go
@@ -0,0 +1,22 @@
+package tableflip
+
+import (
+	"os"
+	"syscall"
+)
+
+var stdEnv = &env{
+	newProc:     newOSProcess,
+	newFile:     os.NewFile,
+	environ:     os.Environ,
+	getenv:      os.Getenv,
+	closeOnExec: syscall.CloseOnExec,
+}
+
+type env struct {
+	newProc     func(string, []string, []*os.File, []string) (process, error)
+	newFile     func(fd uintptr, name string) *os.File
+	environ     func() []string
+	getenv      func(string) string
+	closeOnExec func(fd int)
+}
diff --git a/vendor/github.com/cloudflare/tableflip/fds.go b/vendor/github.com/cloudflare/tableflip/fds.go
new file mode 100644
index 000000000..1fe9ba9f1
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/fds.go
@@ -0,0 +1,325 @@
+package tableflip
+
+import (
+	"net"
+	"os"
+	"strings"
+	"sync"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+// Listener can be shared between processes.
+type Listener interface {
+	net.Listener
+	syscall.Conn
+}
+
+// Conn can be shared between processes.
+type Conn interface {
+	net.Conn
+	syscall.Conn
+}
+
+const (
+	listenKind = "listener"
+	connKind   = "conn"
+	fdKind     = "fd"
+)
+
+type fileName [3]string
+
+func (name fileName) String() string {
+	return strings.Join(name[:], ":")
+}
+
+// file works around the fact that it's not possible
+// to get the fd from an os.File without putting it into
+// blocking mode.
+type file struct {
+	*os.File
+	fd uintptr
+}
+
+func newFile(fd uintptr, name fileName) *file {
+	f := os.NewFile(fd, name.String())
+	if f == nil {
+		return nil
+	}
+
+	return &file{
+		f,
+		fd,
+	}
+}
+
+// Fds holds all file descriptors inherited from the
+// parent process.
+type Fds struct {
+	mu sync.Mutex
+	// NB: Files in these maps may be in blocking mode.
+	inherited map[fileName]*file
+	used      map[fileName]*file
+}
+
+func newFds(inherited map[fileName]*file) *Fds {
+	if inherited == nil {
+		inherited = make(map[fileName]*file)
+	}
+	return &Fds{
+		inherited: inherited,
+		used:      make(map[fileName]*file),
+	}
+}
+
+// Listen returns a listener inherited from the parent process, or creates a new one.
+func (f *Fds) Listen(network, addr string) (net.Listener, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	ln, err := f.listenerLocked(network, addr)
+	if err != nil {
+		return nil, err
+	}
+
+	if ln != nil {
+		return ln, nil
+	}
+
+	ln, err = net.Listen(network, addr)
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create new listener")
+	}
+
+	if _, ok := ln.(Listener); !ok {
+		ln.Close()
+		return nil, errors.Errorf("%T doesn't implement tableflip.Listener", ln)
+	}
+
+	err = f.addListenerLocked(network, addr, ln.(Listener))
+	if err != nil {
+		ln.Close()
+		return nil, err
+	}
+
+	return ln, nil
+}
+
+// Listener returns an inherited listener or nil.
+//
+// It is safe to close the returned listener.
+func (f *Fds) Listener(network, addr string) (net.Listener, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return f.listenerLocked(network, addr)
+}
+
+func (f *Fds) listenerLocked(network, addr string) (net.Listener, error) {
+	key := fileName{listenKind, network, addr}
+	file := f.inherited[key]
+	if file == nil {
+		return nil, nil
+	}
+
+	ln, err := net.FileListener(file.File)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't inherit listener %s %s", network, addr)
+	}
+
+	delete(f.inherited, key)
+	f.used[key] = file
+	return ln, nil
+}
+
+// AddListener adds a listener.
+//
+// It is safe to close ln after calling the method.
+// Any existing listener with the same address is overwitten.
+func (f *Fds) AddListener(network, addr string, ln Listener) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return f.addListenerLocked(network, addr, ln)
+}
+
+type unlinkOnCloser interface {
+	SetUnlinkOnClose(bool)
+}
+
+func (f *Fds) addListenerLocked(network, addr string, ln Listener) error {
+	if ifc, ok := ln.(unlinkOnCloser); ok {
+		ifc.SetUnlinkOnClose(false)
+	}
+
+	return f.addConnLocked(listenKind, network, addr, ln)
+}
+
+// Conn returns an inherited connection or nil.
+//
+// It is safe to close the returned Conn.
+func (f *Fds) Conn(network, addr string) (net.Conn, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	key := fileName{connKind, network, addr}
+	file := f.inherited[key]
+	if file == nil {
+		return nil, nil
+	}
+
+	conn, err := net.FileConn(file.File)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't inherit connection %s %s", network, addr)
+	}
+
+	delete(f.inherited, key)
+	f.used[key] = file
+	return conn, nil
+}
+
+// AddConn adds a connection.
+//
+// It is safe to close conn after calling this method.
+func (f *Fds) AddConn(network, addr string, conn Conn) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return f.addConnLocked(connKind, network, addr, conn)
+}
+
+func (f *Fds) addConnLocked(kind, network, addr string, conn syscall.Conn) error {
+	key := fileName{kind, network, addr}
+	file, err := dupConn(conn, key)
+	if err != nil {
+		return errors.Wrapf(err, "can't dup listener %s %s", network, addr)
+	}
+
+	delete(f.inherited, key)
+	f.used[key] = file
+	return nil
+}
+
+// File returns an inherited file or nil.
+//
+// The descriptor may be in blocking mode.
+func (f *Fds) File(name string) (*os.File, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	key := fileName{fdKind, name}
+	file := f.inherited[key]
+	if file == nil {
+		return nil, nil
+	}
+
+	// Make a copy of the file, since we don't want to
+	// allow the caller to invalidate fds in f.inherited.
+	dup, err := dupFd(file.fd, key)
+	if err != nil {
+		return nil, err
+	}
+
+	delete(f.inherited, key)
+	f.used[key] = file
+	return dup.File, nil
+}
+
+// AddFile adds a file.
+//
+// Until Go 1.12, file will be in blocking mode
+// after this call.
+func (f *Fds) AddFile(name string, file *os.File) error {
+	key := fileName{fdKind, name}
+
+	dup, err := dupFile(file, key)
+	if err != nil {
+		return err
+	}
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	delete(f.inherited, key)
+	f.used[key] = dup
+	return nil
+}
+
+func (f *Fds) copy() map[fileName]*file {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	files := make(map[fileName]*file, len(f.used))
+	for key, file := range f.used {
+		files[key] = file
+	}
+
+	return files
+}
+
+func (f *Fds) closeInherited() {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	for key, file := range f.inherited {
+		if key[0] == listenKind && (key[1] == "unix" || key[1] == "unixpacket") {
+			// Remove inherited but unused Unix sockets from the file system.
+			// This undoes the effect of SetUnlinkOnClose(false).
+			_ = unlinkUnixSocket(key[2])
+		}
+		_ = file.Close()
+	}
+	f.inherited = make(map[fileName]*file)
+}
+
+func unlinkUnixSocket(path string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+
+	if info.Mode()&os.ModeSocket == 0 {
+		return nil
+	}
+
+	return os.Remove(path)
+}
+
+func (f *Fds) closeUsed() {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	for _, file := range f.used {
+		_ = file.Close()
+	}
+	f.used = make(map[fileName]*file)
+}
+
+func dupConn(conn syscall.Conn, name fileName) (*file, error) {
+	// Use SyscallConn instead of File to avoid making the original
+	// fd non-blocking.
+	raw, err := conn.SyscallConn()
+	if err != nil {
+		return nil, err
+	}
+
+	var dup *file
+	var duperr error
+	err = raw.Control(func(fd uintptr) {
+		dup, duperr = dupFd(fd, name)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "can't access fd")
+	}
+	return dup, duperr
+}
+
+func dupFd(fd uintptr, name fileName) (*file, error) {
+	dupfd, _, errno := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_DUPFD_CLOEXEC, 0)
+	if errno != 0 {
+		return nil, errors.Wrap(errno, "can't dup fd using fcntl")
+	}
+
+	return newFile(dupfd, name), nil
+}
diff --git a/vendor/github.com/cloudflare/tableflip/parent.go b/vendor/github.com/cloudflare/tableflip/parent.go
new file mode 100644
index 000000000..6ca085de0
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/parent.go
@@ -0,0 +1,80 @@
+package tableflip
+
+import (
+	"encoding/gob"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	sentinelEnvVar = "TABLEFLIP_HAS_PARENT_7DIU3"
+	notifyReady    = 42
+)
+
+type parent struct {
+	wr     *os.File
+	result <-chan error
+	exited <-chan struct{}
+}
+
+func newParent(env *env) (*parent, map[fileName]*file, error) {
+	if env.getenv(sentinelEnvVar) == "" {
+		return nil, make(map[fileName]*file), nil
+	}
+
+	wr := env.newFile(3, "write")
+	rd := env.newFile(4, "read")
+
+	var names [][]string
+	dec := gob.NewDecoder(rd)
+	if err := dec.Decode(&names); err != nil {
+		return nil, nil, errors.Wrap(err, "can't decode names from parent process")
+	}
+
+	files := make(map[fileName]*file)
+	for i, parts := range names {
+		var key fileName
+		copy(key[:], parts)
+
+		// Start at 5 to account for stdin, etc. and write
+		// and read pipes.
+		fd := 5 + i
+		env.closeOnExec(fd)
+		files[key] = &file{
+			env.newFile(uintptr(fd), key.String()),
+			uintptr(fd),
+		}
+	}
+
+	result := make(chan error, 1)
+	exited := make(chan struct{})
+	go func() {
+		defer rd.Close()
+
+		n, err := io.Copy(ioutil.Discard, rd)
+		if n != 0 {
+			err = errors.New("unexpected data from parent process")
+		} else if err != nil {
+			err = errors.Wrap(err, "unexpected error while waiting for parent to exit")
+		}
+		result <- err
+		close(exited)
+	}()
+
+	return &parent{
+		wr:     wr,
+		result: result,
+		exited: exited,
+	}, files, nil
+}
+
+func (ps *parent) sendReady() error {
+	defer ps.wr.Close()
+	if _, err := ps.wr.Write([]byte{notifyReady}); err != nil {
+		return errors.Wrap(err, "can't notify parent process")
+	}
+	return nil
+}
diff --git a/vendor/github.com/cloudflare/tableflip/process.go b/vendor/github.com/cloudflare/tableflip/process.go
new file mode 100644
index 000000000..c918a88c8
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/process.go
@@ -0,0 +1,47 @@
+package tableflip
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+var initialWD, _ = os.Getwd()
+
+type process interface {
+	fmt.Stringer
+	Signal(sig os.Signal) error
+	Wait() error
+}
+
+type osProcess struct {
+	cmd *exec.Cmd
+}
+
+func newOSProcess(executable string, args []string, files []*os.File, env []string) (process, error) {
+	cmd := exec.Command(executable, args...)
+	cmd.Dir = initialWD
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.ExtraFiles = files
+	cmd.Env = env
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	return &osProcess{cmd}, nil
+}
+
+func (osp *osProcess) Signal(sig os.Signal) error {
+	return osp.cmd.Process.Signal(sig)
+}
+
+func (osp *osProcess) Wait() error {
+	return osp.cmd.Wait()
+}
+
+func (osp *osProcess) String() string {
+	return fmt.Sprintf("pid=%d", osp.cmd.Process.Pid)
+}
diff --git a/vendor/github.com/cloudflare/tableflip/upgrader.go b/vendor/github.com/cloudflare/tableflip/upgrader.go
new file mode 100644
index 000000000..4ab81a79b
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/upgrader.go
@@ -0,0 +1,284 @@
+package tableflip
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// DefaultUpgradeTimeout is the duration before the Upgrader kills the new process if no
+// readiness notification was received.
+const DefaultUpgradeTimeout time.Duration = time.Minute
+
+// Options control the behaviour of the Upgrader.
+type Options struct {
+	// Time after which an upgrade is considered failed. Defaults to
+	// DefaultUpgradeTimeout.
+	UpgradeTimeout time.Duration
+	// The PID of a ready process is written to this file.
+	PIDFile string
+}
+
+// Upgrader handles zero downtime upgrades and passing files between processes.
+type Upgrader struct {
+	*env
+	opts      Options
+	parent    *parent
+	parentErr chan error
+	readyOnce sync.Once
+	readyC    chan struct{}
+	stopOnce  sync.Once
+	stopC     chan struct{}
+	upgradeC  chan chan<- error
+	exitC     chan struct{}
+	exitFd    chan neverCloseThisFile
+
+	Fds *Fds
+}
+
+var (
+	stdEnvMu       sync.Mutex
+	stdEnvUpgrader *Upgrader
+)
+
+// New creates a new Upgrader. Files are passed from the parent and may be empty.
+//
+// Only the first call to this function will succeed.
+func New(opts Options) (upg *Upgrader, err error) {
+	stdEnvMu.Lock()
+	defer stdEnvMu.Unlock()
+
+	if stdEnvUpgrader != nil {
+		return nil, errors.New("tableflip: only a single Upgrader allowed")
+	}
+
+	upg, err = newUpgrader(stdEnv, opts)
+	// Store a reference to upg in a private global variable, to prevent
+	// it from being GC'ed and exitFd being closed prematurely.
+	stdEnvUpgrader = upg
+	return
+}
+
+func newUpgrader(env *env, opts Options) (*Upgrader, error) {
+	parent, files, err := newParent(env)
+	if err != nil {
+		return nil, err
+	}
+
+	if opts.UpgradeTimeout <= 0 {
+		opts.UpgradeTimeout = DefaultUpgradeTimeout
+	}
+
+	u := &Upgrader{
+		env:       env,
+		opts:      opts,
+		parent:    parent,
+		parentErr: make(chan error, 1),
+		readyC:    make(chan struct{}),
+		stopC:     make(chan struct{}),
+		upgradeC:  make(chan chan<- error),
+		exitC:     make(chan struct{}),
+		exitFd:    make(chan neverCloseThisFile, 1),
+		Fds:       newFds(files),
+	}
+
+	go u.run()
+
+	return u, nil
+}
+
+// Ready signals that the current process is ready to accept connections.
+// It must be called to finish the upgrade.
+//
+// All fds which were inherited but not used are closed after the call to Ready.
+func (u *Upgrader) Ready() error {
+	u.readyOnce.Do(func() {
+		u.Fds.closeInherited()
+		close(u.readyC)
+	})
+
+	if u.opts.PIDFile != "" {
+		if err := writePIDFile(u.opts.PIDFile); err != nil {
+			return errors.Wrap(err, "tableflip: can't write PID file")
+		}
+	}
+
+	if u.parent == nil {
+		return nil
+	}
+	return u.parent.sendReady()
+}
+
+// Exit returns a channel which is closed when the process should
+// exit.
+func (u *Upgrader) Exit() <-chan struct{} {
+	return u.exitC
+}
+
+// Stop prevents any more upgrades from happening, and closes
+// the exit channel.
+func (u *Upgrader) Stop() {
+	u.stopOnce.Do(func() {
+		// Interrupt any running Upgrade(), and
+		// prevent new upgrade from happening.
+		close(u.stopC)
+	})
+}
+
+// WaitForParent blocks until the parent has exited.
+//
+// Returns an error if the parent misbehaved during shutdown.
+func (u *Upgrader) WaitForParent(ctx context.Context) error {
+	if u.parent == nil {
+		return nil
+	}
+
+	var err error
+	select {
+	case err = <-u.parent.result:
+	case err = <-u.parentErr:
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	// This is a bit cheeky, since it means that multiple
+	// calls to WaitForParent resolve in sequence, but that
+	// probably doesn't matter.
+	u.parentErr <- err
+	return err
+}
+
+// HasParent checks if the current process is an upgrade or the first invocation.
+func (u *Upgrader) HasParent() bool {
+	return u.parent != nil
+}
+
+// Upgrade triggers an upgrade.
+func (u *Upgrader) Upgrade() error {
+	response := make(chan error, 1)
+	select {
+	case <-u.stopC:
+		return errors.New("terminating")
+	case <-u.exitC:
+		return errors.New("already upgraded")
+	case u.upgradeC <- response:
+	}
+
+	return <-response
+}
+
+var errNotReady = errors.New("process is not ready yet")
+
+func (u *Upgrader) run() {
+	defer close(u.exitC)
+	defer u.Fds.closeUsed()
+
+	var (
+		parentExited <-chan struct{}
+		processReady = u.readyC
+	)
+
+	if u.parent != nil {
+		parentExited = u.parent.exited
+	}
+
+	for {
+		select {
+		case <-parentExited:
+			parentExited = nil
+
+		case <-processReady:
+			processReady = nil
+
+		case <-u.stopC:
+			return
+
+		case request := <-u.upgradeC:
+			if processReady != nil {
+				request <- errNotReady
+				continue
+			}
+
+			if parentExited != nil {
+				request <- errors.New("parent hasn't exited")
+				continue
+			}
+
+			file, err := u.doUpgrade()
+			request <- err
+
+			if err == nil {
+				// Save file in exitFd, so that it's only closed when the process
+				// exits. This signals to the new process that the old process
+				// has exited.
+				u.exitFd <- neverCloseThisFile{file}
+				return
+			}
+		}
+	}
+}
+
+func (u *Upgrader) doUpgrade() (*os.File, error) {
+	child, err := startChild(u.env, u.Fds.copy())
+	if err != nil {
+		return nil, errors.Wrap(err, "can't start child")
+	}
+
+	readyTimeout := time.After(u.opts.UpgradeTimeout)
+	for {
+		select {
+		case request := <-u.upgradeC:
+			request <- errors.New("upgrade in progress")
+
+		case err := <-child.result:
+			if err == nil {
+				return nil, errors.Errorf("child %s exited", child)
+			}
+			return nil, errors.Wrapf(err, "child %s exited", child)
+
+		case <-u.stopC:
+			child.Kill()
+			return nil, errors.New("terminating")
+
+		case <-readyTimeout:
+			child.Kill()
+			return nil, errors.Errorf("new child %s timed out", child)
+
+		case file := <-child.ready:
+			return file, nil
+		}
+	}
+}
+
+// This file must never be closed by the Go runtime, since its used by the
+// child to determine when the parent has died. It must only be closed
+// by the OS.
+// Hence we make sure that this file can't be garbage collected by referencing
+// it from an Upgrader.
+type neverCloseThisFile struct {
+	file *os.File
+}
+
+func writePIDFile(path string) error {
+	dir, file := filepath.Split(path)
+	fh, err := ioutil.TempFile(dir, file)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+	// Remove temporary PID file if something fails
+	defer os.Remove(fh.Name())
+
+	_, err = fh.WriteString(strconv.Itoa(os.Getpid()))
+	if err != nil {
+		return err
+	}
+
+	return os.Rename(fh.Name(), path)
+}
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 6a063e103..ffcdd9aae 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -27,6 +27,12 @@
 			"versionExact": "2018.01.18"
 		},
 		{
+			"checksumSHA1": "Hs6U2VxuKuLu/xUPktH4xJtiCzk=",
+			"path": "github.com/cloudflare/tableflip",
+			"revision": "8392f1641731fb26d41d5796b0fc3e3395e49ab8",
+			"revisionTime": "2019-03-28T08:49:24Z"
+		},
+		{
 			"checksumSHA1": "CSPbwbyzqA6sfORicn4HFtIhF/c=",
 			"path": "github.com/davecgh/go-spew/spew",
 			"revision": "8991bc29aa16c548c550c7ff78260e27b9ab7c73",