Graceful restart PROOF-of-CONCEPTgraceful-restart

11 files changed, 1006 insertions, 13 deletions

diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go
index 9bacea720..a4e294273 100644
--- a/cmd/gitaly/main.go
+++ b/cmd/gitaly/main.go
@@ -8,7 +8,9 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
+	"github.com/cloudflare/tableflip"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
@@ -20,6 +22,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/internal/server"
 	"gitlab.com/gitlab-org/gitaly/internal/tempdir"
 	"gitlab.com/gitlab-org/gitaly/internal/version"
+	"google.golang.org/grpc"
 )
 
 var (
@@ -107,9 +110,14 @@ func main() {
 	tempdir.StartCleaning()
 
 	var listeners []net.Listener
+	upg, err := tableflip.New(tableflip.Options{})
+	if err != nil {
+		log.WithError(err).Fatal("Unable to tableflip")
+	}
+	defer upg.Stop()
 
 	if socketPath := config.Config.SocketPath; socketPath != "" {
-		l, err := createUnixListener(socketPath)
+		l, err := createUnixListener(upg, socketPath)
 		if err != nil {
 			log.WithError(err).Fatal("configure unix listener")
 		}
@@ -118,7 +126,7 @@ func main() {
 	}
 
 	if addr := config.Config.ListenAddr; addr != "" {
-		l, err := net.Listen("tcp", addr)
+		l, err := upg.Fds.Listen("tcp", addr)
 		if err != nil {
 			log.WithError(err).Fatal("configure tcp listener")
 		}
@@ -135,35 +143,66 @@ func main() {
 		server.AddPprofHandlers(promMux)
 
 		go func() {
-			http.ListenAndServe(config.Config.PrometheusListenAddr, promMux)
+			l, err := upg.Fds.Listen("tcp", config.Config.PrometheusListenAddr)
+			if err != nil {
+				log.WithError(err).Fatal("No tableflip")
+			}
+
+			err = http.Serve(l, promMux)
+			if err != nil {
+				log.WithError(err).Fatal("Unable to serve")
+			}
 		}()
 	}
 
-	log.WithError(run(listeners)).Fatal("shutting down")
+	run(upg, listeners)
 }
 
-func createUnixListener(socketPath string) (net.Listener, error) {
+func createUnixListener(upg *tableflip.Upgrader, socketPath string) (net.Listener, error) {
 	if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
 		return nil, err
 	}
-	l, err := net.Listen("unix", socketPath)
+	l, err := upg.Fds.Listen("unix", socketPath)
 	return connectioncounter.New("unix", l), err
 }
 
-// Inside here we can use deferred functions. This is needed because
-// log.Fatal bypasses deferred functions.
-func run(listeners []net.Listener) error {
+// This function will never return
+func run(upg *tableflip.Upgrader, listeners []net.Listener) {
+	go func() {
+		sig := make(chan os.Signal, 1)
+		signal.Notify(sig, syscall.SIGHUP)
+		for range sig {
+			log.Println("SIGHUPSIGHUPSIGHUPSIGHUP")
+
+			err := upg.Upgrade()
+			if err != nil {
+				log.Println("Upgrade failed:", err)
+				continue
+			}
+
+			log.Println("Upgrade succeeded")
+		}
+	}()
+
 	signals := []os.Signal{syscall.SIGTERM, syscall.SIGINT}
 	termCh := make(chan os.Signal, len(signals))
 	signal.Notify(termCh, signals...)
 
 	ruby, err := rubyserver.Start()
 	if err != nil {
-		return err
+		log.WithError(err).Fatal("unable to start ruby server")
 	}
 	defer ruby.Stop()
 
 	server := server.New(ruby)
+
+	err = gracefulListen(server, upg, listeners)
+	if err != nil {
+		log.WithError(err).Fatal("unable to start ruby server")
+	}
+}
+
+func gracefulListen(server *grpc.Server, upg *tableflip.Upgrader, listeners []net.Listener) error {
 	defer server.Stop()
 
 	serverErrors := make(chan error, len(listeners))
@@ -175,11 +214,24 @@ func run(listeners []net.Listener) error {
 		}(listener)
 	}
 
+	if err := upg.Ready(); err != nil {
+		panic(err)
+	}
+
+	var err error
 	select {
-	case s := <-termCh:
-		err = fmt.Errorf("received signal %q", s)
+	case <-upg.Exit():
 	case err = <-serverErrors:
 	}
 
-	return err
+	if err != nil {
+		log.WithError(err).Fatal("unable to listen")
+	}
+
+	time.AfterFunc(30*time.Second, func() {
+		os.Exit(1)
+	})
+
+	server.GracefulStop()
+	return nil
 }
diff --git a/vendor/github.com/cloudflare/tableflip/LICENSE b/vendor/github.com/cloudflare/tableflip/LICENSE
new file mode 100644
index 000000000..6eea1cc21
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/LICENSE
@@ -0,0 +1,11 @@
+Copyright (c) 2017-2018, Cloudflare. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/vendor/github.com/cloudflare/tableflip/README.md b/vendor/github.com/cloudflare/tableflip/README.md
new file mode 100644
index 000000000..025c89d75
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/README.md
@@ -0,0 +1,60 @@
+# Graceful process restarts in Go
+
+[![](https://godoc.org/github.com/cloudflare/tableflip?status.svg)](https://godoc.org/github.com/cloudflare/tableflip)
+
+It is sometimes useful to update the running code and / or configuration of a
+network service, without disrupting existing connections. Usually, this is
+achieved by starting a new process, somehow transferring clients to it and
+then exiting the old process.
+
+There are [many ways to implement graceful upgrades](https://blog.cloudflare.com/graceful-upgrades-in-go/).
+They vary wildly in the trade-offs they make, and how much control they afford the user. This library
+has the following goals:
+
+* No old code keeps running after a successful upgrade
+* The new process has a grace period for performing initialisation
+* Crashing during initialisation is OK
+* Only a single upgrade is ever run in parallel
+
+It's easy to get started:
+
+```Go
+upg, err := tableflip.New(tableflip.Options{})
+if err != nil {
+	panic(err)
+}
+defer upg.Stop()
+
+go func() {
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGHUP)
+	for range sig {
+		err := upg.Upgrade()
+		if err != nil {
+			log.Println("Upgrade failed:", err)
+			continue
+		}
+
+		log.Println("Upgrade succeeded")
+	}
+}()
+
+ln, err := upg.Fds.Listen("tcp", "localhost:8080")
+if err != nil {
+	log.Fatalln("Can't listen:", err)
+}
+
+var server http.Server
+go server.Serve(ln)
+
+if err := upg.Ready(); err != nil {
+	panic(err)
+}
+<-upg.Exit()
+
+time.AfterFunc(30*time.Second, func() {
+	os.Exit(1)
+})
+
+_ = server.Shutdown(context.Background())
+```
diff --git a/vendor/github.com/cloudflare/tableflip/child.go b/vendor/github.com/cloudflare/tableflip/child.go
new file mode 100644
index 000000000..a4d8a9638
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/child.go
@@ -0,0 +1,113 @@
+package tableflip
+
+import (
+	"encoding/gob"
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+type child struct {
+	*env
+	proc           process
+	readyR, namesW *os.File
+	readyC         <-chan *os.File
+	exitedC        <-chan error
+	doneC          <-chan struct{}
+}
+
+func startChild(env *env, passedFiles map[fileName]*file) (*child, error) {
+	// These pipes are used for communication between parent and child
+	// readyW is passed to the child, readyR stays with the parent
+	readyR, readyW, err := os.Pipe()
+	if err != nil {
+		return nil, errors.Wrap(err, "pipe failed")
+	}
+
+	namesR, namesW, err := os.Pipe()
+	if err != nil {
+		readyR.Close()
+		readyW.Close()
+		return nil, errors.Wrap(err, "pipe failed")
+	}
+
+	// Copy passed fds and append the notification pipe
+	fds := []*os.File{readyW, namesR}
+	var fdNames [][]string
+	for name, file := range passedFiles {
+		nameSlice := make([]string, len(name))
+		copy(nameSlice, name[:])
+		fdNames = append(fdNames, nameSlice)
+		fds = append(fds, file.File)
+	}
+
+	// Copy environment and append the notification env vars
+	environ := append([]string(nil), env.environ()...)
+	environ = append(environ,
+		fmt.Sprintf("%s=yes", sentinelEnvVar))
+
+	proc, err := env.newProc(os.Args[0], os.Args[1:], fds, environ)
+	if err != nil {
+		readyR.Close()
+		readyW.Close()
+		namesR.Close()
+		namesW.Close()
+		return nil, errors.Wrapf(err, "can't start process %s", os.Args[0])
+	}
+
+	doneC := make(chan struct{})
+	exitedC := make(chan error, 1)
+	readyC := make(chan *os.File, 1)
+
+	c := &child{
+		env,
+		proc,
+		readyR,
+		namesW,
+		readyC,
+		exitedC,
+		doneC,
+	}
+	go c.writeNames(fdNames)
+	go c.waitExit(exitedC, doneC)
+	go c.waitReady(readyC)
+	return c, nil
+}
+
+func (c *child) String() string {
+	return c.proc.String()
+}
+
+func (c *child) Kill() {
+	c.proc.Signal(os.Kill)
+}
+
+func (c *child) waitExit(exitedC chan<- error, doneC chan<- struct{}) {
+	exitedC <- c.proc.Wait()
+	close(doneC)
+	// Unblock waitReady and writeNames
+	c.readyR.Close()
+	c.namesW.Close()
+}
+
+func (c *child) waitReady(readyC chan<- *os.File) {
+	var b [1]byte
+	if n, _ := c.readyR.Read(b[:]); n > 0 && b[0] == notifyReady {
+		// We know that writeNames has exited by this point.
+		// Closing the FD now signals to the child that the parent
+		// has exited.
+		readyC <- c.namesW
+	}
+	c.readyR.Close()
+}
+
+func (c *child) writeNames(names [][]string) {
+	enc := gob.NewEncoder(c.namesW)
+	if names == nil {
+		// Gob panics on nil
+		_ = enc.Encode([][]string{})
+		return
+	}
+	_ = enc.Encode(names)
+}
diff --git a/vendor/github.com/cloudflare/tableflip/doc.go b/vendor/github.com/cloudflare/tableflip/doc.go
new file mode 100644
index 000000000..bd493ffbf
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/doc.go
@@ -0,0 +1,41 @@
+// Package tableflip implements zero downtime upgrades.
+//
+// An upgrade spawns a new copy of argv[0] and passes
+// file descriptors of used listening sockets to the new process. The old process exits
+// once the new process signals readiness. Thus new code can use sockets allocated
+// in the old process. This is similar to the approach used by nginx, but
+// as a library.
+//
+// At any point in time there are one or two processes, with at most one of them
+// in non-ready state. A successful upgrade fully replaces all old configuration
+// and code.
+//
+// To use this library with systemd you need to use the PIDFile option in the service
+// file.
+//
+//    [Unit]
+//    Description=Service using tableflip
+//
+//    [Service]
+//    ExecStart=/path/to/binary -some-flag /path/to/pid-file
+//    ExecReload=/bin/kill -HUP $MAINPID
+//    PIDFile=/path/to/pid-file
+//
+// Then pass /path/to/pid-file to New. You can use systemd-run to
+// test your implementation:
+//
+//    systemd-run --user -p PIDFile=/path/to/pid-file /path/to/binary
+//
+// systemd-run will print a unit name, which you can use with systemctl to
+// inspect the service.
+//
+// NOTES:
+//
+// Requires at least Go 1.9, since there is a race condition on the
+// pipes used for communication between parent and child.
+//
+// If you're seeing "can't start process: no such file or directory",
+// you're probably using "go run main.go", for graceful reloads to work,
+// you'll need use "go build main.go".
+//
+package tableflip
diff --git a/vendor/github.com/cloudflare/tableflip/env.go b/vendor/github.com/cloudflare/tableflip/env.go
new file mode 100644
index 000000000..3a8d3e7b7
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/env.go
@@ -0,0 +1,22 @@
+package tableflip
+
+import (
+	"os"
+	"syscall"
+)
+
+var stdEnv = &env{
+	newProc:     newOSProcess,
+	newFile:     os.NewFile,
+	environ:     os.Environ,
+	getenv:      os.Getenv,
+	closeOnExec: syscall.CloseOnExec,
+}
+
+type env struct {
+	newProc     func(string, []string, []*os.File, []string) (process, error)
+	newFile     func(fd uintptr, name string) *os.File
+	environ     func() []string
+	getenv      func(string) string
+	closeOnExec func(fd int)
+}
diff --git a/vendor/github.com/cloudflare/tableflip/fds.go b/vendor/github.com/cloudflare/tableflip/fds.go
new file mode 100644
index 000000000..f7e5361c4
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/fds.go
@@ -0,0 +1,324 @@
+package tableflip
+
+import (
+	"net"
+	"os"
+	"strings"
+	"sync"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+// Listener can be shared between processes.
+type Listener interface {
+	net.Listener
+	syscall.Conn
+}
+
+// Conn can be shared between processes.
+type Conn interface {
+	net.Conn
+	syscall.Conn
+}
+
+const (
+	listenKind = "listener"
+	connKind   = "conn"
+	fdKind     = "fd"
+)
+
+type fileName [3]string
+
+func (name fileName) String() string {
+	return strings.Join(name[:], ":")
+}
+
+// file works around the fact that it's not possible
+// to get the fd from an os.File without putting it into
+// blocking mode.
+type file struct {
+	*os.File
+	fd uintptr
+}
+
+func newFile(fd uintptr, name fileName) *file {
+	f := os.NewFile(fd, name.String())
+	if f == nil {
+		return nil
+	}
+
+	return &file{
+		f,
+		fd,
+	}
+}
+
+// Fds holds all file descriptors inherited from the
+// parent process.
+type Fds struct {
+	mu sync.Mutex
+	// NB: Files in these maps may be in blocking mode.
+	inherited map[fileName]*file
+	used      map[fileName]*file
+}
+
+func newFds(inherited map[fileName]*file) *Fds {
+	if inherited == nil {
+		inherited = make(map[fileName]*file)
+	}
+	return &Fds{
+		inherited: inherited,
+		used:      make(map[fileName]*file),
+	}
+}
+
+// Listen returns a listener inherited from the parent process, or creates a new one.
+func (f *Fds) Listen(network, addr string) (net.Listener, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	ln, err := f.listenerLocked(network, addr)
+	if err != nil {
+		return nil, err
+	}
+
+	if ln != nil {
+		return ln, nil
+	}
+
+	ln, err = net.Listen(network, addr)
+	if err != nil {
+		return nil, errors.Wrap(err, "can't create new listener")
+	}
+
+	if _, ok := ln.(Listener); !ok {
+		ln.Close()
+		return nil, errors.Errorf("%T doesn't implement tableflip.Listener", ln)
+	}
+
+	err = f.addListenerLocked(network, addr, ln.(Listener))
+	if err != nil {
+		ln.Close()
+		return nil, err
+	}
+
+	return ln, nil
+}
+
+// Listener returns an inherited listener or nil.
+//
+// It is safe to close the returned listener.
+func (f *Fds) Listener(network, addr string) (net.Listener, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return f.listenerLocked(network, addr)
+}
+
+func (f *Fds) listenerLocked(network, addr string) (net.Listener, error) {
+	key := fileName{listenKind, network, addr}
+	file := f.inherited[key]
+	if file == nil {
+		return nil, nil
+	}
+
+	ln, err := net.FileListener(file.File)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't inherit listener %s %s", network, addr)
+	}
+
+	delete(f.inherited, key)
+	f.used[key] = file
+	return ln, nil
+}
+
+// AddListener adds a listener.
+//
+// It is safe to close ln after calling the method.
+// Any existing listener with the same address is overwitten.
+func (f *Fds) AddListener(network, addr string, ln Listener) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return f.addListenerLocked(network, addr, ln)
+}
+
+type unlinkOnCloser interface {
+	SetUnlinkOnClose(bool)
+}
+
+func (f *Fds) addListenerLocked(network, addr string, ln Listener) error {
+	if ifc, ok := ln.(unlinkOnCloser); ok {
+		ifc.SetUnlinkOnClose(false)
+	}
+
+	return f.addConnLocked(listenKind, network, addr, ln)
+}
+
+// Conn returns an inherited connection or nil.
+//
+// It is safe to close the returned Conn.
+func (f *Fds) Conn(network, addr string) (net.Conn, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	key := fileName{connKind, network, addr}
+	file := f.inherited[key]
+	if file == nil {
+		return nil, nil
+	}
+
+	conn, err := net.FileConn(file.File)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't inherit connection %s %s", network, addr)
+	}
+
+	delete(f.inherited, key)
+	f.used[key] = file
+	return conn, nil
+}
+
+// AddConn adds a connection.
+//
+// It is safe to close conn after calling this method.
+func (f *Fds) AddConn(network, addr string, conn Conn) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	return f.addConnLocked(connKind, network, addr, conn)
+}
+
+func (f *Fds) addConnLocked(kind, network, addr string, conn syscall.Conn) error {
+	key := fileName{kind, network, addr}
+	file, err := dupConn(conn, key)
+	if err != nil {
+		return errors.Wrapf(err, "can't dup listener %s %s", network, addr)
+	}
+
+	delete(f.inherited, key)
+	f.used[key] = file
+	return nil
+}
+
+// File returns an inherited file or nil.
+//
+// The descriptor may be in blocking mode.
+func (f *Fds) File(name string) (*os.File, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	key := fileName{fdKind, name}
+	file := f.inherited[key]
+	if file == nil {
+		return nil, nil
+	}
+
+	// Make a copy of the file, since we don't want to
+	// allow the caller to invalidate fds in f.inherited.
+	dup, err := dupFd(file.fd, key)
+	if err != nil {
+		return nil, err
+	}
+
+	delete(f.inherited, key)
+	f.used[key] = file
+	return dup.File, nil
+}
+
+// AddFile adds a file.
+//
+// As of Go 1.11, file will be in blocking mode
+// after this call.
+func (f *Fds) AddFile(name string, file *os.File) error {
+	key := fileName{fdKind, name}
+	dup, err := dupFd(file.Fd(), key)
+	if err != nil {
+		return err
+	}
+
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	delete(f.inherited, key)
+	f.used[key] = dup
+	return nil
+}
+
+func (f *Fds) copy() map[fileName]*file {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	files := make(map[fileName]*file, len(f.used))
+	for key, file := range f.used {
+		files[key] = file
+	}
+
+	return files
+}
+
+func (f *Fds) closeInherited() {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	for key, file := range f.inherited {
+		if key[0] == listenKind && (key[1] == "unix" || key[1] == "unixpacket") {
+			// Remove inherited but unused Unix sockets from the file system.
+			// This undoes the effect of SetUnlinkOnClose(false).
+			_ = unlinkUnixSocket(key[2])
+		}
+		_ = file.Close()
+	}
+	f.inherited = make(map[fileName]*file)
+}
+
+func unlinkUnixSocket(path string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+
+	if info.Mode()&os.ModeSocket == 0 {
+		return nil
+	}
+
+	return os.Remove(path)
+}
+
+func (f *Fds) closeUsed() {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	for _, file := range f.used {
+		_ = file.Close()
+	}
+	f.used = make(map[fileName]*file)
+}
+
+func dupConn(conn syscall.Conn, name fileName) (*file, error) {
+	// Use SyscallConn instead of File to avoid making the original
+	// fd non-blocking.
+	raw, err := conn.SyscallConn()
+	if err != nil {
+		return nil, err
+	}
+
+	var dup *file
+	var duperr error
+	err = raw.Control(func(fd uintptr) {
+		dup, duperr = dupFd(fd, name)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "can't access fd")
+	}
+	return dup, duperr
+}
+
+func dupFd(fd uintptr, name fileName) (*file, error) {
+	dupfd, _, errno := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_DUPFD_CLOEXEC, 0)
+	if errno != 0 {
+		return nil, errors.Wrap(errno, "can't dup fd using fcntl")
+	}
+
+	return newFile(dupfd, name), nil
+}
diff --git a/vendor/github.com/cloudflare/tableflip/parent.go b/vendor/github.com/cloudflare/tableflip/parent.go
new file mode 100644
index 000000000..8b974d27c
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/parent.go
@@ -0,0 +1,76 @@
+package tableflip
+
+import (
+	"encoding/gob"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	sentinelEnvVar = "TABLEFLIP_HAS_PARENT_7DIU3"
+	notifyReady    = 42
+)
+
+type parent struct {
+	wr     *os.File
+	exited <-chan error
+}
+
+func newParent(env *env) (*parent, map[fileName]*file, error) {
+	if env.getenv(sentinelEnvVar) == "" {
+		return nil, make(map[fileName]*file), nil
+	}
+
+	wr := env.newFile(3, "write")
+	rd := env.newFile(4, "read")
+
+	var names [][]string
+	dec := gob.NewDecoder(rd)
+	if err := dec.Decode(&names); err != nil {
+		return nil, nil, errors.Wrap(err, "can't decode names from parent process")
+	}
+
+	files := make(map[fileName]*file)
+	for i, parts := range names {
+		var key fileName
+		copy(key[:], parts)
+
+		// Start at 5 to account for stdin, etc. and write
+		// and read pipes.
+		fd := 5 + i
+		env.closeOnExec(fd)
+		files[key] = &file{
+			env.newFile(uintptr(fd), key.String()),
+			uintptr(fd),
+		}
+	}
+
+	exited := make(chan error, 1)
+	go func() {
+		defer rd.Close()
+
+		n, err := io.Copy(ioutil.Discard, rd)
+		if n != 0 {
+			exited <- errors.New("unexpected data from parent process")
+		} else if err != nil {
+			exited <- errors.Wrap(err, "unexpected error while waiting for parent to exit")
+		}
+		close(exited)
+	}()
+
+	return &parent{
+		wr:     wr,
+		exited: exited,
+	}, files, nil
+}
+
+func (ps *parent) sendReady() error {
+	defer ps.wr.Close()
+	if _, err := ps.wr.Write([]byte{notifyReady}); err != nil {
+		return errors.Wrap(err, "can't notify parent process")
+	}
+	return nil
+}
diff --git a/vendor/github.com/cloudflare/tableflip/process.go b/vendor/github.com/cloudflare/tableflip/process.go
new file mode 100644
index 000000000..c918a88c8
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/process.go
@@ -0,0 +1,47 @@
+package tableflip
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+var initialWD, _ = os.Getwd()
+
+type process interface {
+	fmt.Stringer
+	Signal(sig os.Signal) error
+	Wait() error
+}
+
+type osProcess struct {
+	cmd *exec.Cmd
+}
+
+func newOSProcess(executable string, args []string, files []*os.File, env []string) (process, error) {
+	cmd := exec.Command(executable, args...)
+	cmd.Dir = initialWD
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.ExtraFiles = files
+	cmd.Env = env
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	return &osProcess{cmd}, nil
+}
+
+func (osp *osProcess) Signal(sig os.Signal) error {
+	return osp.cmd.Process.Signal(sig)
+}
+
+func (osp *osProcess) Wait() error {
+	return osp.cmd.Wait()
+}
+
+func (osp *osProcess) String() string {
+	return fmt.Sprintf("pid=%d", osp.cmd.Process.Pid)
+}
diff --git a/vendor/github.com/cloudflare/tableflip/upgrader.go b/vendor/github.com/cloudflare/tableflip/upgrader.go
new file mode 100644
index 000000000..dcf22b967
--- /dev/null
+++ b/vendor/github.com/cloudflare/tableflip/upgrader.go
@@ -0,0 +1,241 @@
+package tableflip
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// DefaultUpgradeTimeout is the duration before the Upgrader kills the new process if no
+// readiness notification was received.
+const DefaultUpgradeTimeout time.Duration = time.Minute
+
+// Options control the behaviour of the Upgrader.
+type Options struct {
+	// Time after which an upgrade is considered failed. Defaults to
+	// DefaultUpgradeTimeout.
+	UpgradeTimeout time.Duration
+	// The PID of a ready process is written to this file.
+	PIDFile string
+}
+
+// Upgrader handles zero downtime upgrades and passing files between processes.
+type Upgrader struct {
+	*env
+	opts       Options
+	parent     *parent
+	readyOnce  sync.Once
+	readyC     chan struct{}
+	stopOnce   sync.Once
+	stopC      chan struct{}
+	upgradeSem chan struct{}
+	exitC      chan struct{}      // only close this if holding upgradeSem
+	exitFd     neverCloseThisFile // protected by upgradeSem
+	parentErr  error              // protected by upgradeSem
+
+	Fds *Fds
+}
+
+var (
+	stdEnvMu       sync.Mutex
+	stdEnvUpgrader *Upgrader
+)
+
+// New creates a new Upgrader. Files are passed from the parent and may be empty.
+//
+// Only the first call to this function will succeed.
+func New(opts Options) (upg *Upgrader, err error) {
+	stdEnvMu.Lock()
+	defer stdEnvMu.Unlock()
+
+	if stdEnvUpgrader != nil {
+		return nil, errors.New("tableflip: only a single Upgrader allowed")
+	}
+
+	upg, err = newUpgrader(stdEnv, opts)
+	// Store a reference to upg in a private global variable, to prevent
+	// it from being GC'ed and exitFd being closed prematurely.
+	stdEnvUpgrader = upg
+	return
+}
+
+func newUpgrader(env *env, opts Options) (*Upgrader, error) {
+	parent, files, err := newParent(env)
+	if err != nil {
+		return nil, err
+	}
+
+	if opts.UpgradeTimeout <= 0 {
+		opts.UpgradeTimeout = DefaultUpgradeTimeout
+	}
+
+	s := &Upgrader{
+		env:        env,
+		opts:       opts,
+		parent:     parent,
+		readyC:     make(chan struct{}),
+		stopC:      make(chan struct{}),
+		upgradeSem: make(chan struct{}, 1),
+		exitC:      make(chan struct{}),
+		Fds:        newFds(files),
+	}
+
+	return s, nil
+}
+
+// Ready signals that the current process is ready to accept connections.
+// It must be called to finish the upgrade.
+//
+// All fds which were inherited but not used are closed after the call to Ready.
+func (u *Upgrader) Ready() error {
+	u.readyOnce.Do(func() {
+		u.Fds.closeInherited()
+		close(u.readyC)
+	})
+
+	if u.opts.PIDFile != "" {
+		if err := writePIDFile(u.opts.PIDFile); err != nil {
+			return errors.Wrap(err, "tableflip: can't write PID file")
+		}
+	}
+
+	if u.parent == nil {
+		return nil
+	}
+	return u.parent.sendReady()
+}
+
+// Exit returns a channel which is closed when the process should
+// exit.
+func (u *Upgrader) Exit() <-chan struct{} {
+	return u.exitC
+}
+
+// Stop prevents any more upgrades from happening, and closes
+// the exit channel.
+func (u *Upgrader) Stop() {
+	u.stopOnce.Do(func() {
+		// Interrupt any running Upgrade(), and
+		// prevent new upgrade from happening.
+		close(u.stopC)
+
+		// Make sure exitC is closed if no upgrade was running.
+		u.upgradeSem <- struct{}{}
+		select {
+		case <-u.exitC:
+		default:
+			close(u.exitC)
+		}
+		<-u.upgradeSem
+
+		u.Fds.closeUsed()
+	})
+}
+
+// Upgrade triggers an upgrade.
+func (u *Upgrader) Upgrade() error {
+	// Acquire semaphore, but don't block. This allows informing
+	// the user that they are doing too many upgrade requests.
+	select {
+	default:
+		return errors.New("upgrade in progress")
+	case u.upgradeSem <- struct{}{}:
+	}
+
+	defer func() {
+		<-u.upgradeSem
+	}()
+
+	// Make sure we're still ok to perform an upgrade.
+	select {
+	case <-u.exitC:
+		return errors.New("already upgraded")
+	default:
+	}
+
+	if u.parent != nil {
+		if u.parentErr != nil {
+			return u.parentErr
+		}
+
+		// verify clean exit
+		select {
+		case err := <-u.parent.exited:
+			if err != nil {
+				u.parentErr = err
+				return err
+			}
+
+		default:
+			return errors.New("parent hasn't exited")
+		}
+	}
+
+	select {
+	case <-u.readyC:
+	default:
+		return errors.New("process is not ready")
+	}
+
+	child, err := startChild(u.env, u.Fds.copy())
+	if err != nil {
+		return errors.Wrap(err, "can't start child")
+	}
+
+	readyTimeout := time.After(u.opts.UpgradeTimeout)
+	select {
+	case err := <-child.exitedC:
+		if err == nil {
+			return errors.Errorf("child %s exited", child)
+		}
+		return errors.Wrap(err, "child %s exited")
+
+	case <-u.stopC:
+		child.Kill()
+		return errors.New("terminating")
+
+	case <-readyTimeout:
+		child.Kill()
+		return errors.Errorf("new child %s timed out", child)
+
+	case file := <-child.readyC:
+		// Save file in exitFd, so that it's only closed when the process
+		// exits. This signals to the new process that the old process
+		// has exited.
+		u.exitFd = neverCloseThisFile{file}
+		close(u.exitC)
+		return nil
+	}
+}
+
+// This file must never be closed by the Go runtime, since its used by the
+// child to determine when the parent has died. It must only be closed
+// by the OS.
+// Hence we make sure that this file can't be garbage collected by referencing
+// it from an Upgrader.
+type neverCloseThisFile struct {
+	file *os.File
+}
+
+func writePIDFile(path string) error {
+	dir, file := filepath.Split(path)
+	fh, err := ioutil.TempFile(dir, file)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+	// Remove temporary PID file if something fails
+	defer os.Remove(fh.Name())
+
+	_, err = fh.WriteString(strconv.Itoa(os.Getpid()))
+	if err != nil {
+		return err
+	}
+
+	return os.Rename(fh.Name(), path)
+}
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 567dcce62..afad93c57 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -27,6 +27,12 @@
 			"versionExact": "2018.01.18"
 		},
 		{
+			"checksumSHA1": "JUqXsmzfQvqtr7cQdOv1c2HUbXE=",
+			"path": "github.com/cloudflare/tableflip",
+			"revision": "3728e38f7e51da2c570af557aabefd1798b1c4bb",
+			"revisionTime": "2018-10-24T12:14:53Z"
+		},
+		{
 			"checksumSHA1": "CSPbwbyzqA6sfORicn4HFtIhF/c=",
 			"path": "github.com/davecgh/go-spew/spew",
 			"revision": "8991bc29aa16c548c550c7ff78260e27b9ab7c73",