Merge branch 'dirwalk-dirents' into 'master'

Avoid unnecessary stat calls when building domain maps

See merge request gitlab-org/gitlab-pages!60

19 files changed, 1505 insertions, 15 deletions

diff --git a/Makefile.util.mk b/Makefile.util.mk
index 18f6b0ff..b5f8d739 100644
--- a/Makefile.util.mk
+++ b/Makefile.util.mk
@@ -17,6 +17,9 @@ complexity: .GOPATH/.ok bin/gocyclo
 test: .GOPATH/.ok gitlab-pages
 	go test $(if $V,-v) -timeout=1m $(allpackages)
 
+bench: .GOPATH/.ok gitlab-pages
+	go test -bench=. -run=^$$ $(allpackages)
+
 # The acceptance tests cannot count for coverage
 cover: bin/gocovmerge .GOPATH/.ok
 	@echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!"
diff --git a/domains.go b/domains.go
index 7ddceb6b..be013dc3 100644
--- a/domains.go
+++ b/domains.go
@@ -8,7 +8,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/karrick/godirwalk"
 	log "github.com/sirupsen/logrus"
+
 	"gitlab.com/gitlab-org/gitlab-pages/metrics"
 )
 
@@ -83,14 +85,8 @@ func (d domains) readProject(rootDomain, group, projectName string) {
 	d.readProjectConfig(rootDomain, group, projectName)
 }
 
-func (d domains) readProjects(rootDomain, group string) {
-	projects, err := os.Open(group)
-	if err != nil {
-		return
-	}
-	defer projects.Close()
-
-	fis, err := projects.Readdir(0)
+func (d domains) readProjects(rootDomain, group string, buf []byte) {
+	fis, err := godirwalk.ReadDirents(group, buf)
 	if err != nil {
 		log.WithError(err).WithFields(log.Fields{
 			"group": group,
@@ -109,13 +105,9 @@ func (d domains) readProjects(rootDomain, group string) {
 }
 
 func (d domains) ReadGroups(rootDomain string) error {
-	groups, err := os.Open(".")
-	if err != nil {
-		return err
-	}
-	defer groups.Close()
+	buf := make([]byte, 2*os.Getpagesize())
 
-	fis, err := groups.Readdir(0)
+	fis, err := godirwalk.ReadDirents(".", buf)
 	if err != nil {
 		return err
 	}
@@ -128,7 +120,7 @@ func (d domains) ReadGroups(rootDomain string) error {
 			continue
 		}
 
-		d.readProjects(rootDomain, group.Name())
+		d.readProjects(rootDomain, group.Name(), buf)
 	}
 
 	return nil
diff --git a/domains_test.go b/domains_test.go
index 892128e5..ab59ac0d 100644
--- a/domains_test.go
+++ b/domains_test.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"crypto/rand"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -105,3 +106,43 @@ func recvTimeout(t *testing.T, ch <-chan domains) domains {
 		return nil
 	}
 }
+
+func BenchmarkReadGroups(b *testing.B) {
+	testRoot, err := ioutil.TempDir("", "gitlab-pages-test")
+	require.NoError(b, err)
+
+	cwd, err := os.Getwd()
+	require.NoError(b, err)
+
+	defer func(oldWd, testWd string) {
+		os.Chdir(oldWd)
+		fmt.Printf("cleaning up test directory %s\n", testWd)
+		os.RemoveAll(testWd)
+	}(cwd, testRoot)
+
+	require.NoError(b, os.Chdir(testRoot))
+
+	nGroups := 10000
+	b.Logf("creating fake domains directory with %d groups", nGroups)
+	for i := 0; i < nGroups; i++ {
+		for j := 0; j < 5; j++ {
+			dir := fmt.Sprintf("%s/group-%d/project-%d", testRoot, i, j)
+			require.NoError(b, os.MkdirAll(dir+"/public", 0755))
+
+			fakeConfig := fmt.Sprintf(`{"Domains":[{"Domain":"foo.%d.%d.example.io","Certificate":"bar","Key":"baz"}]}`, i, j)
+			require.NoError(b, ioutil.WriteFile(dir+"/config.json", []byte(fakeConfig), 0644))
+		}
+		if i%100 == 0 {
+			fmt.Print(".")
+		}
+	}
+
+	b.Run("ReadGroups", func(b *testing.B) {
+		var testDomains domains
+		for i := 0; i < 2; i++ {
+			testDomains = domains(make(map[string]*domain))
+			require.NoError(b, testDomains.ReadGroups("example.com"))
+		}
+		b.Logf("found %d domains", len(testDomains))
+	})
+}
diff --git a/vendor/github.com/karrick/godirwalk/LICENSE b/vendor/github.com/karrick/godirwalk/LICENSE
new file mode 100644
index 00000000..01ce194c
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/LICENSE
@@ -0,0 +1,25 @@
+BSD 2-Clause License
+
+Copyright (c) 2017, Karrick McDermott
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/vendor/github.com/karrick/godirwalk/README.md b/vendor/github.com/karrick/godirwalk/README.md
new file mode 100644
index 00000000..806a2274
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/README.md
@@ -0,0 +1,194 @@
+# godirwalk
+
+`godirwalk` is a library for traversing a directory tree on a file
+system.
+
+In short, why do I use this library?
+
+1. It's faster than `filepath.Walk`.
+1. It's more correct on Windows than `filepath.Walk`.
+1. It's more easy to use than `filepath.Walk`.
+1. It's more flexible than `filepath.Walk`.
+
+## Usage Example
+
+Additional examples are provided in the `examples/` subdirectory.
+
+This library will normalize the provided top level directory name
+based on the os-specific path separator by calling `filepath.Clean` on
+its first argument. However it always provides the pathname created by
+using the correct os-specific path separator when invoking the
+provided callback function.
+
+```Go
+    dirname := "some/directory/root"
+    err := godirwalk.Walk(dirname, &godirwalk.Options{
+        Callback: func(osPathname string, de *godirwalk.Dirent) error {
+            fmt.Printf("%s %s\n", de.ModeType(), osPathname)
+            return nil
+        },
+    })
+```
+
+This library not only provides functions for traversing a file system
+directory tree, but also for obtaining a list of immediate descendants
+of a particular directory, typically much more quickly than using
+`os.ReadDir` or `os.ReadDirnames`.
+
+Documentation is available via
+[![GoDoc](https://godoc.org/github.com/karrick/godirwalk?status.svg)](https://godoc.org/github.com/karrick/godirwalk).
+
+## Description
+
+Here's why I use `godirwalk` in preference to `filepath.Walk`,
+`os.ReadDir`, and `os.ReadDirnames`.
+
+### It's faster than `filepath.Walk`
+
+When compared against `filepath.Walk` in benchmarks, it has been
+observed to run between five and ten times the speed on darwin, at
+speeds comparable to the that of the unix `find` utility; about twice
+the speed on linux; and about four times the speed on Windows.
+
+How does it obtain this performance boost? Primarily by not invoking
+`os.Stat` on every file system node it encounters.
+
+While traversing a file system directory tree, `filepath.Walk` obtains
+the list of immediate descendants of a directory, and throws away the
+file system node type information provided by the operating system
+that comes with the node's name. Then, immediately prior to invoking
+the callback function, `filepath.Walk` invokes `os.Stat` for each
+node, and passes the returned `os.FileInfo` information to the
+callback.
+
+While the `os.FileInfo` information provided by `os.Stat` is extremely
+helpful--and even includes the `os.FileMode` data--providing it
+requires an additional system call for each node.
+
+Because most callbacks only care about what the node type is, this
+library does not throw the type information away, but rather provides
+that information to the callback function in the form of a
+`os.FileMode` value. Note that the provided `os.FileMode` value that
+this library provides only has the node type information, and does not
+have the permission bits, sticky bits, or other information from the
+file's mode. If the callback does care about a particular node's
+entire `os.FileInfo` data structure, the callback can easiy invoke
+`os.Stat` when needed, and only when needed.
+
+#### Benchmarks
+
+##### macOS
+
+```Bash
+go test -bench=.
+goos: darwin
+goarch: amd64
+pkg: github.com/karrick/godirwalk
+BenchmarkFilepathWalk-8             	       1	3001274570 ns/op
+BenchmarkGoDirWalk-8                	       3	 465573172 ns/op
+BenchmarkFlameGraphFilepathWalk-8   	       1	6957916936 ns/op
+BenchmarkFlameGraphGoDirWalk-8      	       1	4210582571 ns/op
+PASS
+ok  	github.com/karrick/godirwalk	16.822s
+```
+
+##### Linux
+
+```Bash
+go test -bench=.
+goos: linux
+goarch: amd64
+pkg: github.com/karrick/godirwalk
+BenchmarkFilepathWalk-12              	       1	1609189170 ns/op
+BenchmarkGoDirWalk-12                 	       5	 211336628 ns/op
+BenchmarkFlameGraphFilepathWalk-12    	       1	3968119932 ns/op
+BenchmarkFlameGraphGoDirWalk-12       	       1	2139598998 ns/op
+PASS
+ok  	github.com/karrick/godirwalk	9.007s
+```
+
+### It's more correct on Windows than `filepath.Walk`
+
+I did not previously care about this either, but humor me. We all love
+how we can write once and run everywhere. It is essential for the
+language's adoption, growth, and success, that the software we create
+can run unmodified on all architectures and operating systems
+supported by Go.
+
+When the traversed file system has a logical loop caused by symbolic
+links to directories, on unix `filepath.Walk` ignores symbolic links
+and traverses the entire directory tree without error. On Windows
+however, `filepath.Walk` will continue following directory symbolic
+links, even though it is not supposed to, eventually causing
+`filepath.Walk` to terminate early and return an error when the
+pathname gets too long from concatenating endless loops of symbolic
+links onto the pathname. This error comes from Windows, passes through
+`filepath.Walk`, and to the upstream client running `filepath.Walk`.
+
+The takeaway is that behavior is different based on which platform
+`filepath.Walk` is running. While this is clearly not intentional,
+until it is fixed in the standard library, it presents a compatibility
+problem.
+
+This library correctly identifies symbolic links that point to
+directories and will only follow them when `FollowSymbolicLinks` is
+set to true. Behavior on Windows and other operating systems is
+identical.
+
+### It's more easy to use than `filepath.Walk`
+
+Since this library does not invoke `os.Stat` on every file system node
+it encounters, there is no possible error event for the callback
+function to filter on. The third argument in the `filepath.WalkFunc`
+function signature to pass the error from `os.Stat` to the callback
+function is no longer necessary, and thus eliminated from signature of
+the callback function from this library.
+
+Also, `filepath.Walk` invokes the callback function with a solidus
+delimited pathname regardless of the os-specific path separator. This
+library invokes the callback function with the os-specific pathname
+separator, obviating a call to `filepath.Clean` in the callback
+function for each node prior to actually using the provided pathname.
+
+In other words, even on Windows, `filepath.Walk` will invoke the
+callback with `some/path/to/foo.txt`, requiring well written clients
+to perform pathname normalization for every file prior to working with
+the specified file. In truth, many clients developed on unix and not
+tested on Windows neglect this subtlety, and will result in software
+bugs when running on Windows. This library would invoke the callback
+function with `some\path\to\foo.txt` for the same file when running on
+Windows, eliminating the need to normalize the pathname by the client,
+and lessen the likelyhood that a client will work on unix but not on
+Windows.
+
+### It's more flexible than `filepath.Walk`
+
+The default behavior of this library is to ignore symbolic links to
+directories when walking a directory tree, just like `filepath.Walk`
+does. However, it does invoke the callback function with each node it
+finds, including symbolic links. If a particular use case exists to
+follow symbolic links when traversing a directory tree, this library
+can be invoked in manner to do so, by setting the
+`FollowSymbolicLinks` parameter to true.
+
+The default behavior of this library is to always sort the immediate
+descendants of a directory prior to visiting each node, just like
+`filepath.Walk` does. This is usually the desired behavior. However,
+this does come at a performance penalty to sort the names when a
+directory node has many entries. If a particular use case exists that
+does not require sorting the directory's immediate descendants prior
+to visiting its nodes, this library will skip the sorting step when
+the `Unsorted` parameter is set to true.
+
+This library provides upstream code with the ability to specify a
+callback to be invoked for each directory after its children are
+processed. This has been used to recursively delete empty directories
+after traversing the file system in a more efficient manner. See the
+`examples/clean-empties` directory for an example of this usage.
+
+This library provides upstream code with the ability to specify a
+callback to be invoked for errors that the operating system returns,
+allowing the upstream code to determine the next course of action to
+take, whether to halt walking the hierarchy, as it would do were no
+error callback provided, or skip the node that caused the error. See
+the `examples/walk-fast` directory for an example of this usage.
diff --git a/vendor/github.com/karrick/godirwalk/doc.go b/vendor/github.com/karrick/godirwalk/doc.go
new file mode 100644
index 00000000..0dfdabd4
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/doc.go
@@ -0,0 +1,34 @@
+/*
+Package godirwalk provides functions to read and traverse directory trees.
+
+In short, why do I use this library?
+
+* It's faster than `filepath.Walk`.
+
+* It's more correct on Windows than `filepath.Walk`.
+
+* It's more easy to use than `filepath.Walk`.
+
+* It's more flexible than `filepath.Walk`.
+
+USAGE
+
+This library will normalize the provided top level directory name based on the
+os-specific path separator by calling `filepath.Clean` on its first
+argument. However it always provides the pathname created by using the correct
+os-specific path separator when invoking the provided callback function.
+
+    dirname := "some/directory/root"
+    err := godirwalk.Walk(dirname, &godirwalk.Options{
+        Callback: func(osPathname string, de *godirwalk.Dirent) error {
+            fmt.Printf("%s %s\n", de.ModeType(), osPathname)
+            return nil
+        },
+    })
+
+This library not only provides functions for traversing a file system directory
+tree, but also for obtaining a list of immediate descendants of a particular
+directory, typically much more quickly than using `os.ReadDir` or
+`os.ReadDirnames`.
+*/
+package godirwalk
diff --git a/vendor/github.com/karrick/godirwalk/go.mod b/vendor/github.com/karrick/godirwalk/go.mod
new file mode 100644
index 00000000..8d95e69c
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/go.mod
@@ -0,0 +1,3 @@
+module "github.com/karrick/godirwalk"
+
+require "github.com/pkg/errors" v0.8.0
diff --git a/vendor/github.com/karrick/godirwalk/readdir.go b/vendor/github.com/karrick/godirwalk/readdir.go
new file mode 100644
index 00000000..eae4a0ee
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/readdir.go
@@ -0,0 +1,97 @@
+package godirwalk
+
+import (
+	"os"
+)
+
+// Dirent stores the name and file system mode type of discovered file system
+// entries.
+type Dirent struct {
+	name     string
+	modeType os.FileMode
+}
+
+// Name returns the basename of the file system entry.
+func (de Dirent) Name() string { return de.name }
+
+// ModeType returns the mode bits that specify the file system node type. We
+// could make our own enum-like data type for encoding the file type, but Go's
+// runtime already gives us architecture independent file modes, as discussed in
+// `os/types.go`:
+//
+//    Go's runtime FileMode type has same definition on all systems, so that
+//    information about files can be moved from one system to another portably.
+func (de Dirent) ModeType() os.FileMode { return de.modeType }
+
+// IsDir returns true if and only if the Dirent represents a file system
+// directory. Note that on some operating systems, more than one file mode bit
+// may be set for a node. For instance, on Windows, a symbolic link that points
+// to a directory will have both the directory and the symbolic link bits set.
+func (de Dirent) IsDir() bool { return de.modeType&os.ModeDir != 0 }
+
+// IsSymlink returns true if and only if the Dirent represents a file system
+// symbolic link. Note that on some operating systems, more than one file mode
+// bit may be set for a node. For instance, on Windows, a symbolic link that
+// points to a directory will have both the directory and the symbolic link bits
+// set.
+func (de Dirent) IsSymlink() bool { return de.modeType&os.ModeSymlink != 0 }
+
+// Dirents represents a slice of Dirent pointers, which are sortable by
+// name. This type satisfies the `sort.Interface` interface.
+type Dirents []*Dirent
+
+// Len returns the count of Dirent structures in the slice.
+func (l Dirents) Len() int { return len(l) }
+
+// Less returns true if and only if the Name of the element specified by the
+// first index is lexicographically less than that of the second index.
+func (l Dirents) Less(i, j int) bool { return l[i].name < l[j].name }
+
+// Swap exchanges the two Dirent entries specified by the two provided indexes.
+func (l Dirents) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
+
+// ReadDirents returns a sortable slice of pointers to Dirent structures, each
+// representing the file system name and mode type for one of the immediate
+// descendant of the specified directory. If the specified directory is a
+// symbolic link, it will be resolved.
+//
+// If an optional scratch buffer is provided that is at least one page of
+// memory, it will be used when reading directory entries from the file system.
+//
+//    children, err := godirwalk.ReadDirents(osDirname, nil)
+//    if err != nil {
+//        return nil, errors.Wrap(err, "cannot get list of directory children")
+//    }
+//    sort.Sort(children)
+//    for _, child := range children {
+//        fmt.Printf("%s %s\n", child.ModeType, child.Name)
+//    }
+func ReadDirents(osDirname string, scratchBuffer []byte) (Dirents, error) {
+	return readdirents(osDirname, scratchBuffer)
+}
+
+// ReadDirnames returns a slice of strings, representing the immediate
+// descendants of the specified directory. If the specified directory is a
+// symbolic link, it will be resolved.
+//
+// If an optional scratch buffer is provided that is at least one page of
+// memory, it will be used when reading directory entries from the file system.
+//
+// Note that this function, depending on operating system, may or may not invoke
+// the ReadDirents function, in order to prepare the list of immediate
+// descendants. Therefore, if your program needs both the names and the file
+// system mode types of descendants, it will always be faster to invoke
+// ReadDirents directly, rather than calling this function, then looping over
+// the results and calling os.Stat for each child.
+//
+//    children, err := godirwalk.ReadDirnames(osDirname, nil)
+//    if err != nil {
+//        return nil, errors.Wrap(err, "cannot get list of directory children")
+//    }
+//    sort.Strings(children)
+//    for _, child := range children {
+//        fmt.Printf("%s\n", child)
+//    }
+func ReadDirnames(osDirname string, scratchBuffer []byte) ([]string, error) {
+	return readdirnames(osDirname, scratchBuffer)
+}
diff --git a/vendor/github.com/karrick/godirwalk/readdir_unix.go b/vendor/github.com/karrick/godirwalk/readdir_unix.go
new file mode 100644
index 00000000..2d86434e
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/readdir_unix.go
@@ -0,0 +1,116 @@
+// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris
+
+package godirwalk
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+
+	"github.com/pkg/errors"
+)
+
+var defaultBufferSize, pageSize int
+
+func init() {
+	pageSize = os.Getpagesize()
+	defaultBufferSize = 2 * pageSize
+}
+
+func readdirents(osDirname string, scratchBuffer []byte) (Dirents, error) {
+	dh, err := os.Open(osDirname)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot Open")
+	}
+
+	var entries Dirents
+
+	fd := int(dh.Fd())
+
+	if len(scratchBuffer) < pageSize {
+		scratchBuffer = make([]byte, defaultBufferSize)
+	}
+
+	var de *syscall.Dirent
+
+	for {
+		n, err := syscall.ReadDirent(fd, scratchBuffer)
+		if err != nil {
+			_ = dh.Close() // ignore potential error returned by Close
+			return nil, errors.Wrap(err, "cannot ReadDirent")
+		}
+		if n <= 0 {
+			break // end of directory reached
+		}
+		// Loop over the bytes returned by reading the directory entries.
+		buf := scratchBuffer[:n]
+		for len(buf) > 0 {
+			de = (*syscall.Dirent)(unsafe.Pointer(&buf[0])) // point entry to first syscall.Dirent in buffer
+			buf = buf[de.Reclen:]                           // advance buffer
+
+			if de.Ino == 0 {
+				continue // this item has been deleted, but not yet removed from directory
+			}
+
+			nameSlice := nameFromDirent(de)
+			namlen := len(nameSlice)
+			if (namlen == 0) || (namlen == 1 && nameSlice[0] == '.') || (namlen == 2 && nameSlice[0] == '.' && nameSlice[1] == '.') {
+				continue // skip unimportant entries
+			}
+			osChildname := string(nameSlice)
+
+			// Convert syscall constant, which is in purview of OS, to a
+			// constant defined by Go, assumed by this project to be stable.
+			var mode os.FileMode
+			switch de.Type {
+			case syscall.DT_REG:
+				// regular file
+			case syscall.DT_DIR:
+				mode = os.ModeDir
+			case syscall.DT_LNK:
+				mode = os.ModeSymlink
+			case syscall.DT_CHR:
+				mode = os.ModeDevice | os.ModeCharDevice
+			case syscall.DT_BLK:
+				mode = os.ModeDevice
+			case syscall.DT_FIFO:
+				mode = os.ModeNamedPipe
+			case syscall.DT_SOCK:
+				mode = os.ModeSocket
+			default:
+				// If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT),
+				// then resolve actual mode by getting stat.
+				fi, err := os.Stat(filepath.Join(osDirname, osChildname))
+				if err != nil {
+					_ = dh.Close() // ignore potential error returned by Close
+					return nil, errors.Wrap(err, "cannot Stat")
+				}
+				// We only care about the bits that identify the type of a file
+				// system node, and can ignore append, exclusive, temporary,
+				// setuid, setgid, permission bits, and sticky bits, which are
+				// coincident to the bits that declare type of the file system
+				// node.
+				mode = fi.Mode() & os.ModeType
+			}
+
+			entries = append(entries, &Dirent{name: osChildname, modeType: mode})
+		}
+	}
+	if err = dh.Close(); err != nil {
+		return nil, err
+	}
+	return entries, nil
+}
+
+func readdirnames(osDirname string, scratchBuffer []byte) ([]string, error) {
+	des, err := readdirents(osDirname, scratchBuffer)
+	if err != nil {
+		return nil, err
+	}
+	names := make([]string, len(des))
+	for i, v := range des {
+		names[i] = v.name
+	}
+	return names, nil
+}
diff --git a/vendor/github.com/karrick/godirwalk/readdir_windows.go b/vendor/github.com/karrick/godirwalk/readdir_windows.go
new file mode 100644
index 00000000..e0c708d4
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/readdir_windows.go
@@ -0,0 +1,54 @@
+package godirwalk
+
+import (
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+// The functions in this file are mere wrappers of what is already provided by
+// standard library, in order to provide the same API as this library provides.
+//
+// The scratch buffer argument is ignored by this architecture.
+//
+// Please send PR or link to article if you know of a more performant way of
+// enumerating directory contents and hopefully mode types on Windows.
+
+func readdirents(osDirname string, _ []byte) (Dirents, error) {
+	dh, err := os.Open(osDirname)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot Open")
+	}
+
+	fileinfos, err := dh.Readdir(0)
+	if er := dh.Close(); err == nil {
+		err = er
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot Readdir")
+	}
+
+	entries := make(Dirents, len(fileinfos))
+	for i, info := range fileinfos {
+		entries[i] = &Dirent{name: info.Name(), modeType: info.Mode() & os.ModeType}
+	}
+
+	return entries, nil
+}
+
+func readdirnames(osDirname string, _ []byte) ([]string, error) {
+	dh, err := os.Open(osDirname)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot Open")
+	}
+
+	entries, err := dh.Readdirnames(0)
+	if er := dh.Close(); err == nil {
+		err = er
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot Readdirnames")
+	}
+
+	return entries, nil
+}
diff --git a/vendor/github.com/karrick/godirwalk/walk.go b/vendor/github.com/karrick/godirwalk/walk.go
new file mode 100644
index 00000000..26db25f5
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/walk.go
@@ -0,0 +1,329 @@
+package godirwalk
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/pkg/errors"
+)
+
+// Options provide parameters for how the Walk function operates.
+type Options struct {
+	// ErrorCallback specifies a function to be invoked in the case of an error
+	// that could potentially be ignored while walking a file system
+	// hierarchy. When set to nil or left as its zero-value, any error condition
+	// causes Walk to immediately return the error describing what took
+	// place. When non-nil, this user supplied function is invoked with the OS
+	// pathname of the file system object that caused the error along with the
+	// error that took place. The return value of the supplied ErrorCallback
+	// function determines whether the error will cause Walk to halt immediately
+	// as it would were no ErrorCallback value provided, or skip this file
+	// system node yet continue on with the remaining nodes in the file system
+	// hierarchy.
+	//
+	// ErrorCallback is invoked both for errors that are returned by the
+	// runtime, and for errors returned by other user supplied callback
+	// functions.
+	ErrorCallback func(string, error) ErrorAction
+
+	// FollowSymbolicLinks specifies whether Walk will follow symbolic links
+	// that refer to directories. When set to false or left as its zero-value,
+	// Walk will still invoke the callback function with symbolic link nodes,
+	// but if the symbolic link refers to a directory, it will not recurse on
+	// that directory. When set to true, Walk will recurse on symbolic links
+	// that refer to a directory.
+	FollowSymbolicLinks bool
+
+	// Unsorted controls whether or not Walk will sort the immediate descendants
+	// of a directory by their relative names prior to visiting each of those
+	// entries.
+	//
+	// When set to false or left at its zero-value, Walk will get the list of
+	// immediate descendants of a particular directory, sort that list by
+	// lexical order of their names, and then visit each node in the list in
+	// sorted order. This will cause Walk to always traverse the same directory
+	// tree in the same order, however may be inefficient for directories with
+	// many immediate descendants.
+	//
+	// When set to true, Walk skips sorting the list of immediate descendants
+	// for a directory, and simply visits each node in the order the operating
+	// system enumerated them. This will be more fast, but with the side effect
+	// that the traversal order may be different from one invocation to the
+	// next.
+	Unsorted bool
+
+	// Callback is a required function that Walk will invoke for every file
+	// system node it encounters.
+	Callback WalkFunc
+
+	// PostChildrenCallback is an option function that Walk will invoke for
+	// every file system directory it encounters after its children have been
+	// processed.
+	PostChildrenCallback WalkFunc
+
+	// ScratchBuffer is an optional scratch buffer for Walk to use when reading
+	// directory entries, to reduce amount of garbage generation. Not all
+	// architectures take advantage of the scratch buffer.
+	ScratchBuffer []byte
+}
+
+// ErrorAction defines a set of actions the Walk function could take based on
+// the occurrence of an error while walking the file system. See the
+// documentation for the ErrorCallback field of the Options structure for more
+// information.
+type ErrorAction int
+
+const (
+	// Halt is the ErrorAction return value when the upstream code wants to halt
+	// the walk process when a runtime error takes place. It matches the default
+	// action the Walk function would take were no ErrorCallback provided.
+	Halt ErrorAction = iota
+
+	// SkipNode is the ErrorAction return value when the upstream code wants to
+	// ignore the runtime error for the current file system node, skip
+	// processing of the node that caused the error, and continue walking the
+	// file system hierarchy with the remaining nodes.
+	SkipNode
+)
+
+// WalkFunc is the type of the function called for each file system node visited
+// by Walk. The pathname argument will contain the argument to Walk as a prefix;
+// that is, if Walk is called with "dir", which is a directory containing the
+// file "a", the provided WalkFunc will be invoked with the argument "dir/a",
+// using the correct os.PathSeparator for the Go Operating System architecture,
+// GOOS. The directory entry argument is a pointer to a Dirent for the node,
+// providing access to both the basename and the mode type of the file system
+// node.
+//
+// If an error is returned by the Callback or PostChildrenCallback functions,
+// and no ErrorCallback function is provided, processing stops. If an
+// ErrorCallback function is provided, then it is invoked with the OS pathname
+// of the node that caused the error along along with the error. The return
+// value of the ErrorCallback function determines whether to halt processing, or
+// skip this node and continue processing remaining file system nodes.
+//
+// The exception is when the function returns the special value
+// filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a
+// directory, Walk skips the directory's contents entirely. If the function
+// returns filepath.SkipDir when invoked on a non-directory file system node,
+// Walk skips the remaining files in the containing directory. Note that any
+// supplied ErrorCallback function is not invoked with filepath.SkipDir when the
+// Callback or PostChildrenCallback functions return that special value.
+type WalkFunc func(osPathname string, directoryEntry *Dirent) error
+
+// Walk walks the file tree rooted at the specified directory, calling the
+// specified callback function for each file system node in the tree, including
+// root, symbolic links, and other node types. The nodes are walked in lexical
+// order, which makes the output deterministic but means that for very large
+// directories this function can be inefficient.
+//
+// This function is often much faster than filepath.Walk because it does not
+// invoke os.Stat for every node it encounters, but rather obtains the file
+// system node type when it reads the parent directory.
+//
+// If a runtime error occurs, either from the operating system or from the
+// upstream Callback or PostChildrenCallback functions, processing typically
+// halts. However, when an ErrorCallback function is provided in the provided
+// Options structure, that function is invoked with the error along with the OS
+// pathname of the file system node that caused the error. The ErrorCallback
+// function's return value determines the action that Walk will then take.
+//
+//    func main() {
+//        dirname := "."
+//        if len(os.Args) > 1 {
+//            dirname = os.Args[1]
+//        }
+//        err := godirwalk.Walk(dirname, &godirwalk.Options{
+//            Callback: func(osPathname string, de *godirwalk.Dirent) error {
+//                fmt.Printf("%s %s\n", de.ModeType(), osPathname)
+//                return nil
+//            },
+//            ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction {
+//            	// Your program may want to log the error somehow.
+//            	fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
+//
+//            	// For the purposes of this example, a simple SkipNode will suffice,
+//            	// although in reality perhaps additional logic might be called for.
+//            	return godirwalk.SkipNode
+//            },
+//        })
+//        if err != nil {
+//            fmt.Fprintf(os.Stderr, "%s\n", err)
+//            os.Exit(1)
+//        }
+//    }
+func Walk(pathname string, options *Options) error {
+	pathname = filepath.Clean(pathname)
+
+	var fi os.FileInfo
+	var err error
+
+	if options.FollowSymbolicLinks {
+		fi, err = os.Stat(pathname)
+		if err != nil {
+			return errors.Wrap(err, "cannot Stat")
+		}
+	} else {
+		fi, err = os.Lstat(pathname)
+		if err != nil {
+			return errors.Wrap(err, "cannot Lstat")
+		}
+	}
+
+	mode := fi.Mode()
+	if mode&os.ModeDir == 0 {
+		return errors.Errorf("cannot Walk non-directory: %s", pathname)
+	}
+
+	dirent := &Dirent{
+		name:     filepath.Base(pathname),
+		modeType: mode & os.ModeType,
+	}
+
+	// If ErrorCallback is nil, set to a default value that halts the walk
+	// process on all operating system errors. This is done to allow error
+	// handling to be more succinct in the walk code.
+	if options.ErrorCallback == nil {
+		options.ErrorCallback = defaultErrorCallback
+	}
+
+	err = walk(pathname, dirent, options)
+	if err == filepath.SkipDir {
+		return nil // silence SkipDir for top level
+	}
+	return err
+}
+
+// defaultErrorCallback always returns Halt because if the upstream code did not
+// provide an ErrorCallback function, walking the file system hierarchy ought to
+// halt upon any operating system error.
+func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt }
+
+// walk recursively traverses the file system node specified by pathname and the
+// Dirent.
+func walk(osPathname string, dirent *Dirent, options *Options) error {
+	err := options.Callback(osPathname, dirent)
+	if err != nil {
+		if err == filepath.SkipDir {
+			return err
+		}
+		err = errors.Wrap(err, "Callback") // wrap potential errors returned by callback
+		if action := options.ErrorCallback(osPathname, err); action == SkipNode {
+			return nil
+		}
+		return err
+	}
+
+	// On some platforms, an entry can have more than one mode type bit set.
+	// For instance, it could have both the symlink bit and the directory bit
+	// set indicating it's a symlink to a directory.
+	if dirent.IsSymlink() {
+		if !options.FollowSymbolicLinks {
+			return nil
+		}
+		// Only need to Stat entry if platform did not already have os.ModeDir
+		// set, such as would be the case for unix like operating systems. (This
+		// guard eliminates extra os.Stat check on Windows.)
+		if !dirent.IsDir() {
+			referent, err := os.Readlink(osPathname)
+			if err != nil {
+				err = errors.Wrap(err, "cannot Readlink")
+				if action := options.ErrorCallback(osPathname, err); action == SkipNode {
+					return nil
+				}
+				return err
+			}
+
+			osp := filepath.Join(filepath.Dir(osPathname), referent)
+			fi, err := os.Stat(osp)
+			if err != nil {
+				err = errors.Wrap(err, "cannot Stat")
+				if action := options.ErrorCallback(osp, err); action == SkipNode {
+					return nil
+				}
+				return err
+			}
+			dirent.modeType = fi.Mode() & os.ModeType
+		}
+	}
+
+	if !dirent.IsDir() {
+		return nil
+	}
+
+	// If get here, then specified pathname refers to a directory.
+	deChildren, err := ReadDirents(osPathname, options.ScratchBuffer)
+	if err != nil {
+		err = errors.Wrap(err, "cannot ReadDirents")
+		if action := options.ErrorCallback(osPathname, err); action == SkipNode {
+			return nil
+		}
+		return err
+	}
+
+	if !options.Unsorted {
+		sort.Sort(deChildren) // sort children entries unless upstream says to leave unsorted
+	}
+
+	for _, deChild := range deChildren {
+		osChildname := filepath.Join(osPathname, deChild.name)
+		err = walk(osChildname, deChild, options)
+		if err != nil {
+			if err != filepath.SkipDir {
+				return err
+			}
+			// If received skipdir on a directory, stop processing that
+			// directory, but continue to its siblings. If received skipdir on a
+			// non-directory, stop processing remaining siblings.
+			if deChild.IsSymlink() {
+				// Only need to Stat entry if platform did not already have
+				// os.ModeDir set, such as would be the case for unix like
+				// operating systems. (This guard eliminates extra os.Stat check
+				// on Windows.)
+				if !deChild.IsDir() {
+					// Resolve symbolic link referent to determine whether node
+					// is directory or not.
+					referent, err := os.Readlink(osChildname)
+					if err != nil {
+						err = errors.Wrap(err, "cannot Readlink")
+						if action := options.ErrorCallback(osChildname, err); action == SkipNode {
+							continue // with next child
+						}
+						return err
+					}
+					osp := filepath.Join(osPathname, referent)
+					fi, err := os.Stat(osp)
+					if err != nil {
+						err = errors.Wrap(err, "cannot Stat")
+						if action := options.ErrorCallback(osp, err); action == SkipNode {
+							continue // with next child
+						}
+						return err
+					}
+					deChild.modeType = fi.Mode() & os.ModeType
+				}
+			}
+			if !deChild.IsDir() {
+				// If not directory, return immediately, thus skipping remainder
+				// of siblings.
+				return nil
+			}
+		}
+	}
+
+	if options.PostChildrenCallback == nil {
+		return nil
+	}
+
+	err = options.PostChildrenCallback(osPathname, dirent)
+	if err == nil || err == filepath.SkipDir {
+		return err
+	}
+
+	err = errors.Wrap(err, "PostChildrenCallback") // wrap potential errors returned by callback
+	if action := options.ErrorCallback(osPathname, err); action == SkipNode {
+		return nil
+	}
+	return err
+}
diff --git a/vendor/github.com/karrick/godirwalk/withNamlen.go b/vendor/github.com/karrick/godirwalk/withNamlen.go
new file mode 100644
index 00000000..46a4af50
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/withNamlen.go
@@ -0,0 +1,29 @@
+// +build darwin dragonfly freebsd netbsd openbsd
+
+package godirwalk
+
+import (
+	"reflect"
+	"syscall"
+	"unsafe"
+)
+
+func nameFromDirent(de *syscall.Dirent) []byte {
+	// Because this GOOS' syscall.Dirent provides a Namlen field that says how
+	// long the name is, this function does not need to search for the NULL
+	// byte.
+	ml := int(de.Namlen)
+
+	// Convert syscall.Dirent.Name, which is array of int8, to []byte, by
+	// overwriting Cap, Len, and Data slice header fields to values from
+	// syscall.Dirent fields. Setting the Cap, Len, and Data field values for
+	// the slice header modifies what the slice header points to, and in this
+	// case, the name buffer.
+	var name []byte
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&name))
+	sh.Cap = ml
+	sh.Len = ml
+	sh.Data = uintptr(unsafe.Pointer(&de.Name[0]))
+
+	return name
+}
diff --git a/vendor/github.com/karrick/godirwalk/withoutNamlen.go b/vendor/github.com/karrick/godirwalk/withoutNamlen.go
new file mode 100644
index 00000000..dcf9f3a9
--- /dev/null
+++ b/vendor/github.com/karrick/godirwalk/withoutNamlen.go
@@ -0,0 +1,36 @@
+// +build nacl linux solaris
+
+package godirwalk
+
+import (
+	"bytes"
+	"reflect"
+	"syscall"
+	"unsafe"
+)
+
+func nameFromDirent(de *syscall.Dirent) []byte {
+	// Because this GOOS' syscall.Dirent does not provide a field that specifies
+	// the name length, this function must first calculate the max possible name
+	// length, and then search for the NULL byte.
+	ml := int(uint64(de.Reclen) - uint64(unsafe.Offsetof(syscall.Dirent{}.Name)))
+
+	// Convert syscall.Dirent.Name, which is array of int8, to []byte, by
+	// overwriting Cap, Len, and Data slice header fields to values from
+	// syscall.Dirent fields. Setting the Cap, Len, and Data field values for
+	// the slice header modifies what the slice header points to, and in this
+	// case, the name buffer.
+	var name []byte
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&name))
+	sh.Cap = ml
+	sh.Len = ml
+	sh.Data = uintptr(unsafe.Pointer(&de.Name[0]))
+
+	if index := bytes.IndexByte(name, 0); index >= 0 {
+		// Found NULL byte; set slice's cap and len accordingly.
+		sh.Cap = index
+		sh.Len = index
+	}
+
+	return name
+}
diff --git a/vendor/github.com/pkg/errors/LICENSE b/vendor/github.com/pkg/errors/LICENSE
new file mode 100644
index 00000000..835ba3e7
--- /dev/null
+++ b/vendor/github.com/pkg/errors/LICENSE
@@ -0,0 +1,23 @@
+Copyright (c) 2015, Dave Cheney <dave@cheney.net>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/vendor/github.com/pkg/errors/README.md b/vendor/github.com/pkg/errors/README.md
new file mode 100644
index 00000000..6483ba2a
--- /dev/null
+++ b/vendor/github.com/pkg/errors/README.md
@@ -0,0 +1,52 @@
+# errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors) [![Sourcegraph](https://sourcegraph.com/github.com/pkg/errors/-/badge.svg)](https://sourcegraph.com/github.com/pkg/errors?badge)
+
+Package errors provides simple error handling primitives.
+
+`go get github.com/pkg/errors`
+
+The traditional error handling idiom in Go is roughly akin to
+```go
+if err != nil {
+        return err
+}
+```
+which applied recursively up the call stack results in error reports without context or debugging information. The errors package allows programmers to add context to the failure path in their code in a way that does not destroy the original value of the error.
+
+## Adding context to an error
+
+The errors.Wrap function returns a new error that adds context to the original error. For example
+```go
+_, err := ioutil.ReadAll(r)
+if err != nil {
+        return errors.Wrap(err, "read failed")
+}
+```
+## Retrieving the cause of an error
+
+Using `errors.Wrap` constructs a stack of errors, adding context to the preceding error. Depending on the nature of the error it may be necessary to reverse the operation of errors.Wrap to retrieve the original error for inspection. Any error value which implements this interface can be inspected by `errors.Cause`.
+```go
+type causer interface {
+        Cause() error
+}
+```
+`errors.Cause` will recursively retrieve the topmost error which does not implement `causer`, which is assumed to be the original cause. For example:
+```go
+switch err := errors.Cause(err).(type) {
+case *MyError:
+        // handle specifically
+default:
+        // unknown error
+}
+```
+
+[Read the package documentation for more information](https://godoc.org/github.com/pkg/errors).
+
+## Contributing
+
+We welcome pull requests, bug fixes and issue reports. With that said, the bar for adding new symbols to this package is intentionally set high.
+
+Before proposing a change, please discuss your change by raising an issue.
+
+## License
+
+BSD-2-Clause
diff --git a/vendor/github.com/pkg/errors/appveyor.yml b/vendor/github.com/pkg/errors/appveyor.yml
new file mode 100644
index 00000000..a932eade
--- /dev/null
+++ b/vendor/github.com/pkg/errors/appveyor.yml
@@ -0,0 +1,32 @@
+version: build-{build}.{branch}
+
+clone_folder: C:\gopath\src\github.com\pkg\errors
+shallow_clone: true # for startup speed
+
+environment:
+  GOPATH: C:\gopath
+
+platform:
+  - x64
+
+# http://www.appveyor.com/docs/installed-software
+install:
+  # some helpful output for debugging builds
+  - go version
+  - go env
+  # pre-installed MinGW at C:\MinGW is 32bit only
+  # but MSYS2 at C:\msys64 has mingw64
+  - set PATH=C:\msys64\mingw64\bin;%PATH%
+  - gcc --version
+  - g++ --version
+
+build_script:
+  - go install -v ./...
+
+test_script:
+  - set PATH=C:\gopath\bin;%PATH%
+  - go test -v ./...
+
+#artifacts:
+#  - path: '%GOPATH%\bin\*.exe'
+deploy: off
diff --git a/vendor/github.com/pkg/errors/errors.go b/vendor/github.com/pkg/errors/errors.go
new file mode 100644
index 00000000..842ee804
--- /dev/null
+++ b/vendor/github.com/pkg/errors/errors.go
@@ -0,0 +1,269 @@
+// Package errors provides simple error handling primitives.
+//
+// The traditional error handling idiom in Go is roughly akin to
+//
+//     if err != nil {
+//             return err
+//     }
+//
+// which applied recursively up the call stack results in error reports
+// without context or debugging information. The errors package allows
+// programmers to add context to the failure path in their code in a way
+// that does not destroy the original value of the error.
+//
+// Adding context to an error
+//
+// The errors.Wrap function returns a new error that adds context to the
+// original error by recording a stack trace at the point Wrap is called,
+// and the supplied message. For example
+//
+//     _, err := ioutil.ReadAll(r)
+//     if err != nil {
+//             return errors.Wrap(err, "read failed")
+//     }
+//
+// If additional control is required the errors.WithStack and errors.WithMessage
+// functions destructure errors.Wrap into its component operations of annotating
+// an error with a stack trace and an a message, respectively.
+//
+// Retrieving the cause of an error
+//
+// Using errors.Wrap constructs a stack of errors, adding context to the
+// preceding error. Depending on the nature of the error it may be necessary
+// to reverse the operation of errors.Wrap to retrieve the original error
+// for inspection. Any error value which implements this interface
+//
+//     type causer interface {
+//             Cause() error
+//     }
+//
+// can be inspected by errors.Cause. errors.Cause will recursively retrieve
+// the topmost error which does not implement causer, which is assumed to be
+// the original cause. For example:
+//
+//     switch err := errors.Cause(err).(type) {
+//     case *MyError:
+//             // handle specifically
+//     default:
+//             // unknown error
+//     }
+//
+// causer interface is not exported by this package, but is considered a part
+// of stable public API.
+//
+// Formatted printing of errors
+//
+// All error values returned from this package implement fmt.Formatter and can
+// be formatted by the fmt package. The following verbs are supported
+//
+//     %s    print the error. If the error has a Cause it will be
+//           printed recursively
+//     %v    see %s
+//     %+v   extended format. Each Frame of the error's StackTrace will
+//           be printed in detail.
+//
+// Retrieving the stack trace of an error or wrapper
+//
+// New, Errorf, Wrap, and Wrapf record a stack trace at the point they are
+// invoked. This information can be retrieved with the following interface.
+//
+//     type stackTracer interface {
+//             StackTrace() errors.StackTrace
+//     }
+//
+// Where errors.StackTrace is defined as
+//
+//     type StackTrace []Frame
+//
+// The Frame type represents a call site in the stack trace. Frame supports
+// the fmt.Formatter interface that can be used for printing information about
+// the stack trace of this error. For example:
+//
+//     if err, ok := err.(stackTracer); ok {
+//             for _, f := range err.StackTrace() {
+//                     fmt.Printf("%+s:%d", f)
+//             }
+//     }
+//
+// stackTracer interface is not exported by this package, but is considered a part
+// of stable public API.
+//
+// See the documentation for Frame.Format for more details.
+package errors
+
+import (
+	"fmt"
+	"io"
+)
+
+// New returns an error with the supplied message.
+// New also records the stack trace at the point it was called.
+func New(message string) error {
+	return &fundamental{
+		msg:   message,
+		stack: callers(),
+	}
+}
+
+// Errorf formats according to a format specifier and returns the string
+// as a value that satisfies error.
+// Errorf also records the stack trace at the point it was called.
+func Errorf(format string, args ...interface{}) error {
+	return &fundamental{
+		msg:   fmt.Sprintf(format, args...),
+		stack: callers(),
+	}
+}
+
+// fundamental is an error that has a message and a stack, but no caller.
+type fundamental struct {
+	msg string
+	*stack
+}
+
+func (f *fundamental) Error() string { return f.msg }
+
+func (f *fundamental) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		if s.Flag('+') {
+			io.WriteString(s, f.msg)
+			f.stack.Format(s, verb)
+			return
+		}
+		fallthrough
+	case 's':
+		io.WriteString(s, f.msg)
+	case 'q':
+		fmt.Fprintf(s, "%q", f.msg)
+	}
+}
+
+// WithStack annotates err with a stack trace at the point WithStack was called.
+// If err is nil, WithStack returns nil.
+func WithStack(err error) error {
+	if err == nil {
+		return nil
+	}
+	return &withStack{
+		err,
+		callers(),
+	}
+}
+
+type withStack struct {
+	error
+	*stack
+}
+
+func (w *withStack) Cause() error { return w.error }
+
+func (w *withStack) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		if s.Flag('+') {
+			fmt.Fprintf(s, "%+v", w.Cause())
+			w.stack.Format(s, verb)
+			return
+		}
+		fallthrough
+	case 's':
+		io.WriteString(s, w.Error())
+	case 'q':
+		fmt.Fprintf(s, "%q", w.Error())
+	}
+}
+
+// Wrap returns an error annotating err with a stack trace
+// at the point Wrap is called, and the supplied message.
+// If err is nil, Wrap returns nil.
+func Wrap(err error, message string) error {
+	if err == nil {
+		return nil
+	}
+	err = &withMessage{
+		cause: err,
+		msg:   message,
+	}
+	return &withStack{
+		err,
+		callers(),
+	}
+}
+
+// Wrapf returns an error annotating err with a stack trace
+// at the point Wrapf is call, and the format specifier.
+// If err is nil, Wrapf returns nil.
+func Wrapf(err error, format string, args ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+	err = &withMessage{
+		cause: err,
+		msg:   fmt.Sprintf(format, args...),
+	}
+	return &withStack{
+		err,
+		callers(),
+	}
+}
+
+// WithMessage annotates err with a new message.
+// If err is nil, WithMessage returns nil.
+func WithMessage(err error, message string) error {
+	if err == nil {
+		return nil
+	}
+	return &withMessage{
+		cause: err,
+		msg:   message,
+	}
+}
+
+type withMessage struct {
+	cause error
+	msg   string
+}
+
+func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() }
+func (w *withMessage) Cause() error  { return w.cause }
+
+func (w *withMessage) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		if s.Flag('+') {
+			fmt.Fprintf(s, "%+v\n", w.Cause())
+			io.WriteString(s, w.msg)
+			return
+		}
+		fallthrough
+	case 's', 'q':
+		io.WriteString(s, w.Error())
+	}
+}
+
+// Cause returns the underlying cause of the error, if possible.
+// An error value has a cause if it implements the following
+// interface:
+//
+//     type causer interface {
+//            Cause() error
+//     }
+//
+// If the error does not implement Cause, the original error will
+// be returned. If the error is nil, nil will be returned without further
+// investigation.
+func Cause(err error) error {
+	type causer interface {
+		Cause() error
+	}
+
+	for err != nil {
+		cause, ok := err.(causer)
+		if !ok {
+			break
+		}
+		err = cause.Cause()
+	}
+	return err
+}
diff --git a/vendor/github.com/pkg/errors/stack.go b/vendor/github.com/pkg/errors/stack.go
new file mode 100644
index 00000000..2874a048
--- /dev/null
+++ b/vendor/github.com/pkg/errors/stack.go
@@ -0,0 +1,147 @@
+package errors
+
+import (
+	"fmt"
+	"io"
+	"path"
+	"runtime"
+	"strings"
+)
+
+// Frame represents a program counter inside a stack frame.
+type Frame uintptr
+
+// pc returns the program counter for this frame;
+// multiple frames may have the same PC value.
+func (f Frame) pc() uintptr { return uintptr(f) - 1 }
+
+// file returns the full path to the file that contains the
+// function for this Frame's pc.
+func (f Frame) file() string {
+	fn := runtime.FuncForPC(f.pc())
+	if fn == nil {
+		return "unknown"
+	}
+	file, _ := fn.FileLine(f.pc())
+	return file
+}
+
+// line returns the line number of source code of the
+// function for this Frame's pc.
+func (f Frame) line() int {
+	fn := runtime.FuncForPC(f.pc())
+	if fn == nil {
+		return 0
+	}
+	_, line := fn.FileLine(f.pc())
+	return line
+}
+
+// Format formats the frame according to the fmt.Formatter interface.
+//
+//    %s    source file
+//    %d    source line
+//    %n    function name
+//    %v    equivalent to %s:%d
+//
+// Format accepts flags that alter the printing of some verbs, as follows:
+//
+//    %+s   function name and path of source file relative to the compile time
+//          GOPATH separated by \n\t (<funcname>\n\t<path>)
+//    %+v   equivalent to %+s:%d
+func (f Frame) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 's':
+		switch {
+		case s.Flag('+'):
+			pc := f.pc()
+			fn := runtime.FuncForPC(pc)
+			if fn == nil {
+				io.WriteString(s, "unknown")
+			} else {
+				file, _ := fn.FileLine(pc)
+				fmt.Fprintf(s, "%s\n\t%s", fn.Name(), file)
+			}
+		default:
+			io.WriteString(s, path.Base(f.file()))
+		}
+	case 'd':
+		fmt.Fprintf(s, "%d", f.line())
+	case 'n':
+		name := runtime.FuncForPC(f.pc()).Name()
+		io.WriteString(s, funcname(name))
+	case 'v':
+		f.Format(s, 's')
+		io.WriteString(s, ":")
+		f.Format(s, 'd')
+	}
+}
+
+// StackTrace is stack of Frames from innermost (newest) to outermost (oldest).
+type StackTrace []Frame
+
+// Format formats the stack of Frames according to the fmt.Formatter interface.
+//
+//    %s	lists source files for each Frame in the stack
+//    %v	lists the source file and line number for each Frame in the stack
+//
+// Format accepts flags that alter the printing of some verbs, as follows:
+//
+//    %+v   Prints filename, function, and line number for each Frame in the stack.
+func (st StackTrace) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		switch {
+		case s.Flag('+'):
+			for _, f := range st {
+				fmt.Fprintf(s, "\n%+v", f)
+			}
+		case s.Flag('#'):
+			fmt.Fprintf(s, "%#v", []Frame(st))
+		default:
+			fmt.Fprintf(s, "%v", []Frame(st))
+		}
+	case 's':
+		fmt.Fprintf(s, "%s", []Frame(st))
+	}
+}
+
+// stack represents a stack of program counters.
+type stack []uintptr
+
+func (s *stack) Format(st fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		switch {
+		case st.Flag('+'):
+			for _, pc := range *s {
+				f := Frame(pc)
+				fmt.Fprintf(st, "\n%+v", f)
+			}
+		}
+	}
+}
+
+func (s *stack) StackTrace() StackTrace {
+	f := make([]Frame, len(*s))
+	for i := 0; i < len(f); i++ {
+		f[i] = Frame((*s)[i])
+	}
+	return f
+}
+
+func callers() *stack {
+	const depth = 32
+	var pcs [depth]uintptr
+	n := runtime.Callers(3, pcs[:])
+	var st stack = pcs[0:n]
+	return &st
+}
+
+// funcname removes the path prefix component of a function's name reported by func.Name().
+func funcname(name string) string {
+	i := strings.LastIndex(name, "/")
+	name = name[i+1:]
+	i = strings.Index(name, ".")
+	return name[i+1:]
+}
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 092a5f73..a08a35f0 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -18,6 +18,14 @@
 			"revision": "efacde03154693404c65e7aa7d461ac9014acd0c"
 		},
 		{
+			"checksumSHA1": "qqDRn7g1iElj298JH8gecN0mrdE=",
+			"path": "github.com/karrick/godirwalk",
+			"revision": "08cab4384b8f99517f23f3c829a1155465a548fc",
+			"revisionTime": "2018-03-02T10:10:21Z",
+			"version": "v1.5.0",
+			"versionExact": "v1.5.0"
+		},
+		{
 			"checksumSHA1": "sxHnA1plNXr5NVB2HYPGExA22U0=",
 			"path": "github.com/lupine/go-mimedb",
 			"revision": "e8af1d65987739a7ab277dedd9772c7398154ff3",
@@ -36,6 +44,12 @@
 			"revision": "67f268f20922975c067ed799e4be6bacf152208c"
 		},
 		{
+			"checksumSHA1": "ljd3FhYRJ91cLZz3wsH9BQQ2JbA=",
+			"path": "github.com/pkg/errors",
+			"revision": "30136e27e2ac8d167177e8a583aa4c3fea5be833",
+			"revisionTime": "2018-01-27T01:58:12Z"
+		},
+		{
 			"checksumSHA1": "Ph+qmEo8RdBKBHZUhx0y5Oyk/U0=",
 			"comment": "v0.8.0-9-g334af01",
 			"path": "github.com/prometheus/client_golang/prometheus",