Add latest changes from gitlab-org/gitlab@master

24 files changed, 238 insertions, 157 deletions

diff --git a/workhorse/internal/api/api.go b/workhorse/internal/api/api.go
index 4ddc66bf26e..17fea398029 100644
--- a/workhorse/internal/api/api.go
+++ b/workhorse/internal/api/api.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 
@@ -34,14 +35,14 @@ type API struct {
 }
 
 var (
-	requestsCounter = prometheus.NewCounterVec(
+	requestsCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_internal_api_requests",
 			Help: "How many internal API requests have been completed by gitlab-workhorse, partitioned by status code and HTTP method.",
 		},
 		[]string{"code", "method"},
 	)
-	bytesTotal = prometheus.NewCounter(
+	bytesTotal = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_internal_api_failure_response_bytes",
 			Help: "How many bytes have been returned by upstream GitLab in API failure/rejection response bodies.",
@@ -49,11 +50,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(requestsCounter)
-	prometheus.MustRegister(bytesTotal)
-}
-
 func NewAPI(myURL *url.URL, version string, roundTripper http.RoundTripper) *API {
 	return &API{
 		Client:  &http.Client{Transport: roundTripper},
diff --git a/workhorse/internal/artifacts/artifacts_upload.go b/workhorse/internal/artifacts/artifacts_upload.go
index d4ceb39d7c6..3d4b8bf0931 100644
--- a/workhorse/internal/artifacts/artifacts_upload.go
+++ b/workhorse/internal/artifacts/artifacts_upload.go
@@ -12,6 +12,7 @@ import (
 	"syscall"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
@@ -28,16 +29,12 @@ const (
 	ArtifactFormatDefault = ""
 )
 
-var zipSubcommandsErrorsCounter = prometheus.NewCounterVec(
+var zipSubcommandsErrorsCounter = promauto.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "gitlab_workhorse_zip_subcommand_errors_total",
 		Help: "Errors comming from subcommands used for processing ZIP archives",
 	}, []string{"error"})
 
-func init() {
-	prometheus.MustRegister(zipSubcommandsErrorsCounter)
-}
-
 type artifactsUploadProcessor struct {
 	opts   *filestore.SaveFileOpts
 	format string
diff --git a/workhorse/internal/builds/register.go b/workhorse/internal/builds/register.go
index 4a9d9096c5c..77685889cfd 100644
--- a/workhorse/internal/builds/register.go
+++ b/workhorse/internal/builds/register.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/redis"
@@ -20,14 +21,14 @@ const (
 )
 
 var (
-	registerHandlerRequests = prometheus.NewCounterVec(
+	registerHandlerRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_builds_register_handler_requests",
 			Help: "Describes how many requests in different states hit a register handler",
 		},
 		[]string{"status"},
 	)
-	registerHandlerOpen = prometheus.NewGaugeVec(
+	registerHandlerOpen = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_builds_register_handler_open",
 			Help: "Describes how many requests is currently open in given state",
@@ -53,13 +54,6 @@ type largeBodyError struct{ error }
 
 type WatchKeyHandler func(key, value string, timeout time.Duration) (redis.WatchKeyStatus, error)
 
-func init() {
-	prometheus.MustRegister(
-		registerHandlerRequests,
-		registerHandlerOpen,
-	)
-}
-
 type runnerRequest struct {
 	Token      string `json:"token,omitempty"`
 	LastUpdate string `json:"last_update,omitempty"`
diff --git a/workhorse/internal/config/config.go b/workhorse/internal/config/config.go
index 34e5e0cec18..84849c72744 100644
--- a/workhorse/internal/config/config.go
+++ b/workhorse/internal/config/config.go
@@ -105,6 +105,7 @@ type Config struct {
 	ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"`
 	PropagateCorrelationID   bool                     `toml:"-"`
 	ImageResizerConfig       ImageResizerConfig       `toml:"image_resizer"`
+	AltDocumentRoot          string                   `toml:"alt_document_root"`
 }
 
 var DefaultImageResizerConfig = ImageResizerConfig{
diff --git a/workhorse/internal/config/config_test.go b/workhorse/internal/config/config_test.go
index fcae52c7382..102b29a0813 100644
--- a/workhorse/internal/config/config_test.go
+++ b/workhorse/internal/config/config_test.go
@@ -21,6 +21,7 @@ func TestLoadEmptyConfig(t *testing.T) {
 	cfg, err := LoadConfig(config)
 	require.NoError(t, err)
 
+	require.Empty(t, cfg.AltDocumentRoot)
 	require.Equal(t, cfg.ImageResizerConfig.MaxFilesize, uint64(250000))
 	require.GreaterOrEqual(t, cfg.ImageResizerConfig.MaxScalerProcs, uint32(2))
 
@@ -97,3 +98,14 @@ max_filesize = 350000
 
 	require.Equal(t, expected, cfg.ImageResizerConfig)
 }
+
+func TestAltDocumentConfig(t *testing.T) {
+	config := `
+alt_document_root = "/path/to/documents"
+`
+
+	cfg, err := LoadConfig(config)
+	require.NoError(t, err)
+
+	require.Equal(t, "/path/to/documents", cfg.AltDocumentRoot)
+}
diff --git a/workhorse/internal/filestore/file_handler.go b/workhorse/internal/filestore/file_handler.go
index 935eb3b7f3b..19764e9a5cf 100644
--- a/workhorse/internal/filestore/file_handler.go
+++ b/workhorse/internal/filestore/file_handler.go
@@ -166,7 +166,13 @@ func SaveFileFromReader(ctx context.Context, reader io.Reader, size int64, opts
 			return nil, SizeError(fmt.Errorf("the upload size %d is over maximum of %d bytes", size, opts.MaximumSize))
 		}
 
-		reader = &hardLimitReader{r: reader, n: opts.MaximumSize}
+		hlr := &hardLimitReader{r: reader, n: opts.MaximumSize}
+		reader = hlr
+		defer func() {
+			if hlr.n < 0 {
+				err = ErrEntityTooLarge
+			}
+		}()
 	}
 
 	fh.Size, err = uploadDestination.Consume(ctx, reader, opts.Deadline)
diff --git a/workhorse/internal/filestore/file_handler_test.go b/workhorse/internal/filestore/file_handler_test.go
index 5a02075fb98..e79e9d0f292 100644
--- a/workhorse/internal/filestore/file_handler_test.go
+++ b/workhorse/internal/filestore/file_handler_test.go
@@ -2,6 +2,7 @@ package filestore_test
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -428,6 +429,107 @@ func TestSaveMultipartInBodyFailure(t *testing.T) {
 	require.EqualError(t, err, test.MultipartUploadInternalError().Error())
 }
 
+func TestSaveRemoteFileWithLimit(t *testing.T) {
+	testhelper.ConfigureSecret()
+
+	type remote int
+	const (
+		notRemote remote = iota
+		remoteSingle
+		remoteMultipart
+	)
+
+	remoteTypes := []remote{remoteSingle, remoteMultipart}
+
+	tests := []struct {
+		name        string
+		objectSize  int64
+		maxSize     int64
+		expectedErr error
+		testData    string
+	}{
+		{
+			name:       "known size with no limit",
+			testData:   test.ObjectContent,
+			objectSize: test.ObjectSize,
+		},
+		{
+			name:       "unknown size with no limit",
+			testData:   test.ObjectContent,
+			objectSize: -1,
+		},
+		{
+			name:        "unknown object size with limit",
+			testData:    test.ObjectContent,
+			objectSize:  -1,
+			maxSize:     test.ObjectSize - 1,
+			expectedErr: filestore.ErrEntityTooLarge,
+		},
+		{
+			name:        "large object with unknown size with limit",
+			testData:    string(make([]byte, 20000)),
+			objectSize:  -1,
+			maxSize:     19000,
+			expectedErr: filestore.ErrEntityTooLarge,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var opts filestore.SaveFileOpts
+
+			for _, remoteType := range remoteTypes {
+				tmpFolder, err := ioutil.TempDir("", "workhorse-test-tmp")
+				require.NoError(t, err)
+				defer os.RemoveAll(tmpFolder)
+
+				osStub, ts := test.StartObjectStore()
+				defer ts.Close()
+
+				switch remoteType {
+				case remoteSingle:
+					objectURL := ts.URL + test.ObjectPath
+
+					opts.RemoteID = "test-file"
+					opts.RemoteURL = objectURL
+					opts.PresignedPut = objectURL + "?Signature=ASignature"
+					opts.PresignedDelete = objectURL + "?Signature=AnotherSignature"
+					opts.Deadline = testDeadline()
+					opts.MaximumSize = tc.maxSize
+				case remoteMultipart:
+					objectURL := ts.URL + test.ObjectPath
+
+					opts.RemoteID = "test-file"
+					opts.RemoteURL = objectURL
+					opts.PresignedDelete = objectURL + "?Signature=AnotherSignature"
+					opts.PartSize = int64(len(tc.testData)/2) + 1
+					opts.PresignedParts = []string{objectURL + "?partNumber=1", objectURL + "?partNumber=2"}
+					opts.PresignedCompleteMultipart = objectURL + "?Signature=CompleteSignature"
+					opts.Deadline = testDeadline()
+					opts.MaximumSize = tc.maxSize
+
+					require.Less(t, int64(len(tc.testData)), int64(len(opts.PresignedParts))*opts.PartSize, "check part size calculation")
+
+					osStub.InitiateMultipartUpload(test.ObjectPath)
+				}
+
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
+
+				fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(tc.testData), tc.objectSize, &opts)
+
+				if tc.expectedErr == nil {
+					require.NoError(t, err)
+					require.NotNil(t, fh)
+				} else {
+					require.True(t, errors.Is(err, tc.expectedErr))
+					require.Nil(t, fh)
+				}
+			}
+		})
+	}
+}
+
 func checkFileHandlerWithFields(t *testing.T, fh *filestore.FileHandler, fields map[string]string, prefix string) {
 	key := func(field string) string {
 		if prefix == "" {
diff --git a/workhorse/internal/git/archive.go b/workhorse/internal/git/archive.go
index b35fb2e5382..b7575be2c02 100644
--- a/workhorse/internal/git/archive.go
+++ b/workhorse/internal/git/archive.go
@@ -18,6 +18,7 @@ import (
 	"github.com/golang/protobuf/proto" //lint:ignore SA1019 https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/274
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 
@@ -39,7 +40,7 @@ type archiveParams struct {
 
 var (
 	SendArchive     = &archive{"git-archive:"}
-	gitArchiveCache = prometheus.NewCounterVec(
+	gitArchiveCache = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_archive_cache",
 			Help: "Cache hits and misses for 'git archive' streaming",
@@ -48,10 +49,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(gitArchiveCache)
-}
-
 func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
 	var params archiveParams
 	if err := a.Unpack(&params, sendData); err != nil {
diff --git a/workhorse/internal/git/responsewriter.go b/workhorse/internal/git/responsewriter.go
index 12796e985c4..c4d4ac252d4 100644
--- a/workhorse/internal/git/responsewriter.go
+++ b/workhorse/internal/git/responsewriter.go
@@ -5,6 +5,7 @@ import (
 	"strconv"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 )
@@ -15,12 +16,12 @@ const (
 )
 
 var (
-	gitHTTPSessionsActive = prometheus.NewGauge(prometheus.GaugeOpts{
+	gitHTTPSessionsActive = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "gitlab_workhorse_git_http_sessions_active",
 		Help: "Number of Git HTTP request-response cycles currently being handled by gitlab-workhorse.",
 	})
 
-	gitHTTPRequests = prometheus.NewCounterVec(
+	gitHTTPRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_http_requests",
 			Help: "How many Git HTTP requests have been processed by gitlab-workhorse, partitioned by request type and agent.",
@@ -28,7 +29,7 @@ var (
 		[]string{"method", "code", "service", "agent"},
 	)
 
-	gitHTTPBytes = prometheus.NewCounterVec(
+	gitHTTPBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_http_bytes",
 			Help: "How many Git HTTP bytes have been sent by gitlab-workhorse, partitioned by request type, agent and direction.",
@@ -37,12 +38,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(gitHTTPSessionsActive)
-	prometheus.MustRegister(gitHTTPRequests)
-	prometheus.MustRegister(gitHTTPBytes)
-}
-
 type HttpResponseWriter struct {
 	helper.CountingResponseWriter
 }
diff --git a/workhorse/internal/gitaly/gitaly.go b/workhorse/internal/gitaly/gitaly.go
index ba9fbd1c919..c739ac8d9b2 100644
--- a/workhorse/internal/gitaly/gitaly.go
+++ b/workhorse/internal/gitaly/gitaly.go
@@ -10,6 +10,7 @@ import (
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	gitalyauth "gitlab.com/gitlab-org/gitaly/auth"
 	gitalyclient "gitlab.com/gitlab-org/gitaly/client"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
@@ -43,7 +44,7 @@ var (
 		connections: make(map[cacheKey]*grpc.ClientConn),
 	}
 
-	connectionsTotal = prometheus.NewCounterVec(
+	connectionsTotal = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_gitaly_connections_total",
 			Help: "Number of Gitaly connections that have been established",
@@ -52,10 +53,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(connectionsTotal)
-}
-
 func withOutgoingMetadata(ctx context.Context, features map[string]string) context.Context {
 	md := metadata.New(nil)
 	for k, v := range features {
diff --git a/workhorse/internal/imageresizer/image_resizer.go b/workhorse/internal/imageresizer/image_resizer.go
index feefd9c6dee..77318ed1c46 100644
--- a/workhorse/internal/imageresizer/image_resizer.go
+++ b/workhorse/internal/imageresizer/image_resizer.go
@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math"
 	"net"
 	"net/http"
 	"os"
@@ -17,6 +16,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/correlation"
 	"gitlab.com/gitlab-org/labkit/log"
@@ -96,7 +96,7 @@ const (
 )
 
 var (
-	imageResizeConcurrencyLimitExceeds = prometheus.NewCounter(
+	imageResizeConcurrencyLimitExceeds = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -104,7 +104,7 @@ var (
 			Help:      "Amount of image resizing requests that exceeded the maximum allowed scaler processes",
 		},
 	)
-	imageResizeProcesses = prometheus.NewGauge(
+	imageResizeProcesses = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -112,7 +112,7 @@ var (
 			Help:      "Amount of image scaler processes working now",
 		},
 	)
-	imageResizeMaxProcesses = prometheus.NewGauge(
+	imageResizeMaxProcesses = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -120,7 +120,7 @@ var (
 			Help:      "The maximum amount of image scaler processes allowed to run concurrently",
 		},
 	)
-	imageResizeRequests = prometheus.NewCounterVec(
+	imageResizeRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -129,7 +129,7 @@ var (
 		},
 		[]string{"status"},
 	)
-	imageResizeDurations = prometheus.NewHistogramVec(
+	imageResizeDurations = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -154,14 +154,6 @@ const (
 	maxMagicLen = 8                   // 8 first bytes is enough to detect PNG or JPEG
 )
 
-func init() {
-	prometheus.MustRegister(imageResizeConcurrencyLimitExceeds)
-	prometheus.MustRegister(imageResizeProcesses)
-	prometheus.MustRegister(imageResizeMaxProcesses)
-	prometheus.MustRegister(imageResizeRequests)
-	prometheus.MustRegister(imageResizeDurations)
-}
-
 func NewResizer(cfg config.Config) *Resizer {
 	imageResizeMaxProcesses.Set(float64(cfg.ImageResizerConfig.MaxScalerProcs))
 
@@ -279,6 +271,10 @@ func (r *Resizer) tryResizeImage(req *http.Request, f *imageFile, params *resize
 		return f.reader, nil, fmt.Errorf("%d bytes exceeds maximum file size of %d bytes", f.contentLength, cfg.MaxFilesize)
 	}
 
+	if f.contentLength < maxMagicLen {
+		return f.reader, nil, fmt.Errorf("file is too small to resize: %d bytes", f.contentLength)
+	}
+
 	if !r.numScalerProcs.tryIncrement(int32(cfg.MaxScalerProcs)) {
 		return f.reader, nil, fmt.Errorf("too many running scaler processes (%d / %d)", r.numScalerProcs.n, cfg.MaxScalerProcs)
 	}
@@ -289,11 +285,16 @@ func (r *Resizer) tryResizeImage(req *http.Request, f *imageFile, params *resize
 		r.numScalerProcs.decrement()
 	}()
 
-	// Prevents EOF if the file is smaller than 8 bytes
-	bufferSize := int(math.Min(maxMagicLen, float64(f.contentLength)))
-	buffered := bufio.NewReaderSize(f.reader, bufferSize)
+	// Creating buffered Reader is required for us to Peek into first bytes of the image file to detect the format
+	// without advancing the reader (we need to read from the file start in the Scaler binary).
+	// We set `8` as the minimal buffer size by the length of PNG magic bytes sequence (JPEG needs only 2).
+	// In fact, `NewReaderSize` will immediately override it with `16` using its `minReadBufferSize` -
+	// here we are just being explicit about the buffer size required for our code to operate correctly.
+	// Having a reader with such tiny buffer will not hurt the performance during further operations,
+	// because Golang `bufio.Read` avoids double copy: https://golang.org/src/bufio/bufio.go?s=1768:1804#L212
+	buffered := bufio.NewReaderSize(f.reader, maxMagicLen)
 
-	headerBytes, err := buffered.Peek(bufferSize)
+	headerBytes, err := buffered.Peek(maxMagicLen)
 	if err != nil {
 		return buffered, nil, fmt.Errorf("peek stream: %v", err)
 	}
diff --git a/workhorse/internal/imageresizer/image_resizer_test.go b/workhorse/internal/imageresizer/image_resizer_test.go
index 49cd88200aa..bacc97738b8 100644
--- a/workhorse/internal/imageresizer/image_resizer_test.go
+++ b/workhorse/internal/imageresizer/image_resizer_test.go
@@ -198,6 +198,29 @@ func TestServeOriginalImageWhenSourceImageFormatIsNotAllowed(t *testing.T) {
 	require.Equal(t, svgImage, responseData, "expected original image")
 }
 
+func TestServeOriginalImageWhenSourceImageIsTooSmall(t *testing.T) {
+	content := []byte("PNG") // 3 bytes only, invalid as PNG/JPEG image
+
+	img, err := ioutil.TempFile("", "*.png")
+	require.NoError(t, err)
+
+	defer img.Close()
+	defer os.Remove(img.Name())
+
+	_, err = img.Write(content)
+	require.NoError(t, err)
+
+	cfg := config.DefaultImageResizerConfig
+	params := resizeParams{Location: img.Name(), ContentType: "image/png", Width: 64}
+
+	resp := requestScaledImage(t, nil, params, cfg)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	responseData, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, content, responseData, "expected original image")
+}
+
 // The Rails applications sends a Base64 encoded JSON string carrying
 // these parameters in an HTTP response header
 func encodeParams(t *testing.T, p *resizeParams) string {
diff --git a/workhorse/internal/objectstore/prometheus.go b/workhorse/internal/objectstore/prometheus.go
index bfc2beddf34..20762fb52bc 100644
--- a/workhorse/internal/objectstore/prometheus.go
+++ b/workhorse/internal/objectstore/prometheus.go
@@ -1,28 +1,31 @@
 package objectstore
 
-import "github.com/prometheus/client_golang/prometheus"
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
 
 var (
-	objectStorageUploadRequests = prometheus.NewCounterVec(
+	objectStorageUploadRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_object_storage_upload_requests",
 			Help: "How many object storage requests have been processed",
 		},
 		[]string{"status"},
 	)
-	objectStorageUploadsOpen = prometheus.NewGauge(
+	objectStorageUploadsOpen = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_object_storage_upload_open",
 			Help: "Describes many object storage requests are open now",
 		},
 	)
-	objectStorageUploadBytes = prometheus.NewCounter(
+	objectStorageUploadBytes = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_object_storage_upload_bytes",
 			Help: "How many bytes were sent to object storage",
 		},
 	)
-	objectStorageUploadTime = prometheus.NewHistogram(
+	objectStorageUploadTime = promauto.NewHistogram(
 		prometheus.HistogramOpts{
 			Name:    "gitlab_workhorse_object_storage_upload_time",
 			Help:    "How long it took to upload objects",
@@ -34,10 +37,3 @@ var (
 
 	objectStorageUploadTimeBuckets = []float64{.1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100}
 )
-
-func init() {
-	prometheus.MustRegister(
-		objectStorageUploadRequests,
-		objectStorageUploadsOpen,
-		objectStorageUploadBytes)
-}
diff --git a/workhorse/internal/objectstore/uploader.go b/workhorse/internal/objectstore/uploader.go
index fb93cb83630..aedfbe55ead 100644
--- a/workhorse/internal/objectstore/uploader.go
+++ b/workhorse/internal/objectstore/uploader.go
@@ -90,6 +90,8 @@ func (u *uploader) Consume(outerCtx context.Context, reader io.Reader, deadline
 		}
 	}
 
+	objectStorageUploadBytes.Add(float64(cr.n))
+
 	return cr.n, nil
 }
 
diff --git a/workhorse/internal/queueing/queue.go b/workhorse/internal/queueing/queue.go
index b8d6954490c..db082cf19c6 100644
--- a/workhorse/internal/queueing/queue.go
+++ b/workhorse/internal/queueing/queue.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 type errTooManyRequests struct{ error }
@@ -44,7 +45,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 	}
 
 	metrics := &queueMetrics{
-		queueingLimit: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingLimit: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_limit",
 			Help: "Current limit set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -52,7 +53,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingQueueLimit: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingQueueLimit: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_queue_limit",
 			Help: "Current queueLimit set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -60,7 +61,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingQueueTimeout: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingQueueTimeout: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_queue_timeout",
 			Help: "Current queueTimeout set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -68,7 +69,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingBusy: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingBusy: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_busy",
 			Help: "How many queued requests are now processed",
 			ConstLabels: prometheus.Labels{
@@ -76,7 +77,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingWaiting: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingWaiting: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_waiting",
 			Help: "How many requests are now queued",
 			ConstLabels: prometheus.Labels{
@@ -84,7 +85,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingWaitingTime: prometheus.NewHistogram(prometheus.HistogramOpts{
+		queueingWaitingTime: promauto.NewHistogram(prometheus.HistogramOpts{
 			Name: "gitlab_workhorse_queueing_waiting_time",
 			Help: "How many time a request spent in queue",
 			ConstLabels: prometheus.Labels{
@@ -93,7 +94,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			Buckets: waitingTimeBuckets,
 		}),
 
-		queueingErrors: prometheus.NewCounterVec(
+		queueingErrors: promauto.NewCounterVec(
 			prometheus.CounterOpts{
 				Name: "gitlab_workhorse_queueing_errors",
 				Help: "How many times the TooManyRequests or QueueintTimedout errors were returned while queueing, partitioned by error type",
@@ -105,14 +106,6 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 		),
 	}
 
-	prometheus.MustRegister(metrics.queueingLimit)
-	prometheus.MustRegister(metrics.queueingQueueLimit)
-	prometheus.MustRegister(metrics.queueingQueueTimeout)
-	prometheus.MustRegister(metrics.queueingBusy)
-	prometheus.MustRegister(metrics.queueingWaiting)
-	prometheus.MustRegister(metrics.queueingWaitingTime)
-	prometheus.MustRegister(metrics.queueingErrors)
-
 	return metrics
 }
 
diff --git a/workhorse/internal/redis/keywatcher.go b/workhorse/internal/redis/keywatcher.go
index 9861956683a..96e33a64b85 100644
--- a/workhorse/internal/redis/keywatcher.go
+++ b/workhorse/internal/redis/keywatcher.go
@@ -9,6 +9,7 @@ import (
 	"github.com/gomodule/redigo/redis"
 	"github.com/jpillora/backoff"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
@@ -24,13 +25,13 @@ var (
 		Factor: 2,
 		Jitter: true,
 	}
-	keyWatchers = prometheus.NewGauge(
+	keyWatchers = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_keywatcher_keywatchers",
 			Help: "The number of keys that is being watched by gitlab-workhorse",
 		},
 	)
-	totalMessages = prometheus.NewCounter(
+	totalMessages = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_keywatcher_total_messages",
 			Help: "How many messages gitlab-workhorse has received in total on pubsub.",
@@ -38,13 +39,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(
-		keyWatchers,
-		totalMessages,
-	)
-}
-
 const (
 	keySubChannel = "workhorse:notifications"
 )
diff --git a/workhorse/internal/redis/redis.go b/workhorse/internal/redis/redis.go
index 9f96cef9d8f..0029a2a9e2b 100644
--- a/workhorse/internal/redis/redis.go
+++ b/workhorse/internal/redis/redis.go
@@ -10,6 +10,7 @@ import (
 	"github.com/FZambia/sentinel"
 	"github.com/gomodule/redigo/redis"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
@@ -45,14 +46,14 @@ const (
 )
 
 var (
-	totalConnections = prometheus.NewCounter(
+	totalConnections = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_redis_total_connections",
 			Help: "How many connections gitlab-workhorse has opened in total. Can be used to track Redis connection rate for this process",
 		},
 	)
 
-	errorCounter = prometheus.NewCounterVec(
+	errorCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_redis_errors",
 			Help: "Counts different types of Redis errors encountered by workhorse, by type and destination (redis, sentinel)",
@@ -61,13 +62,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(
-		totalConnections,
-		errorCounter,
-	)
-}
-
 func sentinelConn(master string, urls []config.TomlURL) *sentinel.Sentinel {
 	if len(urls) == 0 {
 		return nil
diff --git a/workhorse/internal/senddata/senddata.go b/workhorse/internal/senddata/senddata.go
index d654ae03fa4..c287d2574fa 100644
--- a/workhorse/internal/senddata/senddata.go
+++ b/workhorse/internal/senddata/senddata.go
@@ -8,17 +8,18 @@ import (
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata/contentprocessor"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
-	sendDataResponses = prometheus.NewCounterVec(
+	sendDataResponses = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_senddata_responses",
 			Help: "How many HTTP responses have been hijacked by a workhorse senddata injecter",
 		},
 		[]string{"injecter"},
 	)
-	sendDataResponseBytes = prometheus.NewCounterVec(
+	sendDataResponseBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_senddata_response_bytes",
 			Help: "How many bytes have been written by workhorse senddata response injecters",
@@ -27,11 +28,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(sendDataResponses)
-	prometheus.MustRegister(sendDataResponseBytes)
-}
-
 type sendDataResponseWriter struct {
 	rw        http.ResponseWriter
 	status    int
diff --git a/workhorse/internal/sendfile/sendfile.go b/workhorse/internal/sendfile/sendfile.go
index 7e5e863d5f1..d009f216eb9 100644
--- a/workhorse/internal/sendfile/sendfile.go
+++ b/workhorse/internal/sendfile/sendfile.go
@@ -14,6 +14,7 @@ import (
 	"regexp"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/log"
 	"gitlab.com/gitlab-org/labkit/mask"
@@ -23,7 +24,7 @@ import (
 )
 
 var (
-	sendFileRequests = prometheus.NewCounterVec(
+	sendFileRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_sendfile_requests",
 			Help: "How many X-Sendfile requests have been processed by gitlab-workhorse, partitioned by sendfile type.",
@@ -31,7 +32,7 @@ var (
 		[]string{"type"},
 	)
 
-	sendFileBytes = prometheus.NewCounterVec(
+	sendFileBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_sendfile_bytes",
 			Help: "How many X-Sendfile bytes have been sent by gitlab-workhorse, partitioned by sendfile type.",
@@ -49,11 +50,6 @@ type sendFileResponseWriter struct {
 	req      *http.Request
 }
 
-func init() {
-	prometheus.MustRegister(sendFileRequests)
-	prometheus.MustRegister(sendFileBytes)
-}
-
 func SendFile(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		s := &sendFileResponseWriter{
diff --git a/workhorse/internal/sendurl/sendurl.go b/workhorse/internal/sendurl/sendurl.go
index 316586188f0..cf3d14a2bf0 100644
--- a/workhorse/internal/sendurl/sendurl.go
+++ b/workhorse/internal/sendurl/sendurl.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/correlation"
 	"gitlab.com/gitlab-org/labkit/log"
@@ -68,20 +69,20 @@ var httpClient = &http.Client{
 }
 
 var (
-	sendURLRequests = prometheus.NewCounterVec(
+	sendURLRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_send_url_requests",
 			Help: "How many send URL requests have been processed",
 		},
 		[]string{"status"},
 	)
-	sendURLOpenRequests = prometheus.NewGauge(
+	sendURLOpenRequests = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_send_url_open_requests",
 			Help: "Describes how many send URL requests are open now",
 		},
 	)
-	sendURLBytes = prometheus.NewCounter(
+	sendURLBytes = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_send_url_bytes",
 			Help: "How many bytes were passed with send URL",
@@ -93,13 +94,6 @@ var (
 	sendURLRequestsSucceeded     = sendURLRequests.WithLabelValues("succeeded")
 )
 
-func init() {
-	prometheus.MustRegister(
-		sendURLRequests,
-		sendURLOpenRequests,
-		sendURLBytes)
-}
-
 func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
 	var params entryParams
 
diff --git a/workhorse/internal/staticpages/error_pages.go b/workhorse/internal/staticpages/error_pages.go
index 6ab97f396f8..3cc89d9f811 100644
--- a/workhorse/internal/staticpages/error_pages.go
+++ b/workhorse/internal/staticpages/error_pages.go
@@ -8,12 +8,13 @@ import (
 	"path/filepath"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 )
 
 var (
-	staticErrorResponses = prometheus.NewCounterVec(
+	staticErrorResponses = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_static_error_responses",
 			Help: "How many HTTP responses have been changed to a static error page, by HTTP status code.",
@@ -30,10 +31,6 @@ const (
 	ErrorFormatText
 )
 
-func init() {
-	prometheus.MustRegister(staticErrorResponses)
-}
-
 type errorPageResponseWriter struct {
 	rw       http.ResponseWriter
 	status   int
diff --git a/workhorse/internal/upload/rewrite.go b/workhorse/internal/upload/rewrite.go
index 019563e6bbd..e51604c6ed9 100644
--- a/workhorse/internal/upload/rewrite.go
+++ b/workhorse/internal/upload/rewrite.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
@@ -23,7 +24,7 @@ import (
 var ErrInjectedClientParam = errors.New("injected client parameter")
 
 var (
-	multipartUploadRequests = prometheus.NewCounterVec(
+	multipartUploadRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 
 			Name: "gitlab_workhorse_multipart_upload_requests",
@@ -32,7 +33,7 @@ var (
 		[]string{"type"},
 	)
 
-	multipartFileUploadBytes = prometheus.NewCounterVec(
+	multipartFileUploadBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_multipart_upload_bytes",
 			Help: "How many disk bytes of multipart file parts have been successfully written by gitlab-workhorse. Partitioned by type.",
@@ -40,7 +41,7 @@ var (
 		[]string{"type"},
 	)
 
-	multipartFiles = prometheus.NewCounterVec(
+	multipartFiles = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_multipart_upload_files",
 			Help: "How many multipart file parts have been processed by gitlab-workhorse. Partitioned by type.",
@@ -56,12 +57,6 @@ type rewriter struct {
 	finalizedFields map[string]bool
 }
 
-func init() {
-	prometheus.MustRegister(multipartUploadRequests)
-	prometheus.MustRegister(multipartFileUploadBytes)
-	prometheus.MustRegister(multipartFiles)
-}
-
 func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, preauth *api.Response, filter MultipartFormProcessor, opts *filestore.SaveFileOpts) error {
 	// Create multipart reader
 	reader, err := r.MultipartReader()
diff --git a/workhorse/internal/upstream/metrics.go b/workhorse/internal/upstream/metrics.go
index f5bc276f403..38528056d43 100644
--- a/workhorse/internal/upstream/metrics.go
+++ b/workhorse/internal/upstream/metrics.go
@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -40,14 +41,14 @@ func byteSizeBuckets() []float64 {
 }
 
 var (
-	httpInFlightRequests = prometheus.NewGauge(prometheus.GaugeOpts{
+	httpInFlightRequests = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Subsystem: httpSubsystem,
 		Name:      "in_flight_requests",
 		Help:      "A gauge of requests currently being served by workhorse.",
 	})
 
-	httpRequestsTotal = prometheus.NewCounterVec(
+	httpRequestsTotal = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -57,7 +58,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpRequestDurationSeconds = prometheus.NewHistogramVec(
+	httpRequestDurationSeconds = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -68,7 +69,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpRequestSizeBytes = prometheus.NewHistogramVec(
+	httpRequestSizeBytes = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -79,7 +80,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpResponseSizeBytes = prometheus.NewHistogramVec(
+	httpResponseSizeBytes = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -90,7 +91,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpTimeToWriteHeaderSeconds = prometheus.NewHistogramVec(
+	httpTimeToWriteHeaderSeconds = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -102,14 +103,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(httpInFlightRequests)
-	prometheus.MustRegister(httpRequestsTotal)
-	prometheus.MustRegister(httpRequestDurationSeconds)
-	prometheus.MustRegister(httpRequestSizeBytes)
-	prometheus.MustRegister(httpTimeToWriteHeaderSeconds)
-}
-
 func instrumentRoute(next http.Handler, method string, regexpStr string) http.Handler {
 	handler := next
 
diff --git a/workhorse/internal/upstream/routes.go b/workhorse/internal/upstream/routes.go
index 40a8a52f59b..5bbd245719b 100644
--- a/workhorse/internal/upstream/routes.go
+++ b/workhorse/internal/upstream/routes.go
@@ -192,6 +192,16 @@ func (u *upstream) configureRoutes() {
 	proxy := buildProxy(u.Backend, u.Version, u.RoundTripper, u.Config)
 	cableProxy := proxypkg.NewProxy(u.CableBackend, u.Version, u.CableRoundTripper)
 
+	assetsNotFoundHandler := NotFoundUnless(u.DevelopmentMode, proxy)
+	if u.AltDocumentRoot != "" {
+		altStatic := &staticpages.Static{DocumentRoot: u.AltDocumentRoot}
+		assetsNotFoundHandler = altStatic.ServeExisting(
+			u.URLPrefix,
+			staticpages.CacheExpireMax,
+			NotFoundUnless(u.DevelopmentMode, proxy),
+		)
+	}
+
 	signingTripper := secret.NewRoundTripper(u.RoundTripper, u.Version)
 	signingProxy := buildProxy(u.Backend, u.Version, signingTripper, u.Config)
 
@@ -283,7 +293,7 @@ func (u *upstream) configureRoutes() {
 			static.ServeExisting(
 				u.URLPrefix,
 				staticpages.CacheExpireMax,
-				NotFoundUnless(u.DevelopmentMode, proxy),
+				assetsNotFoundHandler,
 			),
 			withoutTracing(), // Tracing on assets is very noisy
 		),