2 files changed, 364 insertions, 0 deletions
diff --git a/workhorse/internal/sendurl/sendurl.go b/workhorse/internal/sendurl/sendurl.go
new file mode 100644
index 00000000000..cf3d14a2bf0
--- /dev/null
+++ b/workhorse/internal/sendurl/sendurl.go
@@ -0,0 +1,167 @@
+package sendurl
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"gitlab.com/gitlab-org/labkit/correlation"
+	"gitlab.com/gitlab-org/labkit/log"
+	"gitlab.com/gitlab-org/labkit/mask"
+	"gitlab.com/gitlab-org/labkit/tracing"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+)
+
+type entry struct{ senddata.Prefix }
+
+type entryParams struct {
+	URL            string
+	AllowRedirects bool
+}
+
+var SendURL = &entry{"send-url:"}
+
+var rangeHeaderKeys = []string{
+	"If-Match",
+	"If-Unmodified-Since",
+	"If-None-Match",
+	"If-Modified-Since",
+	"If-Range",
+	"Range",
+}
+
+// Keep cache headers from the original response, not the proxied response. The
+// original response comes from the Rails application, which should be the
+// source of truth for caching.
+var preserveHeaderKeys = map[string]bool{
+	"Cache-Control": true,
+	"Expires":       true,
+	"Date":          true, // Support for HTTP 1.0 proxies
+	"Pragma":        true, // Support for HTTP 1.0 proxies
+}
+
+// httpTransport defines a http.Transport with values
+// that are more restrictive than for http.DefaultTransport,
+// they define shorter TLS Handshake, and more aggressive connection closing
+// to prevent the connection hanging and reduce FD usage
+var httpTransport = tracing.NewRoundTripper(correlation.NewInstrumentedRoundTripper(&http.Transport{
+	Proxy: http.ProxyFromEnvironment,
+	DialContext: (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 10 * time.Second,
+	}).DialContext,
+	MaxIdleConns:          2,
+	IdleConnTimeout:       30 * time.Second,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 10 * time.Second,
+	ResponseHeaderTimeout: 30 * time.Second,
+}))
+
+var httpClient = &http.Client{
+	Transport: httpTransport,
+}
+
+var (
+	sendURLRequests = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_send_url_requests",
+			Help: "How many send URL requests have been processed",
+		},
+		[]string{"status"},
+	)
+	sendURLOpenRequests = promauto.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "gitlab_workhorse_send_url_open_requests",
+			Help: "Describes how many send URL requests are open now",
+		},
+	)
+	sendURLBytes = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_send_url_bytes",
+			Help: "How many bytes were passed with send URL",
+		},
+	)
+
+	sendURLRequestsInvalidData   = sendURLRequests.WithLabelValues("invalid-data")
+	sendURLRequestsRequestFailed = sendURLRequests.WithLabelValues("request-failed")
+	sendURLRequestsSucceeded     = sendURLRequests.WithLabelValues("succeeded")
+)
+
+func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+	var params entryParams
+
+	sendURLOpenRequests.Inc()
+	defer sendURLOpenRequests.Dec()
+
+	if err := e.Unpack(&params, sendData); err != nil {
+		helper.Fail500(w, r, fmt.Errorf("SendURL: unpack sendData: %v", err))
+		return
+	}
+
+	log.WithContextFields(r.Context(), log.Fields{
+		"url":  mask.URL(params.URL),
+		"path": r.URL.Path,
+	}).Info("SendURL: sending")
+
+	if params.URL == "" {
+		sendURLRequestsInvalidData.Inc()
+		helper.Fail500(w, r, fmt.Errorf("SendURL: URL is empty"))
+		return
+	}
+
+	// create new request and copy range headers
+	newReq, err := http.NewRequest("GET", params.URL, nil)
+	if err != nil {
+		sendURLRequestsInvalidData.Inc()
+		helper.Fail500(w, r, fmt.Errorf("SendURL: NewRequest: %v", err))
+		return
+	}
+	newReq = newReq.WithContext(r.Context())
+
+	for _, header := range rangeHeaderKeys {
+		newReq.Header[header] = r.Header[header]
+	}
+
+	// execute new request
+	var resp *http.Response
+	if params.AllowRedirects {
+		resp, err = httpClient.Do(newReq)
+	} else {
+		resp, err = httpTransport.RoundTrip(newReq)
+	}
+	if err != nil {
+		sendURLRequestsRequestFailed.Inc()
+		helper.Fail500(w, r, fmt.Errorf("SendURL: Do request: %v", err))
+		return
+	}
+
+	// Prevent Go from adding a Content-Length header automatically
+	w.Header().Del("Content-Length")
+
+	// copy response headers and body, except the headers from preserveHeaderKeys
+	for key, value := range resp.Header {
+		if !preserveHeaderKeys[key] {
+			w.Header()[key] = value
+		}
+	}
+	w.WriteHeader(resp.StatusCode)
+
+	defer resp.Body.Close()
+	n, err := io.Copy(w, resp.Body)
+	sendURLBytes.Add(float64(n))
+
+	if err != nil {
+		sendURLRequestsRequestFailed.Inc()
+		helper.LogError(r, fmt.Errorf("SendURL: Copy response: %v", err))
+		return
+	}
+
+	sendURLRequestsSucceeded.Inc()
+}
diff --git a/workhorse/internal/sendurl/sendurl_test.go b/workhorse/internal/sendurl/sendurl_test.go
new file mode 100644
index 00000000000..41e1dbb8e0f
--- /dev/null
+++ b/workhorse/internal/sendurl/sendurl_test.go
@@ -0,0 +1,197 @@
+package sendurl
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+)
+
+const testData = `123456789012345678901234567890`
+const testDataEtag = `W/"myetag"`
+
+func testEntryServer(t *testing.T, requestURL string, httpHeaders http.Header, allowRedirects bool) *httptest.ResponseRecorder {
+	requestHandler := func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+
+		url := r.URL.String() + "/file"
+		jsonParams := fmt.Sprintf(`{"URL":%q,"AllowRedirects":%s}`,
+			url, strconv.FormatBool(allowRedirects))
+		data := base64.URLEncoding.EncodeToString([]byte(jsonParams))
+
+		// The server returns a Content-Disposition
+		w.Header().Set("Content-Disposition", "attachment; filename=\"archive.txt\"")
+		w.Header().Set("Cache-Control", "no-cache")
+		w.Header().Set("Expires", "")
+		w.Header().Set("Date", "Wed, 21 Oct 2015 05:28:00 GMT")
+		w.Header().Set("Pragma", "no-cache")
+
+		SendURL.Inject(w, r, data)
+	}
+	serveFile := func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+
+		tempFile, err := ioutil.TempFile("", "download_file")
+		require.NoError(t, err)
+		require.NoError(t, os.Remove(tempFile.Name()))
+		defer tempFile.Close()
+		_, err = tempFile.Write([]byte(testData))
+		require.NoError(t, err)
+
+		w.Header().Set("Etag", testDataEtag)
+		w.Header().Set("Cache-Control", "public")
+		w.Header().Set("Expires", "Wed, 21 Oct 2015 07:28:00 GMT")
+		w.Header().Set("Date", "Wed, 21 Oct 2015 06:28:00 GMT")
+		w.Header().Set("Pragma", "")
+
+		http.ServeContent(w, r, "archive.txt", time.Now(), tempFile)
+	}
+	redirectFile := func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+		http.Redirect(w, r, r.URL.String()+"/download", http.StatusTemporaryRedirect)
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/get/request", requestHandler)
+	mux.HandleFunc("/get/request/file", serveFile)
+	mux.HandleFunc("/get/redirect", requestHandler)
+	mux.HandleFunc("/get/redirect/file", redirectFile)
+	mux.HandleFunc("/get/redirect/file/download", serveFile)
+	mux.HandleFunc("/get/file-not-existing", requestHandler)
+
+	server := httptest.NewServer(mux)
+	defer server.Close()
+
+	httpRequest, err := http.NewRequest("GET", server.URL+requestURL, nil)
+	require.NoError(t, err)
+	if httpHeaders != nil {
+		httpRequest.Header = httpHeaders
+	}
+
+	response := httptest.NewRecorder()
+	mux.ServeHTTP(response, httpRequest)
+	return response
+}
+
+func TestDownloadingUsingSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/request", nil, false)
+	require.Equal(t, http.StatusOK, response.Code)
+
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"archive.txt\"")
+
+	testhelper.RequireResponseBody(t, response, testData)
+}
+
+func TestDownloadingAChunkOfDataWithSendURL(t *testing.T) {
+	httpHeaders := http.Header{
+		"Range": []string{
+			"bytes=1-2",
+		},
+	}
+
+	response := testEntryServer(t, "/get/request", httpHeaders, false)
+	require.Equal(t, http.StatusPartialContent, response.Code)
+
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"archive.txt\"")
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Range",
+		"bytes 1-2/30")
+
+	testhelper.RequireResponseBody(t, response, "23")
+}
+
+func TestAccessingAlreadyDownloadedFileWithSendURL(t *testing.T) {
+	httpHeaders := http.Header{
+		"If-None-Match": []string{testDataEtag},
+	}
+
+	response := testEntryServer(t, "/get/request", httpHeaders, false)
+	require.Equal(t, http.StatusNotModified, response.Code)
+}
+
+func TestAccessingRedirectWithSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/redirect", nil, false)
+	require.Equal(t, http.StatusTemporaryRedirect, response.Code)
+}
+
+func TestAccessingAllowedRedirectWithSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/redirect", nil, true)
+	require.Equal(t, http.StatusOK, response.Code)
+
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"archive.txt\"")
+}
+
+func TestAccessingAllowedRedirectWithChunkOfDataWithSendURL(t *testing.T) {
+	httpHeaders := http.Header{
+		"Range": []string{
+			"bytes=1-2",
+		},
+	}
+
+	response := testEntryServer(t, "/get/redirect", httpHeaders, true)
+	require.Equal(t, http.StatusPartialContent, response.Code)
+
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"archive.txt\"")
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Range",
+		"bytes 1-2/30")
+
+	testhelper.RequireResponseBody(t, response, "23")
+}
+
+func TestOriginalCacheHeadersPreservedWithSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/redirect", nil, true)
+	require.Equal(t, http.StatusOK, response.Code)
+
+	testhelper.RequireResponseHeader(t, response,
+		"Cache-Control",
+		"no-cache")
+	testhelper.RequireResponseHeader(t, response,
+		"Expires",
+		"")
+	testhelper.RequireResponseHeader(t, response,
+		"Date",
+		"Wed, 21 Oct 2015 05:28:00 GMT")
+	testhelper.RequireResponseHeader(t, response,
+		"Pragma",
+		"no-cache")
+}
+
+func TestDownloadingNonExistingFileUsingSendURL(t *testing.T) {
+	response := testEntryServer(t, "/invalid/path", nil, false)
+	require.Equal(t, http.StatusNotFound, response.Code)
+}
+
+func TestDownloadingNonExistingRemoteFileWithSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/file-not-existing", nil, false)
+	require.Equal(t, http.StatusNotFound, response.Code)
+}