diff options
Diffstat (limited to 'workhorse/internal/sendurl')
-rw-r--r-- | workhorse/internal/sendurl/sendurl.go | 167 | ||||
-rw-r--r-- | workhorse/internal/sendurl/sendurl_test.go | 197 |
2 files changed, 364 insertions, 0 deletions
diff --git a/workhorse/internal/sendurl/sendurl.go b/workhorse/internal/sendurl/sendurl.go new file mode 100644 index 00000000000..cf3d14a2bf0 --- /dev/null +++ b/workhorse/internal/sendurl/sendurl.go @@ -0,0 +1,167 @@ +package sendurl + +import ( + "fmt" + "io" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "gitlab.com/gitlab-org/labkit/correlation" + "gitlab.com/gitlab-org/labkit/log" + "gitlab.com/gitlab-org/labkit/mask" + "gitlab.com/gitlab-org/labkit/tracing" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata" +) + +type entry struct{ senddata.Prefix } + +type entryParams struct { + URL string + AllowRedirects bool +} + +var SendURL = &entry{"send-url:"} + +var rangeHeaderKeys = []string{ + "If-Match", + "If-Unmodified-Since", + "If-None-Match", + "If-Modified-Since", + "If-Range", + "Range", +} + +// Keep cache headers from the original response, not the proxied response. The +// original response comes from the Rails application, which should be the +// source of truth for caching. +var preserveHeaderKeys = map[string]bool{ + "Cache-Control": true, + "Expires": true, + "Date": true, // Support for HTTP 1.0 proxies + "Pragma": true, // Support for HTTP 1.0 proxies +} + +// httpTransport defines a http.Transport with values +// that are more restrictive than for http.DefaultTransport, +// they define shorter TLS Handshake, and more aggressive connection closing +// to prevent the connection hanging and reduce FD usage +var httpTransport = tracing.NewRoundTripper(correlation.NewInstrumentedRoundTripper(&http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 10 * time.Second, + }).DialContext, + MaxIdleConns: 2, + IdleConnTimeout: 30 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 10 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, +})) + +var httpClient = &http.Client{ + Transport: httpTransport, +} + +var ( + sendURLRequests = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitlab_workhorse_send_url_requests", + Help: "How many send URL requests have been processed", + }, + []string{"status"}, + ) + sendURLOpenRequests = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "gitlab_workhorse_send_url_open_requests", + Help: "Describes how many send URL requests are open now", + }, + ) + sendURLBytes = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "gitlab_workhorse_send_url_bytes", + Help: "How many bytes were passed with send URL", + }, + ) + + sendURLRequestsInvalidData = sendURLRequests.WithLabelValues("invalid-data") + sendURLRequestsRequestFailed = sendURLRequests.WithLabelValues("request-failed") + sendURLRequestsSucceeded = sendURLRequests.WithLabelValues("succeeded") +) + +func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) { + var params entryParams + + sendURLOpenRequests.Inc() + defer sendURLOpenRequests.Dec() + + if err := e.Unpack(¶ms, sendData); err != nil { + helper.Fail500(w, r, fmt.Errorf("SendURL: unpack sendData: %v", err)) + return + } + + log.WithContextFields(r.Context(), log.Fields{ + "url": mask.URL(params.URL), + "path": r.URL.Path, + }).Info("SendURL: sending") + + if params.URL == "" { + sendURLRequestsInvalidData.Inc() + helper.Fail500(w, r, fmt.Errorf("SendURL: URL is empty")) + return + } + + // create new request and copy range headers + newReq, err := http.NewRequest("GET", params.URL, nil) + if err != nil { + sendURLRequestsInvalidData.Inc() + helper.Fail500(w, r, fmt.Errorf("SendURL: NewRequest: %v", err)) + return + } + newReq = newReq.WithContext(r.Context()) + + for _, header := range rangeHeaderKeys { + newReq.Header[header] = r.Header[header] + } + + // execute new request + var resp *http.Response + if params.AllowRedirects { + resp, err = httpClient.Do(newReq) + } else { + resp, err = httpTransport.RoundTrip(newReq) + } + if err != nil { + sendURLRequestsRequestFailed.Inc() + helper.Fail500(w, r, fmt.Errorf("SendURL: Do request: %v", err)) + return + } + + // Prevent Go from adding a Content-Length header automatically + w.Header().Del("Content-Length") + + // copy response headers and body, except the headers from preserveHeaderKeys + for key, value := range resp.Header { + if !preserveHeaderKeys[key] { + w.Header()[key] = value + } + } + w.WriteHeader(resp.StatusCode) + + defer resp.Body.Close() + n, err := io.Copy(w, resp.Body) + sendURLBytes.Add(float64(n)) + + if err != nil { + sendURLRequestsRequestFailed.Inc() + helper.LogError(r, fmt.Errorf("SendURL: Copy response: %v", err)) + return + } + + sendURLRequestsSucceeded.Inc() +} diff --git a/workhorse/internal/sendurl/sendurl_test.go b/workhorse/internal/sendurl/sendurl_test.go new file mode 100644 index 00000000000..41e1dbb8e0f --- /dev/null +++ b/workhorse/internal/sendurl/sendurl_test.go @@ -0,0 +1,197 @@ +package sendurl + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper" +) + +const testData = `123456789012345678901234567890` +const testDataEtag = `W/"myetag"` + +func testEntryServer(t *testing.T, requestURL string, httpHeaders http.Header, allowRedirects bool) *httptest.ResponseRecorder { + requestHandler := func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "GET", r.Method) + + url := r.URL.String() + "/file" + jsonParams := fmt.Sprintf(`{"URL":%q,"AllowRedirects":%s}`, + url, strconv.FormatBool(allowRedirects)) + data := base64.URLEncoding.EncodeToString([]byte(jsonParams)) + + // The server returns a Content-Disposition + w.Header().Set("Content-Disposition", "attachment; filename=\"archive.txt\"") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Expires", "") + w.Header().Set("Date", "Wed, 21 Oct 2015 05:28:00 GMT") + w.Header().Set("Pragma", "no-cache") + + SendURL.Inject(w, r, data) + } + serveFile := func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "GET", r.Method) + + tempFile, err := ioutil.TempFile("", "download_file") + require.NoError(t, err) + require.NoError(t, os.Remove(tempFile.Name())) + defer tempFile.Close() + _, err = tempFile.Write([]byte(testData)) + require.NoError(t, err) + + w.Header().Set("Etag", testDataEtag) + w.Header().Set("Cache-Control", "public") + w.Header().Set("Expires", "Wed, 21 Oct 2015 07:28:00 GMT") + w.Header().Set("Date", "Wed, 21 Oct 2015 06:28:00 GMT") + w.Header().Set("Pragma", "") + + http.ServeContent(w, r, "archive.txt", time.Now(), tempFile) + } + redirectFile := func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "GET", r.Method) + http.Redirect(w, r, r.URL.String()+"/download", http.StatusTemporaryRedirect) + } + + mux := http.NewServeMux() + mux.HandleFunc("/get/request", requestHandler) + mux.HandleFunc("/get/request/file", serveFile) + mux.HandleFunc("/get/redirect", requestHandler) + mux.HandleFunc("/get/redirect/file", redirectFile) + mux.HandleFunc("/get/redirect/file/download", serveFile) + mux.HandleFunc("/get/file-not-existing", requestHandler) + + server := httptest.NewServer(mux) + defer server.Close() + + httpRequest, err := http.NewRequest("GET", server.URL+requestURL, nil) + require.NoError(t, err) + if httpHeaders != nil { + httpRequest.Header = httpHeaders + } + + response := httptest.NewRecorder() + mux.ServeHTTP(response, httpRequest) + return response +} + +func TestDownloadingUsingSendURL(t *testing.T) { + response := testEntryServer(t, "/get/request", nil, false) + require.Equal(t, http.StatusOK, response.Code) + + testhelper.RequireResponseHeader(t, response, + "Content-Type", + "text/plain; charset=utf-8") + testhelper.RequireResponseHeader(t, response, + "Content-Disposition", + "attachment; filename=\"archive.txt\"") + + testhelper.RequireResponseBody(t, response, testData) +} + +func TestDownloadingAChunkOfDataWithSendURL(t *testing.T) { + httpHeaders := http.Header{ + "Range": []string{ + "bytes=1-2", + }, + } + + response := testEntryServer(t, "/get/request", httpHeaders, false) + require.Equal(t, http.StatusPartialContent, response.Code) + + testhelper.RequireResponseHeader(t, response, + "Content-Type", + "text/plain; charset=utf-8") + testhelper.RequireResponseHeader(t, response, + "Content-Disposition", + "attachment; filename=\"archive.txt\"") + testhelper.RequireResponseHeader(t, response, + "Content-Range", + "bytes 1-2/30") + + testhelper.RequireResponseBody(t, response, "23") +} + +func TestAccessingAlreadyDownloadedFileWithSendURL(t *testing.T) { + httpHeaders := http.Header{ + "If-None-Match": []string{testDataEtag}, + } + + response := testEntryServer(t, "/get/request", httpHeaders, false) + require.Equal(t, http.StatusNotModified, response.Code) +} + +func TestAccessingRedirectWithSendURL(t *testing.T) { + response := testEntryServer(t, "/get/redirect", nil, false) + require.Equal(t, http.StatusTemporaryRedirect, response.Code) +} + +func TestAccessingAllowedRedirectWithSendURL(t *testing.T) { + response := testEntryServer(t, "/get/redirect", nil, true) + require.Equal(t, http.StatusOK, response.Code) + + testhelper.RequireResponseHeader(t, response, + "Content-Type", + "text/plain; charset=utf-8") + testhelper.RequireResponseHeader(t, response, + "Content-Disposition", + "attachment; filename=\"archive.txt\"") +} + +func TestAccessingAllowedRedirectWithChunkOfDataWithSendURL(t *testing.T) { + httpHeaders := http.Header{ + "Range": []string{ + "bytes=1-2", + }, + } + + response := testEntryServer(t, "/get/redirect", httpHeaders, true) + require.Equal(t, http.StatusPartialContent, response.Code) + + testhelper.RequireResponseHeader(t, response, + "Content-Type", + "text/plain; charset=utf-8") + testhelper.RequireResponseHeader(t, response, + "Content-Disposition", + "attachment; filename=\"archive.txt\"") + testhelper.RequireResponseHeader(t, response, + "Content-Range", + "bytes 1-2/30") + + testhelper.RequireResponseBody(t, response, "23") +} + +func TestOriginalCacheHeadersPreservedWithSendURL(t *testing.T) { + response := testEntryServer(t, "/get/redirect", nil, true) + require.Equal(t, http.StatusOK, response.Code) + + testhelper.RequireResponseHeader(t, response, + "Cache-Control", + "no-cache") + testhelper.RequireResponseHeader(t, response, + "Expires", + "") + testhelper.RequireResponseHeader(t, response, + "Date", + "Wed, 21 Oct 2015 05:28:00 GMT") + testhelper.RequireResponseHeader(t, response, + "Pragma", + "no-cache") +} + +func TestDownloadingNonExistingFileUsingSendURL(t *testing.T) { + response := testEntryServer(t, "/invalid/path", nil, false) + require.Equal(t, http.StatusNotFound, response.Code) +} + +func TestDownloadingNonExistingRemoteFileWithSendURL(t *testing.T) { + response := testEntryServer(t, "/get/file-not-existing", nil, false) + require.Equal(t, http.StatusNotFound, response.Code) +} |