diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
tree | dceebdc68925362117480a5d672bcff122fb625b /workhorse/internal | |
parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'workhorse/internal')
21 files changed, 142 insertions, 31 deletions
diff --git a/workhorse/internal/headers/content_headers.go b/workhorse/internal/headers/content_headers.go index 54c7c1bdd95..3dd8216eec5 100644 --- a/workhorse/internal/headers/content_headers.go +++ b/workhorse/internal/headers/content_headers.go @@ -53,13 +53,15 @@ func SafeContentHeaders(data []byte, contentDisposition string) (string, string) // Some browsers will render XML inline unless a filename directive is provided with a non-xml file extension // This overrides the filename directive in the case of XML data - for _, element := range htmlRenderingTypes { - if isType(detectedContentType, element) { - disposition, directives, err := mime.ParseMediaType(contentDisposition) - if err == nil { - directives["filename"] = dummyFilename - contentDisposition = mime.FormatMediaType(disposition, directives) - break + if !attachmentRegex.MatchString(contentDisposition) { + for _, element := range htmlRenderingTypes { + if isType(detectedContentType, element) { + disposition, directives, err := mime.ParseMediaType(contentDisposition) + if err == nil { + directives["filename"] = dummyFilename + contentDisposition = mime.FormatMediaType(disposition, directives) + break + } } } } diff --git a/workhorse/internal/headers/content_headers_test.go b/workhorse/internal/headers/content_headers_test.go index 7cfce335d88..bd329dbc199 100644 --- a/workhorse/internal/headers/content_headers_test.go +++ b/workhorse/internal/headers/content_headers_test.go @@ -13,33 +13,63 @@ func fileContents(fileName string) []byte { } func TestHeaders(t *testing.T) { + xmlFileContents := fileContents("../../testdata/test.xml") + svgFileContents := fileContents("../../testdata/xml.svg") + xhtmlFileContents := fileContents("../../testdata/index.xhtml") + tests := []struct { desc string fileContents []byte + contentDisposition string expectedContentType string expectedContentDisposition string }{ { - desc: "XML file", - fileContents: fileContents("../../testdata/test.xml"), + desc: "inline XML file", + fileContents: xmlFileContents, + contentDisposition: "inline; filename=test.xml", expectedContentType: "text/plain; charset=utf-8", expectedContentDisposition: "inline; filename=blob", }, { - desc: "XHTML file", - fileContents: fileContents("../../testdata/index.xhtml"), + desc: "attachment XML file", + fileContents: xmlFileContents, + contentDisposition: "attachment; filename=test.xml", + expectedContentType: "application/octet-stream", + expectedContentDisposition: "attachment; filename=test.xml", + }, + { + desc: "inline XHTML file", + fileContents: xhtmlFileContents, + contentDisposition: "inline; filename=index.xhtml", expectedContentType: "text/plain; charset=utf-8", expectedContentDisposition: "inline; filename=blob", }, { + desc: "attachment XHTML file", + fileContents: xhtmlFileContents, + contentDisposition: "attachment; filename=index.xhtml", + expectedContentType: "application/octet-stream", + expectedContentDisposition: "attachment; filename=index.xhtml", + }, + { desc: "svg+xml file", - fileContents: fileContents("../../testdata/xml.svg"), + fileContents: svgFileContents, + contentDisposition: "", expectedContentType: "image/svg+xml", expectedContentDisposition: "attachment", }, { + desc: "svg+xml file", + fileContents: svgFileContents, + contentDisposition: "inline; filename=xml.svg", + expectedContentType: "image/svg+xml", + expectedContentDisposition: "attachment; filename=xml.svg", + }, + { desc: "text file", fileContents: []byte(`a text file`), + contentDisposition: "", expectedContentType: "text/plain; charset=utf-8", expectedContentDisposition: "inline", }, @@ -47,7 +77,7 @@ func TestHeaders(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - contentType, newContentDisposition := SafeContentHeaders(test.fileContents, "") + contentType, newContentDisposition := SafeContentHeaders(test.fileContents, test.contentDisposition) require.Equal(t, test.expectedContentType, contentType) require.Equal(t, test.expectedContentDisposition, newContentDisposition) diff --git a/workhorse/internal/httprs/httprs.go b/workhorse/internal/httprs/httprs.go index f7767d2ee28..811a50035f3 100644 --- a/workhorse/internal/httprs/httprs.go +++ b/workhorse/internal/httprs/httprs.go @@ -11,6 +11,7 @@ Usage : io.ReadFull(rs, buf) // does a range request and reads from the response body If you want use a specific http.Client for additional range requests : + rs := httprs.NewHttpReadSeeker(resp, client) */ package httprs diff --git a/workhorse/internal/queueing/queue.go b/workhorse/internal/queueing/queue.go index c8d32280355..266f8897450 100644 --- a/workhorse/internal/queueing/queue.go +++ b/workhorse/internal/queueing/queue.go @@ -27,8 +27,9 @@ type queueMetrics struct { // newQueueMetrics prepares Prometheus metrics for queueing mechanism // name specifies name of the queue, used to label metrics with ConstLabel `queue_name` // timeout specifies the timeout of storing a request in queue - queueMetrics -// uses it to calculate histogram buckets for gitlab_workhorse_queueing_waiting_time -// metric +// +// uses it to calculate histogram buckets for gitlab_workhorse_queueing_waiting_time +// metric func newQueueMetrics(name string, timeout time.Duration, reg prometheus.Registerer) *queueMetrics { waitingTimeBuckets := []float64{ timeout.Seconds() * 0.01, diff --git a/workhorse/internal/queueing/requests.go b/workhorse/internal/queueing/requests.go index c3df614de41..a6ca98a674a 100644 --- a/workhorse/internal/queueing/requests.go +++ b/workhorse/internal/queueing/requests.go @@ -16,7 +16,9 @@ const ( // QueueRequests creates a new request queue // name specifies the name of queue, used to label Prometheus metrics -// Don't call QueueRequests twice with the same name argument! +// +// Don't call QueueRequests twice with the same name argument! +// // h specifies a http.Handler which will handle the queue requests // limit specifies number of requests run concurrently // queueLimit specifies maximum number of requests that can be queued diff --git a/workhorse/internal/secret/jwt.go b/workhorse/internal/secret/jwt.go index ce0de6ca38d..70970cfc1b3 100644 --- a/workhorse/internal/secret/jwt.go +++ b/workhorse/internal/secret/jwt.go @@ -3,7 +3,7 @@ package secret import ( "fmt" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" ) var ( diff --git a/workhorse/internal/sendurl/sendurl.go b/workhorse/internal/sendurl/sendurl.go index e689fc84a0f..9cc1cd352b7 100644 --- a/workhorse/internal/sendurl/sendurl.go +++ b/workhorse/internal/sendurl/sendurl.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "net/http" + "strings" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -21,6 +22,9 @@ type entry struct{ senddata.Prefix } type entryParams struct { URL string AllowRedirects bool + Body string + Header http.Header + Method string } var SendURL = &entry{"send-url:"} @@ -86,6 +90,9 @@ func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) fail.Request(w, r, fmt.Errorf("SendURL: unpack sendData: %v", err)) return } + if params.Method == "" { + params.Method = http.MethodGet + } log.WithContextFields(r.Context(), log.Fields{ "url": mask.URL(params.URL), @@ -99,7 +106,7 @@ func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) } // create new request and copy range headers - newReq, err := http.NewRequest("GET", params.URL, nil) + newReq, err := http.NewRequest(params.Method, params.URL, strings.NewReader(params.Body)) if err != nil { sendURLRequestsInvalidData.Inc() fail.Request(w, r, fmt.Errorf("SendURL: NewRequest: %v", err)) @@ -111,6 +118,12 @@ func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) newReq.Header[header] = r.Header[header] } + for key, values := range params.Header { + for _, value := range values { + newReq.Header.Add(key, value) + } + } + // execute new request var resp *http.Response if params.AllowRedirects { diff --git a/workhorse/internal/sendurl/sendurl_test.go b/workhorse/internal/sendurl/sendurl_test.go index bca6b7d3075..ea0c6a43af6 100644 --- a/workhorse/internal/sendurl/sendurl_test.go +++ b/workhorse/internal/sendurl/sendurl_test.go @@ -2,7 +2,9 @@ package sendurl import ( "encoding/base64" + "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -194,3 +196,49 @@ func TestDownloadingNonExistingRemoteFileWithSendURL(t *testing.T) { response := testEntryServer(t, "/get/file-not-existing", nil, false) require.Equal(t, http.StatusNotFound, response.Code) } + +func TestPostRequest(t *testing.T) { + body := "any string" + header := map[string][]string{"Authorization": []string{"Bearer token"}} + postRequestHandler := func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST", r.Method) + + url := r.URL.String() + "/external/url" + + jsonParams, err := json.Marshal(entryParams{URL: url, Body: body, Header: header, Method: "POST"}) + require.NoError(t, err) + + data := base64.URLEncoding.EncodeToString([]byte(jsonParams)) + + SendURL.Inject(w, r, data) + } + externalPostUrlHandler := func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST", r.Method) + + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.Equal(t, body, string(b)) + + require.Equal(t, []string{"Bearer token"}, r.Header["Authorization"]) + + w.Write([]byte(testData)) + } + + mux := http.NewServeMux() + mux.HandleFunc("/post/request/external/url", externalPostUrlHandler) + + server := httptest.NewServer(mux) + defer server.Close() + + httpRequest, err := http.NewRequest("POST", server.URL+"/post/request", nil) + require.NoError(t, err) + + response := httptest.NewRecorder() + postRequestHandler(response, httpRequest) + + require.Equal(t, http.StatusOK, response.Code) + + result, err := io.ReadAll(response.Body) + require.NoError(t, err) + require.Equal(t, testData, string(result)) +} diff --git a/workhorse/internal/testhelper/testhelper.go b/workhorse/internal/testhelper/testhelper.go index 6ea5c1c73e1..c21d60bdf36 100644 --- a/workhorse/internal/testhelper/testhelper.go +++ b/workhorse/internal/testhelper/testhelper.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/labkit/log" diff --git a/workhorse/internal/upload/artifacts_upload_test.go b/workhorse/internal/upload/artifacts_upload_test.go index c94129092c6..cd2761f8376 100644 --- a/workhorse/internal/upload/artifacts_upload_test.go +++ b/workhorse/internal/upload/artifacts_upload_test.go @@ -13,7 +13,7 @@ import ( "os" "testing" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "gitlab.com/gitlab-org/labkit/log" diff --git a/workhorse/internal/upload/body_uploader_test.go b/workhorse/internal/upload/body_uploader_test.go index 837d119e72e..97cd05be8a3 100644 --- a/workhorse/internal/upload/body_uploader_test.go +++ b/workhorse/internal/upload/body_uploader_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" diff --git a/workhorse/internal/upload/destination/destination.go b/workhorse/internal/upload/destination/destination.go index a9fb81540d5..6a82a69b8ba 100644 --- a/workhorse/internal/upload/destination/destination.go +++ b/workhorse/internal/upload/destination/destination.go @@ -12,7 +12,7 @@ import ( "strconv" "time" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "gitlab.com/gitlab-org/labkit/log" diff --git a/workhorse/internal/upload/destination/destination_test.go b/workhorse/internal/upload/destination/destination_test.go index 69dd02ca7c2..0ed588a5204 100644 --- a/workhorse/internal/upload/destination/destination_test.go +++ b/workhorse/internal/upload/destination/destination_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/stretchr/testify/require" "gocloud.dev/blob" diff --git a/workhorse/internal/upload/saved_file_tracker_test.go b/workhorse/internal/upload/saved_file_tracker_test.go index 4f323bf8888..1c05a1183fe 100644 --- a/workhorse/internal/upload/saved_file_tracker_test.go +++ b/workhorse/internal/upload/saved_file_tracker_test.go @@ -3,7 +3,7 @@ package upload import ( "context" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "net/http" "testing" diff --git a/workhorse/internal/upload/uploads.go b/workhorse/internal/upload/uploads.go index a3072aa5d00..91a0e0ca79d 100644 --- a/workhorse/internal/upload/uploads.go +++ b/workhorse/internal/upload/uploads.go @@ -9,7 +9,7 @@ import ( "mime/multipart" "net/http" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper/fail" diff --git a/workhorse/internal/upstream/routes.go b/workhorse/internal/upstream/routes.go index 982f3a5b5f8..c5af9dc3f00 100644 --- a/workhorse/internal/upstream/routes.go +++ b/workhorse/internal/upstream/routes.go @@ -325,6 +325,9 @@ func configureRoutes(u *upstream) { // Requirements Import via UI upload acceleration u.route("POST", projectPattern+`requirements_management/requirements/import_csv`, mimeMultipartUploader), + // Work items Import via UI upload acceleration + u.route("POST", projectPattern+`work_items/import_csv`, mimeMultipartUploader), + // Uploads via API u.route("POST", apiProjectPattern+`/uploads\z`, mimeMultipartUploader), @@ -403,6 +406,8 @@ func configureRoutes(u *upstream) { u.route("", "^/api/v4/geo_replication", defaultUpstream), u.route("", "^/api/v4/geo/proxy_git_ssh", defaultUpstream), u.route("", "^/api/v4/geo/graphql", defaultUpstream), + u.route("", "^/api/v4/geo_nodes/current/failures", defaultUpstream), + u.route("", "^/api/v4/geo_sites/current/failures", defaultUpstream), // Internal API routes u.route("", "^/api/v4/internal", defaultUpstream), diff --git a/workhorse/internal/upstream/routes_test.go b/workhorse/internal/upstream/routes_test.go index 8a032519bdf..13c000bf791 100644 --- a/workhorse/internal/upstream/routes_test.go +++ b/workhorse/internal/upstream/routes_test.go @@ -22,6 +22,17 @@ func TestAdminGeoPathsWithGeoProxy(t *testing.T) { runTestCasesWithGeoProxyEnabled(t, testCases) } +func TestApiGeoPathsWithGeoProxy(t *testing.T) { + testCases := []testCase{ + {"Geo replication endpoint", "/api/v4/geo_replication", "Local Rails server received request to path /api/v4/geo_replication"}, + {"Geo GraphQL endpoint", "/api/v4/geo/graphql", "Local Rails server received request to path /api/v4/geo/graphql"}, + {"Current geo node failures", "/api/v4/geo_nodes/current/failures", "Local Rails server received request to path /api/v4/geo_nodes/current/failures"}, + {"Current geo sites failures", "/api/v4/geo_sites/current/failures", "Local Rails server received request to path /api/v4/geo_sites/current/failures"}, + } + + runTestCasesWithGeoProxyEnabled(t, testCases) +} + func TestProjectNotExistingGitHttpPullWithGeoProxy(t *testing.T) { testCases := []testCase{ {"secondary info/refs", "/group/project.git/info/refs", "Local Rails server received request to path /group/project.git/info/refs"}, diff --git a/workhorse/internal/zipartifacts/metadata.go b/workhorse/internal/zipartifacts/metadata.go index 456d2c101cb..9fd158529e5 100644 --- a/workhorse/internal/zipartifacts/metadata.go +++ b/workhorse/internal/zipartifacts/metadata.go @@ -1,6 +1,7 @@ package zipartifacts import ( + "archive/zip" "compress/gzip" "encoding/binary" "encoding/json" @@ -8,8 +9,6 @@ import ( "path" "sort" "strconv" - - zip "gitlab.com/gitlab-org/golang-archive-zip" ) type metadata struct { @@ -30,7 +29,6 @@ func newMetadata(file *zip.File) metadata { } return metadata{ - //lint:ignore SA1019 Remove this once the minimum supported version is go 1.10 (go 1.9 and down do not support an alternative) Modified: file.ModTime().Unix(), Mode: strconv.FormatUint(uint64(file.Mode().Perm()), 8), CRC: file.CRC32, diff --git a/workhorse/internal/zipartifacts/metadata_test.go b/workhorse/internal/zipartifacts/metadata_test.go index 6bde56ef27d..4ebb522d199 100644 --- a/workhorse/internal/zipartifacts/metadata_test.go +++ b/workhorse/internal/zipartifacts/metadata_test.go @@ -1,6 +1,7 @@ package zipartifacts import ( + "archive/zip" "bytes" "compress/gzip" "context" @@ -10,7 +11,6 @@ import ( "testing" "github.com/stretchr/testify/require" - zip "gitlab.com/gitlab-org/golang-archive-zip" ) func generateTestArchive(w io.Writer) error { diff --git a/workhorse/internal/zipartifacts/open_archive.go b/workhorse/internal/zipartifacts/open_archive.go index d477725a39f..e8fdf9b8133 100644 --- a/workhorse/internal/zipartifacts/open_archive.go +++ b/workhorse/internal/zipartifacts/open_archive.go @@ -1,6 +1,7 @@ package zipartifacts import ( + "archive/zip" "context" "fmt" "io" @@ -11,7 +12,6 @@ import ( "gitlab.com/gitlab-org/gitlab/workhorse/internal/httprs" "gitlab.com/gitlab-org/gitlab/workhorse/internal/transport" - zip "gitlab.com/gitlab-org/golang-archive-zip" "gitlab.com/gitlab-org/labkit/mask" ) diff --git a/workhorse/internal/zipartifacts/open_archive_test.go b/workhorse/internal/zipartifacts/open_archive_test.go index e339454345b..22f02ab6354 100644 --- a/workhorse/internal/zipartifacts/open_archive_test.go +++ b/workhorse/internal/zipartifacts/open_archive_test.go @@ -1,6 +1,7 @@ package zipartifacts import ( + "archive/zip" "bytes" "context" "fmt" @@ -12,7 +13,6 @@ import ( "testing" "github.com/stretchr/testify/require" - zip "gitlab.com/gitlab-org/golang-archive-zip" ) func createArchive(t *testing.T, dir string) (map[string][]byte, int64) { |