Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/workhorse/internal/artifacts
diff options
context:
space:
mode:
Diffstat (limited to 'workhorse/internal/artifacts')
-rw-r--r--workhorse/internal/artifacts/artifacts_store_test.go338
-rw-r--r--workhorse/internal/artifacts/artifacts_test.go19
-rw-r--r--workhorse/internal/artifacts/artifacts_upload.go167
-rw-r--r--workhorse/internal/artifacts/artifacts_upload_test.go322
-rw-r--r--workhorse/internal/artifacts/entry.go123
-rw-r--r--workhorse/internal/artifacts/entry_test.go134
-rw-r--r--workhorse/internal/artifacts/escape_quotes.go10
7 files changed, 1113 insertions, 0 deletions
diff --git a/workhorse/internal/artifacts/artifacts_store_test.go b/workhorse/internal/artifacts/artifacts_store_test.go
new file mode 100644
index 00000000000..bd56d9ea725
--- /dev/null
+++ b/workhorse/internal/artifacts/artifacts_store_test.go
@@ -0,0 +1,338 @@
+package artifacts
+
+import (
+ "archive/zip"
+ "bytes"
+ "crypto/md5"
+ "encoding/hex"
+ "fmt"
+ "io/ioutil"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/objectstore/test"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+)
+
+func createTestZipArchive(t *testing.T) (data []byte, md5Hash string) {
+ var buffer bytes.Buffer
+ archive := zip.NewWriter(&buffer)
+ fileInArchive, err := archive.Create("test.file")
+ require.NoError(t, err)
+ fmt.Fprint(fileInArchive, "test")
+ archive.Close()
+ data = buffer.Bytes()
+
+ hasher := md5.New()
+ hasher.Write(data)
+ hexHash := hasher.Sum(nil)
+ md5Hash = hex.EncodeToString(hexHash)
+
+ return data, md5Hash
+}
+
+func createTestMultipartForm(t *testing.T, data []byte) (bytes.Buffer, string) {
+ var buffer bytes.Buffer
+ writer := multipart.NewWriter(&buffer)
+ file, err := writer.CreateFormFile("file", "my.file")
+ require.NoError(t, err)
+ file.Write(data)
+ writer.Close()
+ return buffer, writer.FormDataContentType()
+}
+
+func testUploadArtifactsFromTestZip(t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
+ archiveData, _ := createTestZipArchive(t)
+ contentBuffer, contentType := createTestMultipartForm(t, archiveData)
+
+ return testUploadArtifacts(t, contentType, ts.URL+Path, &contentBuffer)
+}
+
+func TestUploadHandlerSendingToExternalStorage(t *testing.T) {
+ tempPath, err := ioutil.TempDir("", "uploads")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(tempPath)
+
+ archiveData, md5 := createTestZipArchive(t)
+ archiveFile, err := ioutil.TempFile("", "artifact.zip")
+ require.NoError(t, err)
+ defer os.Remove(archiveFile.Name())
+ _, err = archiveFile.Write(archiveData)
+ require.NoError(t, err)
+ archiveFile.Close()
+
+ storeServerCalled := 0
+ storeServerMux := http.NewServeMux()
+ storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
+ require.Equal(t, "PUT", r.Method)
+
+ receivedData, err := ioutil.ReadAll(r.Body)
+ require.NoError(t, err)
+ require.Equal(t, archiveData, receivedData)
+
+ storeServerCalled++
+ w.Header().Set("ETag", md5)
+ w.WriteHeader(200)
+ })
+ storeServerMux.HandleFunc("/store-id", func(w http.ResponseWriter, r *http.Request) {
+ http.ServeFile(w, r, archiveFile.Name())
+ })
+
+ responseProcessorCalled := 0
+ responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+ require.Equal(t, "store-id", r.FormValue("file.remote_id"))
+ require.NotEmpty(t, r.FormValue("file.remote_url"))
+ w.WriteHeader(200)
+ responseProcessorCalled++
+ }
+
+ storeServer := httptest.NewServer(storeServerMux)
+ defer storeServer.Close()
+
+ qs := fmt.Sprintf("?%s=%s", ArtifactFormatKey, ArtifactFormatZip)
+
+ tests := []struct {
+ name string
+ preauth api.Response
+ }{
+ {
+ name: "ObjectStore Upload",
+ preauth: api.Response{
+ RemoteObject: api.RemoteObject{
+ StoreURL: storeServer.URL + "/url/put" + qs,
+ ID: "store-id",
+ GetURL: storeServer.URL + "/store-id",
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ storeServerCalled = 0
+ responseProcessorCalled = 0
+
+ ts := testArtifactsUploadServer(t, test.preauth, responseProcessor)
+ defer ts.Close()
+
+ contentBuffer, contentType := createTestMultipartForm(t, archiveData)
+ response := testUploadArtifacts(t, contentType, ts.URL+Path+qs, &contentBuffer)
+ require.Equal(t, http.StatusOK, response.Code)
+ testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
+ require.Equal(t, 1, storeServerCalled, "store should be called only once")
+ require.Equal(t, 1, responseProcessorCalled, "response processor should be called only once")
+ })
+ }
+}
+
+func TestUploadHandlerSendingToExternalStorageAndStorageServerUnreachable(t *testing.T) {
+ tempPath, err := ioutil.TempDir("", "uploads")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(tempPath)
+
+ responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+ t.Fatal("it should not be called")
+ }
+
+ authResponse := api.Response{
+ TempPath: tempPath,
+ RemoteObject: api.RemoteObject{
+ StoreURL: "http://localhost:12323/invalid/url",
+ ID: "store-id",
+ },
+ }
+
+ ts := testArtifactsUploadServer(t, authResponse, responseProcessor)
+ defer ts.Close()
+
+ response := testUploadArtifactsFromTestZip(t, ts)
+ require.Equal(t, http.StatusInternalServerError, response.Code)
+}
+
+func TestUploadHandlerSendingToExternalStorageAndInvalidURLIsUsed(t *testing.T) {
+ tempPath, err := ioutil.TempDir("", "uploads")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(tempPath)
+
+ responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+ t.Fatal("it should not be called")
+ }
+
+ authResponse := api.Response{
+ TempPath: tempPath,
+ RemoteObject: api.RemoteObject{
+ StoreURL: "htt:////invalid-url",
+ ID: "store-id",
+ },
+ }
+
+ ts := testArtifactsUploadServer(t, authResponse, responseProcessor)
+ defer ts.Close()
+
+ response := testUploadArtifactsFromTestZip(t, ts)
+ require.Equal(t, http.StatusInternalServerError, response.Code)
+}
+
+func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T) {
+ putCalledTimes := 0
+
+ storeServerMux := http.NewServeMux()
+ storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
+ putCalledTimes++
+ require.Equal(t, "PUT", r.Method)
+ w.WriteHeader(510)
+ })
+
+ responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+ t.Fatal("it should not be called")
+ }
+
+ storeServer := httptest.NewServer(storeServerMux)
+ defer storeServer.Close()
+
+ authResponse := api.Response{
+ RemoteObject: api.RemoteObject{
+ StoreURL: storeServer.URL + "/url/put",
+ ID: "store-id",
+ },
+ }
+
+ ts := testArtifactsUploadServer(t, authResponse, responseProcessor)
+ defer ts.Close()
+
+ response := testUploadArtifactsFromTestZip(t, ts)
+ require.Equal(t, http.StatusInternalServerError, response.Code)
+ require.Equal(t, 1, putCalledTimes, "upload should be called only once")
+}
+
+func TestUploadHandlerSendingToExternalStorageAndSupportRequestTimeout(t *testing.T) {
+ putCalledTimes := 0
+
+ storeServerMux := http.NewServeMux()
+ storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
+ putCalledTimes++
+ require.Equal(t, "PUT", r.Method)
+ time.Sleep(10 * time.Second)
+ w.WriteHeader(510)
+ })
+
+ responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+ t.Fatal("it should not be called")
+ }
+
+ storeServer := httptest.NewServer(storeServerMux)
+ defer storeServer.Close()
+
+ authResponse := api.Response{
+ RemoteObject: api.RemoteObject{
+ StoreURL: storeServer.URL + "/url/put",
+ ID: "store-id",
+ Timeout: 1,
+ },
+ }
+
+ ts := testArtifactsUploadServer(t, authResponse, responseProcessor)
+ defer ts.Close()
+
+ response := testUploadArtifactsFromTestZip(t, ts)
+ require.Equal(t, http.StatusInternalServerError, response.Code)
+ require.Equal(t, 1, putCalledTimes, "upload should be called only once")
+}
+
+func TestUploadHandlerMultipartUploadSizeLimit(t *testing.T) {
+ os, server := test.StartObjectStore()
+ defer server.Close()
+
+ err := os.InitiateMultipartUpload(test.ObjectPath)
+ require.NoError(t, err)
+
+ objectURL := server.URL + test.ObjectPath
+
+ uploadSize := 10
+ preauth := api.Response{
+ RemoteObject: api.RemoteObject{
+ ID: "store-id",
+ MultipartUpload: &api.MultipartUploadParams{
+ PartSize: 1,
+ PartURLs: []string{objectURL + "?partNumber=1"},
+ AbortURL: objectURL, // DELETE
+ CompleteURL: objectURL, // POST
+ },
+ },
+ }
+
+ responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+ t.Fatal("it should not be called")
+ }
+
+ ts := testArtifactsUploadServer(t, preauth, responseProcessor)
+ defer ts.Close()
+
+ contentBuffer, contentType := createTestMultipartForm(t, make([]byte, uploadSize))
+ response := testUploadArtifacts(t, contentType, ts.URL+Path, &contentBuffer)
+ require.Equal(t, http.StatusRequestEntityTooLarge, response.Code)
+
+ // Poll because AbortMultipartUpload is async
+ for i := 0; os.IsMultipartUpload(test.ObjectPath) && i < 100; i++ {
+ time.Sleep(10 * time.Millisecond)
+ }
+ require.False(t, os.IsMultipartUpload(test.ObjectPath), "MultipartUpload should not be in progress anymore")
+ require.Empty(t, os.GetObjectMD5(test.ObjectPath), "upload should have failed, so the object should not exists")
+}
+
+func TestUploadHandlerMultipartUploadMaximumSizeFromApi(t *testing.T) {
+ os, server := test.StartObjectStore()
+ defer server.Close()
+
+ err := os.InitiateMultipartUpload(test.ObjectPath)
+ require.NoError(t, err)
+
+ objectURL := server.URL + test.ObjectPath
+
+ uploadSize := int64(10)
+ maxSize := uploadSize - 1
+ preauth := api.Response{
+ MaximumSize: maxSize,
+ RemoteObject: api.RemoteObject{
+ ID: "store-id",
+ MultipartUpload: &api.MultipartUploadParams{
+ PartSize: uploadSize,
+ PartURLs: []string{objectURL + "?partNumber=1"},
+ AbortURL: objectURL, // DELETE
+ CompleteURL: objectURL, // POST
+ },
+ },
+ }
+
+ responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+ t.Fatal("it should not be called")
+ }
+
+ ts := testArtifactsUploadServer(t, preauth, responseProcessor)
+ defer ts.Close()
+
+ contentBuffer, contentType := createTestMultipartForm(t, make([]byte, uploadSize))
+ response := testUploadArtifacts(t, contentType, ts.URL+Path, &contentBuffer)
+ require.Equal(t, http.StatusRequestEntityTooLarge, response.Code)
+
+ testhelper.Retry(t, 5*time.Second, func() error {
+ if os.GetObjectMD5(test.ObjectPath) == "" {
+ return nil
+ }
+
+ return fmt.Errorf("file is still present")
+ })
+}
diff --git a/workhorse/internal/artifacts/artifacts_test.go b/workhorse/internal/artifacts/artifacts_test.go
new file mode 100644
index 00000000000..b9a42cc60c1
--- /dev/null
+++ b/workhorse/internal/artifacts/artifacts_test.go
@@ -0,0 +1,19 @@
+package artifacts
+
+import (
+ "os"
+ "testing"
+
+ "gitlab.com/gitlab-org/labkit/log"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+)
+
+func TestMain(m *testing.M) {
+ if err := testhelper.BuildExecutables(); err != nil {
+ log.WithError(err).Fatal()
+ }
+
+ os.Exit(m.Run())
+
+}
diff --git a/workhorse/internal/artifacts/artifacts_upload.go b/workhorse/internal/artifacts/artifacts_upload.go
new file mode 100644
index 00000000000..3d4b8bf0931
--- /dev/null
+++ b/workhorse/internal/artifacts/artifacts_upload.go
@@ -0,0 +1,167 @@
+package artifacts
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "mime/multipart"
+ "net/http"
+ "os"
+ "os/exec"
+ "strings"
+ "syscall"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+ "gitlab.com/gitlab-org/labkit/log"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
+)
+
+// Sent by the runner: https://gitlab.com/gitlab-org/gitlab-runner/blob/c24da19ecce8808d9d2950896f70c94f5ea1cc2e/network/gitlab.go#L580
+const (
+ ArtifactFormatKey = "artifact_format"
+ ArtifactFormatZip = "zip"
+ ArtifactFormatDefault = ""
+)
+
+var zipSubcommandsErrorsCounter = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "gitlab_workhorse_zip_subcommand_errors_total",
+ Help: "Errors comming from subcommands used for processing ZIP archives",
+ }, []string{"error"})
+
+type artifactsUploadProcessor struct {
+ opts *filestore.SaveFileOpts
+ format string
+
+ upload.SavedFileTracker
+}
+
+func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, file *filestore.FileHandler) (*filestore.FileHandler, error) {
+ metaReader, metaWriter := io.Pipe()
+ defer metaWriter.Close()
+
+ metaOpts := &filestore.SaveFileOpts{
+ LocalTempPath: a.opts.LocalTempPath,
+ TempFilePrefix: "metadata.gz",
+ }
+ if metaOpts.LocalTempPath == "" {
+ metaOpts.LocalTempPath = os.TempDir()
+ }
+
+ fileName := file.LocalPath
+ if fileName == "" {
+ fileName = file.RemoteURL
+ }
+
+ zipMd := exec.CommandContext(ctx, "gitlab-zip-metadata", fileName)
+ zipMd.Stderr = log.ContextLogger(ctx).Writer()
+ zipMd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ zipMd.Stdout = metaWriter
+
+ if err := zipMd.Start(); err != nil {
+ return nil, err
+ }
+ defer helper.CleanUpProcessGroup(zipMd)
+
+ type saveResult struct {
+ error
+ *filestore.FileHandler
+ }
+ done := make(chan saveResult)
+ go func() {
+ var result saveResult
+ result.FileHandler, result.error = filestore.SaveFileFromReader(ctx, metaReader, -1, metaOpts)
+
+ done <- result
+ }()
+
+ if err := zipMd.Wait(); err != nil {
+ st, ok := helper.ExitStatus(err)
+
+ if !ok {
+ return nil, err
+ }
+
+ zipSubcommandsErrorsCounter.WithLabelValues(zipartifacts.ErrorLabelByCode(st)).Inc()
+
+ if st == zipartifacts.CodeNotZip {
+ return nil, nil
+ }
+
+ if st == zipartifacts.CodeLimitsReached {
+ return nil, zipartifacts.ErrBadMetadata
+ }
+ }
+
+ metaWriter.Close()
+ result := <-done
+ return result.FileHandler, result.error
+}
+
+func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName string, file *filestore.FileHandler, writer *multipart.Writer) error {
+ // ProcessFile for artifacts requires file form-data field name to eq `file`
+
+ if formName != "file" {
+ return fmt.Errorf("invalid form field: %q", formName)
+ }
+ if a.Count() > 0 {
+ return fmt.Errorf("artifacts request contains more than one file")
+ }
+ a.Track(formName, file.LocalPath)
+
+ select {
+ case <-ctx.Done():
+ return fmt.Errorf("ProcessFile: context done")
+ default:
+ }
+
+ if !strings.EqualFold(a.format, ArtifactFormatZip) && a.format != ArtifactFormatDefault {
+ return nil
+ }
+
+ // TODO: can we rely on disk for shipping metadata? Not if we split workhorse and rails in 2 different PODs
+ metadata, err := a.generateMetadataFromZip(ctx, file)
+ if err != nil {
+ return err
+ }
+
+ if metadata != nil {
+ fields, err := metadata.GitLabFinalizeFields("metadata")
+ if err != nil {
+ return fmt.Errorf("finalize metadata field error: %v", err)
+ }
+
+ for k, v := range fields {
+ writer.WriteField(k, v)
+ }
+
+ a.Track("metadata", metadata.LocalPath)
+ }
+
+ return nil
+}
+
+func (a *artifactsUploadProcessor) Name() string {
+ return "artifacts"
+}
+
+func UploadArtifacts(myAPI *api.API, h http.Handler, p upload.Preparer) http.Handler {
+ return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+ opts, _, err := p.Prepare(a)
+ if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("UploadArtifacts: error preparing file storage options"))
+ return
+ }
+
+ format := r.URL.Query().Get(ArtifactFormatKey)
+
+ mg := &artifactsUploadProcessor{opts: opts, format: format, SavedFileTracker: upload.SavedFileTracker{Request: r}}
+ upload.HandleFileUploads(w, r, h, a, mg, opts)
+ }, "/authorize")
+}
diff --git a/workhorse/internal/artifacts/artifacts_upload_test.go b/workhorse/internal/artifacts/artifacts_upload_test.go
new file mode 100644
index 00000000000..c82ae791239
--- /dev/null
+++ b/workhorse/internal/artifacts/artifacts_upload_test.go
@@ -0,0 +1,322 @@
+package artifacts
+
+import (
+ "archive/zip"
+ "bytes"
+ "compress/gzip"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "testing"
+
+ "github.com/dgrijalva/jwt-go"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/roundtripper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
+
+ "github.com/stretchr/testify/require"
+)
+
+const (
+ MetadataHeaderKey = "Metadata-Status"
+ MetadataHeaderPresent = "present"
+ MetadataHeaderMissing = "missing"
+ Path = "/url/path"
+)
+
+func testArtifactsUploadServer(t *testing.T, authResponse api.Response, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *httptest.Server {
+ mux := http.NewServeMux()
+ mux.HandleFunc(Path+"/authorize", func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != "POST" {
+ t.Fatal("Expected POST request")
+ }
+
+ w.Header().Set("Content-Type", api.ResponseContentType)
+
+ data, err := json.Marshal(&authResponse)
+ if err != nil {
+ t.Fatal("Expected to marshal")
+ }
+ w.Write(data)
+ })
+ mux.HandleFunc(Path, func(w http.ResponseWriter, r *http.Request) {
+ opts, err := filestore.GetOpts(&authResponse)
+ require.NoError(t, err)
+
+ if r.Method != "POST" {
+ t.Fatal("Expected POST request")
+ }
+ if opts.IsLocal() {
+ if r.FormValue("file.path") == "" {
+ t.Fatal("Expected file to be present")
+ return
+ }
+
+ _, err := ioutil.ReadFile(r.FormValue("file.path"))
+ if err != nil {
+ t.Fatal("Expected file to be readable")
+ return
+ }
+ } else {
+ if r.FormValue("file.remote_url") == "" {
+ t.Fatal("Expected file to be remote accessible")
+ return
+ }
+ }
+
+ if r.FormValue("metadata.path") != "" {
+ metadata, err := ioutil.ReadFile(r.FormValue("metadata.path"))
+ if err != nil {
+ t.Fatal("Expected metadata to be readable")
+ return
+ }
+ gz, err := gzip.NewReader(bytes.NewReader(metadata))
+ if err != nil {
+ t.Fatal("Expected metadata to be valid gzip")
+ return
+ }
+ defer gz.Close()
+ metadata, err = ioutil.ReadAll(gz)
+ if err != nil {
+ t.Fatal("Expected metadata to be valid")
+ return
+ }
+ if !bytes.HasPrefix(metadata, []byte(zipartifacts.MetadataHeaderPrefix+zipartifacts.MetadataHeader)) {
+ t.Fatal("Expected metadata to be of valid format")
+ return
+ }
+
+ w.Header().Set(MetadataHeaderKey, MetadataHeaderPresent)
+
+ } else {
+ w.Header().Set(MetadataHeaderKey, MetadataHeaderMissing)
+ }
+
+ w.WriteHeader(http.StatusOK)
+
+ if bodyProcessor != nil {
+ bodyProcessor(w, r)
+ }
+ })
+ return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
+}
+
+type testServer struct {
+ url string
+ writer *multipart.Writer
+ buffer *bytes.Buffer
+ fileWriter io.Writer
+ cleanup func()
+}
+
+func setupWithTmpPath(t *testing.T, filename string, includeFormat bool, format string, authResponse *api.Response, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *testServer {
+ tempPath, err := ioutil.TempDir("", "uploads")
+ require.NoError(t, err)
+
+ if authResponse == nil {
+ authResponse = &api.Response{TempPath: tempPath}
+ }
+
+ ts := testArtifactsUploadServer(t, *authResponse, bodyProcessor)
+
+ var buffer bytes.Buffer
+ writer := multipart.NewWriter(&buffer)
+ fileWriter, err := writer.CreateFormFile(filename, "my.file")
+ require.NotNil(t, fileWriter)
+ require.NoError(t, err)
+
+ cleanup := func() {
+ ts.Close()
+ require.NoError(t, os.RemoveAll(tempPath))
+ require.NoError(t, writer.Close())
+ }
+
+ qs := ""
+
+ if includeFormat {
+ qs = fmt.Sprintf("?%s=%s", ArtifactFormatKey, format)
+ }
+
+ return &testServer{url: ts.URL + Path + qs, writer: writer, buffer: &buffer, fileWriter: fileWriter, cleanup: cleanup}
+}
+
+func testUploadArtifacts(t *testing.T, contentType, url string, body io.Reader) *httptest.ResponseRecorder {
+ httpRequest, err := http.NewRequest("POST", url, body)
+ require.NoError(t, err)
+
+ httpRequest.Header.Set("Content-Type", contentType)
+ response := httptest.NewRecorder()
+ parsedURL := helper.URLMustParse(url)
+ roundTripper := roundtripper.NewTestBackendRoundTripper(parsedURL)
+ testhelper.ConfigureSecret()
+ apiClient := api.NewAPI(parsedURL, "123", roundTripper)
+ proxyClient := proxy.NewProxy(parsedURL, "123", roundTripper)
+ UploadArtifacts(apiClient, proxyClient, &upload.DefaultPreparer{}).ServeHTTP(response, httpRequest)
+ return response
+}
+
+func TestUploadHandlerAddingMetadata(t *testing.T) {
+ testCases := []struct {
+ desc string
+ format string
+ includeFormat bool
+ }{
+ {
+ desc: "ZIP format",
+ format: ArtifactFormatZip,
+ includeFormat: true,
+ },
+ {
+ desc: "default format",
+ format: ArtifactFormatDefault,
+ includeFormat: true,
+ },
+ {
+ desc: "default format without artifact_format",
+ format: ArtifactFormatDefault,
+ includeFormat: false,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.desc, func(t *testing.T) {
+ s := setupWithTmpPath(t, "file", tc.includeFormat, tc.format, nil,
+ func(w http.ResponseWriter, r *http.Request) {
+ token, err := jwt.ParseWithClaims(r.Header.Get(upload.RewrittenFieldsHeader), &upload.MultipartClaims{}, testhelper.ParseJWT)
+ require.NoError(t, err)
+
+ rewrittenFields := token.Claims.(*upload.MultipartClaims).RewrittenFields
+ require.Equal(t, 2, len(rewrittenFields))
+
+ require.Contains(t, rewrittenFields, "file")
+ require.Contains(t, rewrittenFields, "metadata")
+ require.Contains(t, r.PostForm, "file.gitlab-workhorse-upload")
+ require.Contains(t, r.PostForm, "metadata.gitlab-workhorse-upload")
+ },
+ )
+ defer s.cleanup()
+
+ archive := zip.NewWriter(s.fileWriter)
+ file, err := archive.Create("test.file")
+ require.NotNil(t, file)
+ require.NoError(t, err)
+
+ require.NoError(t, archive.Close())
+ require.NoError(t, s.writer.Close())
+
+ response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+ require.Equal(t, http.StatusOK, response.Code)
+ testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
+ })
+ }
+}
+
+func TestUploadHandlerTarArtifact(t *testing.T) {
+ s := setupWithTmpPath(t, "file", true, "tar", nil,
+ func(w http.ResponseWriter, r *http.Request) {
+ token, err := jwt.ParseWithClaims(r.Header.Get(upload.RewrittenFieldsHeader), &upload.MultipartClaims{}, testhelper.ParseJWT)
+ require.NoError(t, err)
+
+ rewrittenFields := token.Claims.(*upload.MultipartClaims).RewrittenFields
+ require.Equal(t, 1, len(rewrittenFields))
+
+ require.Contains(t, rewrittenFields, "file")
+ require.Contains(t, r.PostForm, "file.gitlab-workhorse-upload")
+ },
+ )
+ defer s.cleanup()
+
+ file, err := os.Open("../../testdata/tarfile.tar")
+ require.NoError(t, err)
+
+ _, err = io.Copy(s.fileWriter, file)
+ require.NoError(t, err)
+ require.NoError(t, file.Close())
+ require.NoError(t, s.writer.Close())
+
+ response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+ require.Equal(t, http.StatusOK, response.Code)
+ testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderMissing)
+}
+
+func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
+ s := setupWithTmpPath(t, "file", true, "other", nil, nil)
+ defer s.cleanup()
+ require.NoError(t, s.writer.Close())
+
+ response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+ require.Equal(t, http.StatusOK, response.Code)
+ testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderMissing)
+}
+
+func TestUploadHandlerForMultipleFiles(t *testing.T) {
+ s := setupWithTmpPath(t, "file", true, "", nil, nil)
+ defer s.cleanup()
+
+ file, err := s.writer.CreateFormFile("file", "my.file")
+ require.NotNil(t, file)
+ require.NoError(t, err)
+ require.NoError(t, s.writer.Close())
+
+ response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+ require.Equal(t, http.StatusInternalServerError, response.Code)
+}
+
+func TestUploadFormProcessing(t *testing.T) {
+ s := setupWithTmpPath(t, "metadata", true, "", nil, nil)
+ defer s.cleanup()
+ require.NoError(t, s.writer.Close())
+
+ response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+ require.Equal(t, http.StatusInternalServerError, response.Code)
+}
+
+func TestLsifFileProcessing(t *testing.T) {
+ tempPath, err := ioutil.TempDir("", "uploads")
+ require.NoError(t, err)
+
+ s := setupWithTmpPath(t, "file", true, "zip", &api.Response{TempPath: tempPath, ProcessLsif: true}, nil)
+ defer s.cleanup()
+
+ file, err := os.Open("../../testdata/lsif/valid.lsif.zip")
+ require.NoError(t, err)
+
+ _, err = io.Copy(s.fileWriter, file)
+ require.NoError(t, err)
+ require.NoError(t, file.Close())
+ require.NoError(t, s.writer.Close())
+
+ response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+ require.Equal(t, http.StatusOK, response.Code)
+ testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
+}
+
+func TestInvalidLsifFileProcessing(t *testing.T) {
+ tempPath, err := ioutil.TempDir("", "uploads")
+ require.NoError(t, err)
+
+ s := setupWithTmpPath(t, "file", true, "zip", &api.Response{TempPath: tempPath, ProcessLsif: true}, nil)
+ defer s.cleanup()
+
+ file, err := os.Open("../../testdata/lsif/invalid.lsif.zip")
+ require.NoError(t, err)
+
+ _, err = io.Copy(s.fileWriter, file)
+ require.NoError(t, err)
+ require.NoError(t, file.Close())
+ require.NoError(t, s.writer.Close())
+
+ response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+ require.Equal(t, http.StatusInternalServerError, response.Code)
+}
diff --git a/workhorse/internal/artifacts/entry.go b/workhorse/internal/artifacts/entry.go
new file mode 100644
index 00000000000..0c697d40020
--- /dev/null
+++ b/workhorse/internal/artifacts/entry.go
@@ -0,0 +1,123 @@
+package artifacts
+
+import (
+ "bufio"
+ "context"
+ "fmt"
+ "io"
+ "mime"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "syscall"
+
+ "gitlab.com/gitlab-org/labkit/log"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
+)
+
+type entry struct{ senddata.Prefix }
+type entryParams struct{ Archive, Entry string }
+
+var SendEntry = &entry{"artifacts-entry:"}
+
+// Artifacts downloader doesn't support ranges when downloading a single file
+func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params entryParams
+ if err := e.Unpack(&params, sendData); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendEntry: unpack sendData: %v", err))
+ return
+ }
+
+ log.WithContextFields(r.Context(), log.Fields{
+ "entry": params.Entry,
+ "archive": params.Archive,
+ "path": r.URL.Path,
+ }).Print("SendEntry: sending")
+
+ if params.Archive == "" || params.Entry == "" {
+ helper.Fail500(w, r, fmt.Errorf("SendEntry: Archive or Entry is empty"))
+ return
+ }
+
+ err := unpackFileFromZip(r.Context(), params.Archive, params.Entry, w.Header(), w)
+
+ if os.IsNotExist(err) {
+ http.NotFound(w, r)
+ } else if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendEntry: %v", err))
+ }
+}
+
+func detectFileContentType(fileName string) string {
+ contentType := mime.TypeByExtension(filepath.Ext(fileName))
+ if contentType == "" {
+ contentType = "application/octet-stream"
+ }
+ return contentType
+}
+
+func unpackFileFromZip(ctx context.Context, archivePath, encodedFilename string, headers http.Header, output io.Writer) error {
+ fileName, err := zipartifacts.DecodeFileEntry(encodedFilename)
+ if err != nil {
+ return err
+ }
+
+ catFile := exec.Command("gitlab-zip-cat")
+ catFile.Env = append(os.Environ(),
+ "ARCHIVE_PATH="+archivePath,
+ "ENCODED_FILE_NAME="+encodedFilename,
+ )
+ catFile.Stderr = log.ContextLogger(ctx).Writer()
+ catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ stdout, err := catFile.StdoutPipe()
+ if err != nil {
+ return fmt.Errorf("create gitlab-zip-cat stdout pipe: %v", err)
+ }
+
+ if err := catFile.Start(); err != nil {
+ return fmt.Errorf("start %v: %v", catFile.Args, err)
+ }
+ defer helper.CleanUpProcessGroup(catFile)
+
+ basename := filepath.Base(fileName)
+ reader := bufio.NewReader(stdout)
+ contentLength, err := reader.ReadString('\n')
+ if err != nil {
+ if catFileErr := waitCatFile(catFile); catFileErr != nil {
+ return catFileErr
+ }
+ return fmt.Errorf("read content-length: %v", err)
+ }
+ contentLength = strings.TrimSuffix(contentLength, "\n")
+
+ // Write http headers about the file
+ headers.Set("Content-Length", contentLength)
+ headers.Set("Content-Type", detectFileContentType(fileName))
+ headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
+ // Copy file body to client
+ if _, err := io.Copy(output, reader); err != nil {
+ return fmt.Errorf("copy stdout of %v: %v", catFile.Args, err)
+ }
+
+ return waitCatFile(catFile)
+}
+
+func waitCatFile(cmd *exec.Cmd) error {
+ err := cmd.Wait()
+ if err == nil {
+ return nil
+ }
+
+ st, ok := helper.ExitStatus(err)
+
+ if ok && (st == zipartifacts.CodeArchiveNotFound || st == zipartifacts.CodeEntryNotFound) {
+ return os.ErrNotExist
+ }
+ return fmt.Errorf("wait for %v to finish: %v", cmd.Args, err)
+
+}
diff --git a/workhorse/internal/artifacts/entry_test.go b/workhorse/internal/artifacts/entry_test.go
new file mode 100644
index 00000000000..6f1e9d360aa
--- /dev/null
+++ b/workhorse/internal/artifacts/entry_test.go
@@ -0,0 +1,134 @@
+package artifacts
+
+import (
+ "archive/zip"
+ "encoding/base64"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+)
+
+func testEntryServer(t *testing.T, archive string, entry string) *httptest.ResponseRecorder {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
+ require.Equal(t, "GET", r.Method)
+
+ encodedEntry := base64.StdEncoding.EncodeToString([]byte(entry))
+ jsonParams := fmt.Sprintf(`{"Archive":"%s","Entry":"%s"}`, archive, encodedEntry)
+ data := base64.URLEncoding.EncodeToString([]byte(jsonParams))
+
+ SendEntry.Inject(w, r, data)
+ })
+
+ httpRequest, err := http.NewRequest("GET", "/url/path", nil)
+ require.NoError(t, err)
+ response := httptest.NewRecorder()
+ mux.ServeHTTP(response, httpRequest)
+ return response
+}
+
+func TestDownloadingFromValidArchive(t *testing.T) {
+ tempFile, err := ioutil.TempFile("", "uploads")
+ require.NoError(t, err)
+ defer tempFile.Close()
+ defer os.Remove(tempFile.Name())
+
+ archive := zip.NewWriter(tempFile)
+ defer archive.Close()
+ fileInArchive, err := archive.Create("test.txt")
+ require.NoError(t, err)
+ fmt.Fprint(fileInArchive, "testtest")
+ archive.Close()
+
+ response := testEntryServer(t, tempFile.Name(), "test.txt")
+
+ require.Equal(t, 200, response.Code)
+
+ testhelper.RequireResponseHeader(t, response,
+ "Content-Type",
+ "text/plain; charset=utf-8")
+ testhelper.RequireResponseHeader(t, response,
+ "Content-Disposition",
+ "attachment; filename=\"test.txt\"")
+
+ testhelper.RequireResponseBody(t, response, "testtest")
+}
+
+func TestDownloadingFromValidHTTPArchive(t *testing.T) {
+ tempDir, err := ioutil.TempDir("", "uploads")
+ require.NoError(t, err)
+ defer os.RemoveAll(tempDir)
+
+ f, err := os.Create(filepath.Join(tempDir, "archive.zip"))
+ require.NoError(t, err)
+ defer f.Close()
+
+ archive := zip.NewWriter(f)
+ defer archive.Close()
+ fileInArchive, err := archive.Create("test.txt")
+ require.NoError(t, err)
+ fmt.Fprint(fileInArchive, "testtest")
+ archive.Close()
+ f.Close()
+
+ fileServer := httptest.NewServer(http.FileServer(http.Dir(tempDir)))
+ defer fileServer.Close()
+
+ response := testEntryServer(t, fileServer.URL+"/archive.zip", "test.txt")
+
+ require.Equal(t, 200, response.Code)
+
+ testhelper.RequireResponseHeader(t, response,
+ "Content-Type",
+ "text/plain; charset=utf-8")
+ testhelper.RequireResponseHeader(t, response,
+ "Content-Disposition",
+ "attachment; filename=\"test.txt\"")
+
+ testhelper.RequireResponseBody(t, response, "testtest")
+}
+
+func TestDownloadingNonExistingFile(t *testing.T) {
+ tempFile, err := ioutil.TempFile("", "uploads")
+ require.NoError(t, err)
+ defer tempFile.Close()
+ defer os.Remove(tempFile.Name())
+
+ archive := zip.NewWriter(tempFile)
+ defer archive.Close()
+ archive.Close()
+
+ response := testEntryServer(t, tempFile.Name(), "test")
+ require.Equal(t, 404, response.Code)
+}
+
+func TestDownloadingFromInvalidArchive(t *testing.T) {
+ response := testEntryServer(t, "path/to/non/existing/file", "test")
+ require.Equal(t, 404, response.Code)
+}
+
+func TestIncompleteApiResponse(t *testing.T) {
+ response := testEntryServer(t, "", "")
+ require.Equal(t, 500, response.Code)
+}
+
+func TestDownloadingFromNonExistingHTTPArchive(t *testing.T) {
+ tempDir, err := ioutil.TempDir("", "uploads")
+ require.NoError(t, err)
+ defer os.RemoveAll(tempDir)
+
+ fileServer := httptest.NewServer(http.FileServer(http.Dir(tempDir)))
+ defer fileServer.Close()
+
+ response := testEntryServer(t, fileServer.URL+"/not-existing-archive-file.zip", "test.txt")
+
+ require.Equal(t, 404, response.Code)
+}
diff --git a/workhorse/internal/artifacts/escape_quotes.go b/workhorse/internal/artifacts/escape_quotes.go
new file mode 100644
index 00000000000..94db2be39b7
--- /dev/null
+++ b/workhorse/internal/artifacts/escape_quotes.go
@@ -0,0 +1,10 @@
+package artifacts
+
+import "strings"
+
+// taken from mime/multipart/writer.go
+var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+
+func escapeQuotes(s string) string {
+ return quoteEscaper.Replace(s)
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-12 23:18:08 +0300