diff options
Diffstat (limited to 'workhorse/internal/upload/destination/objectstore')
9 files changed, 40 insertions, 50 deletions
diff --git a/workhorse/internal/upload/destination/objectstore/gocloud_object_test.go b/workhorse/internal/upload/destination/objectstore/gocloud_object_test.go index 57b3a35b41e..55d886087be 100644 --- a/workhorse/internal/upload/destination/objectstore/gocloud_object_test.go +++ b/workhorse/internal/upload/destination/objectstore/gocloud_object_test.go @@ -15,8 +15,7 @@ import ( ) func TestGoCloudObjectUpload(t *testing.T) { - mux, _, cleanup := test.SetupGoCloudFileBucket(t, "azuretest") - defer cleanup() + mux, _ := test.SetupGoCloudFileBucket(t, "azuretest") ctx, cancel := context.WithCancel(context.Background()) deadline := time.Now().Add(testTimeout) diff --git a/workhorse/internal/upload/destination/objectstore/multipart.go b/workhorse/internal/upload/destination/objectstore/multipart.go index 4c5b64b27ee..df336d2d901 100644 --- a/workhorse/internal/upload/destination/objectstore/multipart.go +++ b/workhorse/internal/upload/destination/objectstore/multipart.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "os" @@ -66,7 +65,7 @@ func (m *Multipart) Upload(ctx context.Context, r io.Reader) error { } } - n, err := io.Copy(ioutil.Discard, r) + n, err := io.Copy(io.Discard, r) if err != nil { return fmt.Errorf("drain pipe: %v", err) } @@ -93,19 +92,19 @@ func (m *Multipart) Delete() { } func (m *Multipart) readAndUploadOnePart(ctx context.Context, partURL string, putHeaders map[string]string, src io.Reader, partNumber int) (*completeMultipartUploadPart, error) { - file, err := ioutil.TempFile("", "part-buffer") + file, err := os.CreateTemp("", "part-buffer") if err != nil { return nil, fmt.Errorf("create temporary buffer file: %v", err) } defer file.Close() if err := os.Remove(file.Name()); err != nil { - return nil, err + return nil, fmt.Errorf("remove temporary buffer file: %v", err) } n, err := io.Copy(file, src) if err != nil { - return nil, err + return nil, fmt.Errorf("copy to temporary buffer file: %v", err) } if n == 0 { return nil, nil diff --git a/workhorse/internal/upload/destination/objectstore/multipart_test.go b/workhorse/internal/upload/destination/objectstore/multipart_test.go index 4aff3467e30..2a5161e42e7 100644 --- a/workhorse/internal/upload/destination/objectstore/multipart_test.go +++ b/workhorse/internal/upload/destination/objectstore/multipart_test.go @@ -2,7 +2,7 @@ package objectstore_test import ( "context" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strings" @@ -22,7 +22,7 @@ func TestMultipartUploadWithUpcaseETags(t *testing.T) { var putCnt, postCnt int ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, err := ioutil.ReadAll(r.Body) + _, err := io.ReadAll(r.Body) require.NoError(t, err) defer r.Body.Close() diff --git a/workhorse/internal/upload/destination/objectstore/object.go b/workhorse/internal/upload/destination/objectstore/object.go index b7c4f12f009..68c566861af 100644 --- a/workhorse/internal/upload/destination/objectstore/object.go +++ b/workhorse/internal/upload/destination/objectstore/object.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "gitlab.com/gitlab-org/labkit/mask" @@ -53,7 +52,7 @@ func newObject(putURL, deleteURL string, putHeaders map[string]string, size int6 func (o *Object) Upload(ctx context.Context, r io.Reader) error { // we should prevent pr.Close() otherwise it may shadow error set with pr.CloseWithError(err) - req, err := http.NewRequest(http.MethodPut, o.putURL, ioutil.NopCloser(r)) + req, err := http.NewRequest(http.MethodPut, o.putURL, io.NopCloser(r)) if err != nil { return fmt.Errorf("PUT %q: %v", mask.URL(o.putURL), err) diff --git a/workhorse/internal/upload/destination/objectstore/s3_object_test.go b/workhorse/internal/upload/destination/objectstore/s3_object_test.go index b81b0ae2024..0ed14a2e844 100644 --- a/workhorse/internal/upload/destination/objectstore/s3_object_test.go +++ b/workhorse/internal/upload/destination/objectstore/s3_object_test.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "io" - "io/ioutil" - "os" "path/filepath" "strings" "sync" @@ -47,9 +45,7 @@ func TestS3ObjectUpload(t *testing.T) { defer ts.Close() deadline := time.Now().Add(testTimeout) - tmpDir, err := ioutil.TempDir("", "workhorse-test-") - require.NoError(t, err) - defer os.Remove(tmpDir) + tmpDir := t.TempDir() objectName := filepath.Join(tmpDir, "s3-test-data") ctx, cancel := context.WithCancel(context.Background()) @@ -87,9 +83,7 @@ func TestConcurrentS3ObjectUpload(t *testing.T) { defer artifactsServer.Close() deadline := time.Now().Add(testTimeout) - tmpDir, err := ioutil.TempDir("", "workhorse-test-") - require.NoError(t, err) - defer os.Remove(tmpDir) + tmpDir := t.TempDir() var wg sync.WaitGroup @@ -136,9 +130,7 @@ func TestS3ObjectUploadCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) deadline := time.Now().Add(testTimeout) - tmpDir, err := ioutil.TempDir("", "workhorse-test-") - require.NoError(t, err) - defer os.Remove(tmpDir) + tmpDir := t.TempDir() objectName := filepath.Join(tmpDir, "s3-test-data") @@ -160,9 +152,7 @@ func TestS3ObjectUploadLimitReached(t *testing.T) { defer ts.Close() deadline := time.Now().Add(testTimeout) - tmpDir, err := ioutil.TempDir("", "workhorse-test-") - require.NoError(t, err) - defer os.Remove(tmpDir) + tmpDir := t.TempDir() objectName := filepath.Join(tmpDir, "s3-test-data") object, err := objectstore.NewS3Object(objectName, creds, config) diff --git a/workhorse/internal/upload/destination/objectstore/s3_session.go b/workhorse/internal/upload/destination/objectstore/s3_session.go index aa38f18ed7a..d71b38eb22e 100644 --- a/workhorse/internal/upload/destination/objectstore/s3_session.go +++ b/workhorse/internal/upload/destination/objectstore/s3_session.go @@ -10,6 +10,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" + + "gitlab.com/gitlab-org/labkit/fips" ) type s3Session struct { @@ -61,8 +63,9 @@ func setupS3Session(s3Credentials config.S3Credentials, s3Config config.S3Config } cfg := &aws.Config{ - Region: aws.String(s3Config.Region), - S3ForcePathStyle: aws.Bool(s3Config.PathStyle), + Region: aws.String(s3Config.Region), + S3ForcePathStyle: aws.Bool(s3Config.PathStyle), + S3DisableContentMD5Validation: aws.Bool(fips.Enabled()), } // In case IAM profiles aren't being used, use the static credentials diff --git a/workhorse/internal/upload/destination/objectstore/test/gocloud_stub.go b/workhorse/internal/upload/destination/objectstore/test/gocloud_stub.go index cf22075e407..bff0eabaee5 100644 --- a/workhorse/internal/upload/destination/objectstore/test/gocloud_stub.go +++ b/workhorse/internal/upload/destination/objectstore/test/gocloud_stub.go @@ -2,9 +2,7 @@ package test import ( "context" - "io/ioutil" "net/url" - "os" "testing" "github.com/stretchr/testify/require" @@ -20,18 +18,14 @@ func (o *dirOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket return fileblob.OpenBucket(o.tmpDir, nil) } -func SetupGoCloudFileBucket(t *testing.T, scheme string) (m *blob.URLMux, bucketDir string, cleanup func()) { - tmpDir, err := ioutil.TempDir("", "") - require.NoError(t, err) +func SetupGoCloudFileBucket(t *testing.T, scheme string) (m *blob.URLMux, bucketDir string) { + tmpDir := t.TempDir() mux := new(blob.URLMux) fake := &dirOpener{tmpDir: tmpDir} mux.RegisterBucket(scheme, fake) - cleanup = func() { - os.RemoveAll(tmpDir) - } - return mux, tmpDir, cleanup + return mux, tmpDir } func GoCloudObjectExists(t *testing.T, bucketDir string, objectName string) { diff --git a/workhorse/internal/upload/destination/objectstore/test/objectstore_stub.go b/workhorse/internal/upload/destination/objectstore/test/objectstore_stub.go index d51a2de7456..1a380bd5083 100644 --- a/workhorse/internal/upload/destination/objectstore/test/objectstore_stub.go +++ b/workhorse/internal/upload/destination/objectstore/test/objectstore_stub.go @@ -6,7 +6,6 @@ import ( "encoding/xml" "fmt" "io" - "io/ioutil" "net/http" "net/http/httptest" "strconv" @@ -22,7 +21,8 @@ type partsEtagMap map[int]string // Instead of storing objects it will just save md5sum. type ObjectstoreStub struct { // bucket contains md5sum of uploaded objects - bucket map[string]string + bucket map[string]string + contents map[string][]byte // overwriteMD5 contains overwrites for md5sum that should be return instead of the regular hash overwriteMD5 map[string]string // multipart is a map of MultipartUploads @@ -48,6 +48,7 @@ func StartObjectStoreWithCustomMD5(md5Hashes map[string]string) (*ObjectstoreStu multipart: make(map[string]partsEtagMap), overwriteMD5: make(map[string]string), headers: make(map[string]*http.Header), + contents: make(map[string][]byte), } for k, v := range md5Hashes { @@ -82,6 +83,15 @@ func (o *ObjectstoreStub) GetObjectMD5(path string) string { return o.bucket[path] } +// GetObject returns the contents of the uploaded object. The caller must +// not modify the byte slice. +func (o *ObjectstoreStub) GetObject(path string) []byte { + o.m.Lock() + defer o.m.Unlock() + + return o.contents[path] +} + // GetHeader returns a given HTTP header of the object uploaded to the path func (o *ObjectstoreStub) GetHeader(path, key string) string { o.m.Lock() @@ -154,11 +164,11 @@ func (o *ObjectstoreStub) putObject(w http.ResponseWriter, r *http.Request) { etag, overwritten := o.overwriteMD5[objectPath] if !overwritten { + buf, _ := io.ReadAll(r.Body) + o.contents[objectPath] = buf hasher := md5.New() - io.Copy(hasher, r.Body) - - checksum := hasher.Sum(nil) - etag = hex.EncodeToString(checksum) + hasher.Write(buf) + etag = hex.EncodeToString(hasher.Sum(nil)) } o.headers[objectPath] = &r.Header @@ -196,7 +206,7 @@ func (o *ObjectstoreStub) completeMultipartUpload(w http.ResponseWriter, r *http return } - buf, err := ioutil.ReadAll(r.Body) + buf, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), 500) return diff --git a/workhorse/internal/upload/destination/objectstore/test/s3_stub.go b/workhorse/internal/upload/destination/objectstore/test/s3_stub.go index 6b83426b852..6a6b4662904 100644 --- a/workhorse/internal/upload/destination/objectstore/test/s3_stub.go +++ b/workhorse/internal/upload/destination/objectstore/test/s3_stub.go @@ -1,7 +1,6 @@ package test import ( - "io/ioutil" "net/http/httptest" "os" "strings" @@ -76,7 +75,7 @@ func S3ObjectExists(t *testing.T, sess *session.Session, config config.S3Config, require.NoError(t, err) require.Equal(t, int64(len(expectedBytes)), numBytes) - output, err := ioutil.ReadFile(tmpfile.Name()) + output, err := os.ReadFile(tmpfile.Name()) require.NoError(t, err) require.Equal(t, []byte(expectedBytes), output) @@ -124,13 +123,10 @@ func S3ObjectDoesNotExist(t *testing.T, sess *session.Session, config config.S3C } func downloadObject(t *testing.T, sess *session.Session, config config.S3Config, objectName string, handler func(tmpfile *os.File, numBytes int64, err error)) { - tmpDir, err := ioutil.TempDir("", "workhorse-test-") - require.NoError(t, err) - defer os.Remove(tmpDir) + tmpDir := t.TempDir() - tmpfile, err := ioutil.TempFile(tmpDir, "s3-output") + tmpfile, err := os.CreateTemp(tmpDir, "s3-output") require.NoError(t, err) - defer os.Remove(tmpfile.Name()) downloadSvc := s3manager.NewDownloader(sess) numBytes, err := downloadSvc.Download(tmpfile, &s3.GetObjectInput{ |