diff options
author | Arran Walker <ajwalker@gitlab.com> | 2021-12-16 01:53:57 +0300 |
---|---|---|
committer | Jaime Martinez <jmartinez@gitlab.com> | 2021-12-16 01:53:57 +0300 |
commit | 26e1d310b513cc235f00336f8866c8c059f7ce80 (patch) | |
tree | 3646bdf78353a7f79f5d89c548cdf02d9d730df3 | |
parent | cc23d6ef34be2fd5e6dff719dd1eb62a51b7a82f (diff) |
fix: update vfs/zip implementation to ensure minimum range requests for go1.17
Using `gitlab.com/gitlab-org/golang-archive-zip` instead of the `archive/zip`
package from the stdlib.
Changelog: changed
-rw-r--r-- | go.mod | 1 | ||||
-rw-r--r-- | go.sum | 2 | ||||
-rw-r--r-- | internal/vfs/zip/archive.go | 4 | ||||
-rw-r--r-- | internal/vfs/zip/archive_test.go | 82 |
4 files changed, 88 insertions, 1 deletions
@@ -25,6 +25,7 @@ require ( github.com/tj/assert v0.0.3 // indirect github.com/tj/go-redirects v0.0.0-20180508180010-5c02ead0bbc5 gitlab.com/gitlab-org/go-mimedb v1.45.0 + gitlab.com/gitlab-org/golang-archive-zip v0.1.1 gitlab.com/gitlab-org/labkit v1.11.0 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 @@ -443,6 +443,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= gitlab.com/gitlab-org/go-mimedb v1.45.0 h1:PO8dx6HEWzPYU6MQTYnCbpQEJzhJLW/Bh43+2VUHTgc= gitlab.com/gitlab-org/go-mimedb v1.45.0/go.mod h1:wa9y/zOSFKmTXLyBs4clz2FNVhZQmmEQM9TxslPAjZ0= +gitlab.com/gitlab-org/golang-archive-zip v0.1.1 h1:35k9giivbxwF03+8A05Cm8YoxoakU8FBCj5gysjCTCE= +gitlab.com/gitlab-org/golang-archive-zip v0.1.1/go.mod h1:ZDtqpWPGPB9qBuZnZDrKQjIdJtkN7ZAoVwhT6H2o2kE= gitlab.com/gitlab-org/labkit v1.11.0 h1:oUi02wj097Xr8NBkyDEHQGtCThI/oocoYSAv9sT8BLU= gitlab.com/gitlab-org/labkit v1.11.0/go.mod h1:tnvyKiPF/Y0MeBy+ACYWRWexOEQk6PTRGUc9GstcnXc= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= diff --git a/internal/vfs/zip/archive.go b/internal/vfs/zip/archive.go index 588fb76d..551d10c0 100644 --- a/internal/vfs/zip/archive.go +++ b/internal/vfs/zip/archive.go @@ -1,7 +1,6 @@ package zip import ( - "archive/zip" "context" "errors" "fmt" @@ -14,6 +13,9 @@ import ( "sync/atomic" "time" + // TODO: Revert back to zip/archive once we no longer support go1.17 + // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/673 + zip "gitlab.com/gitlab-org/golang-archive-zip" "gitlab.com/gitlab-org/labkit/log" "gitlab.com/gitlab-org/gitlab-pages/internal/httprange" diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index 423d9eaf..37529e22 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -5,11 +5,14 @@ import ( "bytes" "context" "crypto/rand" + "fmt" "io" "net/http" "net/http/httptest" "os" + "path/filepath" "strconv" + "strings" "sync/atomic" "testing" "time" @@ -382,6 +385,85 @@ func TestReadArchiveFails(t *testing.T) { require.EqualError(t, err, os.ErrNotExist.Error()) } +func createArchive(t *testing.T, dir string) (map[string][]byte, int64) { + t.Helper() + + f, err := os.Create(filepath.Join(dir, "public.zip")) + require.NoError(t, err) + defer f.Close() + zw := zip.NewWriter(f) + + entries := make(map[string][]byte) + for _, size := range []int{0, 32 * 1024, 128 * 1024, 5 * 1024 * 1024} { + entryName := fmt.Sprintf("public/file_%d", size) + entries[entryName] = bytes.Repeat([]byte{'z'}, size) + + w, err := zw.Create(entryName) + require.NoError(t, err) + + _, err = w.Write(entries[entryName]) + require.NoError(t, err) + } + + require.NoError(t, zw.Close()) + + fi, err := f.Stat() + require.NoError(t, err) + require.NoError(t, f.Close()) + + return entries, fi.Size() +} + +func TestMinimalRangeRequests(t *testing.T) { + dir := t.TempDir() + entries, size := createArchive(t, dir) + + mux := http.NewServeMux() + + var ranges []string + mux.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) { + rangeHdr := r.Header.Get("Range") + if rangeHdr == "" { + rw.Header().Add("Content-Length", fmt.Sprintf("%d", size)) + return + } + + ranges = append(ranges, rangeHdr) + + http.FileServer(http.Dir(dir)).ServeHTTP(rw, r) + }) + + srv := httptest.NewServer(mux) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fs := New(&zipCfg).(*zipVFS) + zip := newArchive(fs, time.Second) + err := zip.openArchive(ctx, srv.URL+"/public.zip") + require.NoError(t, err) + + require.Len(t, zip.files, len(entries)) + require.Len(t, ranges, 3, "range requests should be minimal") + + for _, zf := range zip.files { + if !zf.Mode().IsRegular() { + continue + } + + f, err := zip.Open(context.Background(), strings.TrimPrefix(zf.Name, "public/")) + require.NoError(t, err) + + io.Copy(io.Discard, f) + + require.NoError(t, f.Close()) + } + + // ensure minimal requests: https://gitlab.com/gitlab-org/gitlab-pages/-/issues/625 + require.Len(t, ranges, 11, "range requests should be minimal") +} + func openZipArchive(t *testing.T, requests *int64, fromDisk bool) (*zipArchive, func()) { t.Helper() |