diff options
author | Vladimir Shushlin <vshushlin@gitlab.com> | 2020-10-13 13:52:51 +0300 |
---|---|---|
committer | Vladimir Shushlin <vshushlin@gitlab.com> | 2020-10-13 13:52:51 +0300 |
commit | 16a1e15632fa670e15ed512753380f45965f26d0 (patch) | |
tree | 8d298b27342decb4c146f386f093082c2103b2f0 /internal | |
parent | 6755ba75aff4044f7dfa40578426195b9922907a (diff) | |
parent | 77b42fad727a8d8166d8d9a3aa6951f530a664ea (diff) |
Merge branch 'feature/brotli' into 'master'
Support for statically compressed brotli content-encoding
See merge request gitlab-org/gitlab-pages!359
Diffstat (limited to 'internal')
-rw-r--r-- | internal/serving/disk/helpers.go | 61 | ||||
-rw-r--r-- | internal/serving/disk/reader.go | 4 | ||||
-rw-r--r-- | internal/source/disk/domain_test.go | 130 |
3 files changed, 160 insertions, 35 deletions
diff --git a/internal/serving/disk/helpers.go b/internal/serving/disk/helpers.go index 1ff83683..5456724a 100644 --- a/internal/serving/disk/helpers.go +++ b/internal/serving/disk/helpers.go @@ -5,6 +5,7 @@ import ( "io" "mime" "net/http" + "os" "path/filepath" "strconv" "strings" @@ -13,6 +14,19 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" ) +var compressedEncodings = map[string]string{ + "br": ".br", + "gzip": ".gz", +} + +// Server side content encoding priority. +// Map iteration order is not deterministic in go, so we need this array to specify the priority +// when the client doesn't provide one +var compressedEncodingsPriority = []string{ + "br", + "gzip", +} + func endsWithSlash(path string) bool { return strings.HasSuffix(path, "/") } @@ -46,33 +60,42 @@ func (reader *Reader) detectContentType(ctx context.Context, root vfs.Root, path return contentType, nil } -func acceptsGZip(r *http.Request) bool { +func (reader *Reader) handleContentEncoding(ctx context.Context, w http.ResponseWriter, r *http.Request, root vfs.Root, fullPath string) string { + // don't accept range requests for compressed content if r.Header.Get("Range") != "" { - return false + return fullPath } - offers := []string{"gzip", "identity"} - acceptedEncoding := httputil.NegotiateContentEncoding(r, offers) - return acceptedEncoding == "gzip" -} + files := map[string]os.FileInfo{} -func (reader *Reader) handleGZip(ctx context.Context, w http.ResponseWriter, r *http.Request, root vfs.Root, fullPath string) string { - if !acceptsGZip(r) { - return fullPath - } + // finding compressed files + for encoding, extension := range compressedEncodings { + path := fullPath + extension - gzipPath := fullPath + ".gz" + // Ensure the file is not a symlink + if fi, err := root.Lstat(ctx, path); err == nil && fi.Mode().IsRegular() { + files[encoding] = fi + } + } - // Ensure the .gz file is not a symlink - fi, err := root.Lstat(ctx, gzipPath) - if err != nil || !fi.Mode().IsRegular() { - return fullPath + offers := make([]string, 0, len(files)+1) + for _, encoding := range compressedEncodingsPriority { + if _, ok := files[encoding]; ok { + offers = append(offers, encoding) + } } + offers = append(offers, "identity") - w.Header().Set("Content-Encoding", "gzip") + acceptedEncoding := httputil.NegotiateContentEncoding(r, offers) + + if fi, ok := files[acceptedEncoding]; ok { + w.Header().Set("Content-Encoding", acceptedEncoding) - // http.ServeContent doesn't set Content-Length if Content-Encoding is set - w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) + // http.ServeContent doesn't set Content-Length if Content-Encoding is set + w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) + + return fullPath + compressedEncodings[acceptedEncoding] + } - return gzipPath + return fullPath } diff --git a/internal/serving/disk/reader.go b/internal/serving/disk/reader.go index 6afe7947..e7f15a1e 100644 --- a/internal/serving/disk/reader.go +++ b/internal/serving/disk/reader.go @@ -169,7 +169,7 @@ func (reader *Reader) resolvePath(ctx context.Context, root vfs.Root, subPath .. } func (reader *Reader) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, root vfs.Root, origPath string, accessControl bool) error { - fullPath := reader.handleGZip(ctx, w, r, root, origPath) + fullPath := reader.handleContentEncoding(ctx, w, r, root, origPath) file, err := root.Open(ctx, fullPath) if err != nil { @@ -212,7 +212,7 @@ func (reader *Reader) serveFile(ctx context.Context, w http.ResponseWriter, r *h } func (reader *Reader) serveCustomFile(ctx context.Context, w http.ResponseWriter, r *http.Request, code int, root vfs.Root, origPath string) error { - fullPath := reader.handleGZip(ctx, w, r, root, origPath) + fullPath := reader.handleContentEncoding(ctx, w, r, root, origPath) // Open and serve content of file file, err := root.Open(ctx, fullPath) diff --git a/internal/source/disk/domain_test.go b/internal/source/disk/domain_test.go index 5c7bdbea..1c81ba22 100644 --- a/internal/source/disk/domain_test.go +++ b/internal/source/disk/domain_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/andybalholm/brotli" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" @@ -176,7 +177,7 @@ func TestIsHTTPSOnly(t *testing.T) { } } -func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, contentType string, ungzip bool) { +func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, contentType string, expectCompressed bool) { w := httptest.NewRecorder() req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) require.NoError(t, err) @@ -185,17 +186,17 @@ func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, valu } handler(w, req) - if ungzip { + if expectCompressed { contentLength := w.Header().Get("Content-Length") require.Equal(t, strconv.Itoa(w.Body.Len()), contentLength, "Content-Length") + contentEncoding := w.Header().Get("Content-Encoding") + require.Equal(t, "gzip", contentEncoding, "Content-Encoding") + reader, err := gzip.NewReader(w.Body) require.NoError(t, err) defer reader.Close() - contentEncoding := w.Header().Get("Content-Encoding") - require.Equal(t, "gzip", contentEncoding, "Content-Encoding") - bytes, err := ioutil.ReadAll(reader) require.NoError(t, err) require.Contains(t, string(bytes), str) @@ -223,19 +224,18 @@ func TestGroupServeHTTPGzip(t *testing.T) { } testSet := []struct { - mode string // HTTP mode - url string // Test URL - acceptEncoding string // Accept encoding header - body interface{} // Expected body at above URL - contentType string // Expected content-type - ungzip bool // Expect the response to be gzipped? + mode string // HTTP mode + url string // Test URL + acceptEncoding string // Accept encoding header + body interface{} // Expected body at above URL + contentType string // Expected content-type + expectCompressed bool // Expect the response to be gzipped? }{ // No gzip encoding requested {"GET", "/index.html", "", "main-dir", "text/html; charset=utf-8", false}, {"GET", "/index.html", "identity", "main-dir", "text/html; charset=utf-8", false}, {"GET", "/index.html", "gzip; q=0", "main-dir", "text/html; charset=utf-8", false}, // gzip encoding requested, - {"GET", "/index.html", "*", "main-dir", "text/html; charset=utf-8", true}, {"GET", "/index.html", "identity, gzip", "main-dir", "text/html; charset=utf-8", true}, {"GET", "/index.html", "gzip", "main-dir", "text/html; charset=utf-8", true}, {"GET", "/index.html", "gzip; q=1", "main-dir", "text/html; charset=utf-8", true}, @@ -243,6 +243,10 @@ func TestGroupServeHTTPGzip(t *testing.T) { {"GET", "/index.html", "gzip, deflate", "main-dir", "text/html; charset=utf-8", true}, {"GET", "/index.html", "gzip; q=1, deflate", "main-dir", "text/html; charset=utf-8", true}, {"GET", "/index.html", "gzip; q=0.9, deflate", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "br; q=0.9, gzip; q=1", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "br; q=0, gzip; q=1", "main-dir", "text/html; charset=utf-8", true}, + // fallback to gzip because .br is missing + {"GET", "/index2.html", "*", "main-dir", "text/html; charset=utf-8", true}, // gzip encoding requested, but url does not have compressed content on disk {"GET", "/project2/index.html", "*", "project2-main", "text/html; charset=utf-8", false}, {"GET", "/project2/index.html", "identity, gzip", "project2-main", "text/html; charset=utf-8", false}, @@ -259,6 +263,102 @@ func TestGroupServeHTTPGzip(t *testing.T) { // Symlinked .gz files are not supported {"GET", "/gz-symlink", "*", "data", "text/plain; charset=utf-8", false}, // Unknown file-extension, with text content + {"GET", "/text.unknown", "gzip", "hello", "text/plain; charset=utf-8", true}, + {"GET", "/text-nogzip.unknown", "*", "hello", "text/plain; charset=utf-8", false}, + // Unknown file-extension, with PNG content + {"GET", "/image.unknown", "gzip", "GIF89a", "image/gif", true}, + {"GET", "/image-nogzip.unknown", "*", "GIF89a", "image/gif", false}, + } + + for _, tt := range testSet { + t.Run(tt.url+" acceptEncoding: "+tt.acceptEncoding, func(t *testing.T) { + URL := "http://group.test.io" + tt.url + testHTTPGzip(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.expectCompressed) + }) + } +} + +func testHTTPBrotli(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, contentType string, expectCompressed bool) { + w := httptest.NewRecorder() + req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) + require.NoError(t, err) + if acceptEncoding != "" { + req.Header.Add("Accept-Encoding", acceptEncoding) + } + handler(w, req) + + if expectCompressed { + contentLength := w.Header().Get("Content-Length") + require.Equal(t, strconv.Itoa(w.Body.Len()), contentLength, "Content-Length") + + contentEncoding := w.Header().Get("Content-Encoding") + require.Equal(t, "br", contentEncoding, "Content-Encoding") + + reader := brotli.NewReader(w.Body) + bytes, err := ioutil.ReadAll(reader) + require.NoError(t, err) + require.Contains(t, string(bytes), str) + } else { + require.Contains(t, w.Body.String(), str) + } + + require.Equal(t, contentType, w.Header().Get("Content-Type")) +} + +func TestGroupServeHTTPBrotli(t *testing.T) { + cleanup := setUpTests(t) + defer cleanup() + + testGroup := &domain.Domain{ + Resolver: &Group{ + name: "group", + projects: map[string]*projectConfig{ + "group.test.io": &projectConfig{}, + "group.gitlab-example.com": &projectConfig{}, + "project": &projectConfig{}, + "project2": &projectConfig{}, + }, + }, + } + + testSet := []struct { + mode string // HTTP mode + url string // Test URL + acceptEncoding string // Accept encoding header + body interface{} // Expected body at above URL + contentType string // Expected content-type + expectCompressed bool // Expect the response to be br compressed? + }{ + // No br encoding requested + {"GET", "/index.html", "", "main-dir", "text/html; charset=utf-8", false}, + {"GET", "/index.html", "identity", "main-dir", "text/html; charset=utf-8", false}, + {"GET", "/index.html", "br; q=0", "main-dir", "text/html; charset=utf-8", false}, + // br encoding requested, + {"GET", "/index.html", "*", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "identity, br", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "br", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "br; q=1", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "br; q=0.9", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "br, deflate", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "br; q=1, deflate", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "br; q=0.9, deflate", "main-dir", "text/html; charset=utf-8", true}, + {"GET", "/index.html", "gzip; q=0.5, br; q=1", "main-dir", "text/html; charset=utf-8", true}, + // br encoding requested, but url does not have compressed content on disk + {"GET", "/project2/index.html", "*", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "identity, br", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "br", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "br; q=1", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "br; q=0.9", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "br, deflate", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "br; q=1, deflate", "project2-main", "text/html; charset=utf-8", false}, + {"GET", "/project2/index.html", "br; q=0.9, deflate", "project2-main", "text/html; charset=utf-8", false}, + // malformed headers + {"GET", "/index.html", ";; br", "main-dir", "text/html; charset=utf-8", false}, + {"GET", "/index.html", "middle-out", "main-dir", "text/html; charset=utf-8", false}, + {"GET", "/index.html", "br; quality=1", "main-dir", "text/html; charset=utf-8", false}, + // Symlinked .br files are not supported + {"GET", "/gz-symlink", "*", "data", "text/plain; charset=utf-8", false}, + // Unknown file-extension, with text content {"GET", "/text.unknown", "*", "hello", "text/plain; charset=utf-8", true}, {"GET", "/text-nogzip.unknown", "*", "hello", "text/plain; charset=utf-8", false}, // Unknown file-extension, with PNG content @@ -267,8 +367,10 @@ func TestGroupServeHTTPGzip(t *testing.T) { } for _, tt := range testSet { - URL := "http://group.test.io" + tt.url - testHTTPGzip(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.ungzip) + t.Run(tt.url+" acceptEncoding: "+tt.acceptEncoding, func(t *testing.T) { + URL := "http://group.test.io" + tt.url + testHTTPBrotli(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.expectCompressed) + }) } } |