diff options
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | domain.go | 38 | ||||
| -rw-r--r-- | domain_test.go | 81 | ||||
| -rw-r--r-- | internal/httputil/LICENSE | 27 | ||||
| -rw-r--r-- | internal/httputil/README.md | 5 | ||||
| -rw-r--r-- | internal/httputil/header/header.go | 298 | ||||
| -rw-r--r-- | internal/httputil/negotiate.go | 80 | ||||
| -rw-r--r-- | shared/pages/group/group.test.io/public/index.html.gz | bin | 0 -> 40 bytes |
8 files changed, 528 insertions, 5 deletions
@@ -40,11 +40,11 @@ complexity: test: go get golang.org/x/tools/cmd/cover - go test ./... -short -cover -v -timeout 1m + go test . -short -cover -v -timeout 1m acceptance: gitlab-pages go get golang.org/x/tools/cmd/cover - go test ./... -cover -v -timeout 1m + go test . -cover -v -timeout 1m docker: docker run --rm -it -v ${PWD}:/go/src/pages -w /go/src/pages golang:1.5 /bin/bash @@ -11,6 +11,8 @@ import ( "path/filepath" "strconv" "strings" + + "gitlab.com/gitlab-org/gitlab-pages/internal/httputil" ) type domain struct { @@ -21,8 +23,33 @@ type domain struct { certificateError error } +func acceptsGZip(r *http.Request) bool { + if r.Header.Get("Range") != "" { + return false + } + offers := []string{"gzip", "identity"} + acceptedEncoding := httputil.NegotiateContentEncoding(r, offers) + return acceptedEncoding == "gzip" +} + func (d *domain) serveFile(w http.ResponseWriter, r *http.Request, fullPath string) error { // Open and serve content of file + if acceptsGZip(r) { + _, err := os.Stat(fullPath + ".gz") + if err == nil { + // Set the content type based on the non-gzipped extension + _, haveType := w.Header()["Content-Type"] + if !haveType { + ctype := mime.TypeByExtension(filepath.Ext(fullPath)) + w.Header().Set("Content-Type", ctype) + } + + // Serve up the gzipped version + fullPath += ".gz" + w.Header().Set("Content-Encoding", "gzip") + } + } + file, err := os.Open(fullPath) if err != nil { return err @@ -41,6 +68,15 @@ func (d *domain) serveFile(w http.ResponseWriter, r *http.Request, fullPath stri func (d *domain) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, fullPath string) error { // Open and serve content of file + ext := filepath.Ext(fullPath) + if acceptsGZip(r) { + _, err := os.Stat(fullPath + ".gz") + if err == nil { + // Serve up the gzipped version + fullPath += ".gz" + w.Header().Set("Content-Encoding", "gzip") + } + } file, err := os.Open(fullPath) if err != nil { return err @@ -57,7 +93,7 @@ func (d *domain) serveCustomFile(w http.ResponseWriter, r *http.Request, code in // Serve the file _, haveType := w.Header()["Content-Type"] if !haveType { - ctype := mime.TypeByExtension(filepath.Ext(fullPath)) + ctype := mime.TypeByExtension(ext) w.Header().Set("Content-Type", ctype) } w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) diff --git a/domain_test.go b/domain_test.go index 7298c22b..b4879ee0 100644 --- a/domain_test.go +++ b/domain_test.go @@ -1,13 +1,16 @@ package main import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "compress/gzip" + "io/ioutil" "mime" "net/http" "net/http/httptest" "net/url" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGroupServeHTTP(t *testing.T) { @@ -51,6 +54,80 @@ func TestDomainServeHTTP(t *testing.T) { assert.HTTPError(t, testDomain.ServeHTTP, "GET", "/not-existing-file", nil) } +func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, ungzip bool) { + w := httptest.NewRecorder() + req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) + require.NoError(t, err) + if acceptEncoding != "" { + req.Header.Add("Accept-Encoding", acceptEncoding) + } + handler(w, req) + + if ungzip { + reader, err := gzip.NewReader(w.Body) + require.NoError(t, err) + defer reader.Close() + + contentEncoding := w.Header().Get("Content-Encoding") + assert.Equal(t, "gzip", contentEncoding, "Content-Encoding") + + bytes, err := ioutil.ReadAll(reader) + require.NoError(t, err) + assert.Contains(t, string(bytes), str) + } else { + assert.Contains(t, w.Body.String(), str) + } +} + +func TestGroupServeHTTPGzip(t *testing.T) { + setUpTests() + + testGroup := &domain{ + Group: "group", + Project: "", + } + + testSet := []struct { + mode string // HTTP mode + url string // Test URL + params url.Values // Test URL params + acceptEncoding string // Accept encoding header + body interface{} // Expected body at above URL + ungzip bool // Do we expect the request to require unzip? + }{ + // No gzip encoding requested + {"GET", "http://group.test.io/", nil, "", "main-dir", false}, + {"GET", "http://group.test.io/", nil, "identity", "main-dir", false}, + {"GET", "http://group.test.io/", nil, "gzip; q=0", "main-dir", false}, + // gzip encoding requeste}, + {"GET", "http://group.test.io/", nil, "*", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "identity, gzip", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip; q=1", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip; q=0.9", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip, deflate", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip; q=1, deflate", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip; q=0.9, deflate", "main-dir", true}, + // gzip encoding requested, but url does not have compressed content on disk + {"GET", "http://group.test.io/project2/", nil, "*", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "identity, gzip", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip; q=1", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip; q=0.9", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip, deflate", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip; q=1, deflate", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip; q=0.9, deflate", "project2-main", false}, + // malformed headers + {"GET", "http://group.test.io/", nil, ";; gzip", "main-dir", false}, + {"GET", "http://group.test.io/", nil, "middle-out", "main-dir", false}, + {"GET", "http://group.test.io/", nil, "gzip; quality=1", "main-dir", false}, + } + + for _, tt := range testSet { + testHTTPGzip(t, testGroup.ServeHTTP, tt.mode, tt.url, tt.params, tt.acceptEncoding, tt.body, tt.ungzip) + } +} + func testHTTP404(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, str interface{}) { w := httptest.NewRecorder() req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) diff --git a/internal/httputil/LICENSE b/internal/httputil/LICENSE new file mode 100644 index 00000000..65d761bc --- /dev/null +++ b/internal/httputil/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/httputil/README.md b/internal/httputil/README.md new file mode 100644 index 00000000..5027c2f5 --- /dev/null +++ b/internal/httputil/README.md @@ -0,0 +1,5 @@ +This folder is a partial import of the [GoDoc API package](https://github.com/golang/gddo), +``` +github.com/golang/gddo/httputil +``` +where the original license (see `LICENSE`) has been incorporated herein. diff --git a/internal/httputil/header/header.go b/internal/httputil/header/header.go new file mode 100644 index 00000000..0f1572e3 --- /dev/null +++ b/internal/httputil/header/header.go @@ -0,0 +1,298 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +// Package header provides functions for parsing HTTP headers. +package header + +import ( + "net/http" + "strings" + "time" +) + +// Octet types from RFC 2616. +var octetTypes [256]octetType + +type octetType byte + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = <any 8-bit sequence of data> + // CHAR = <any US-ASCII character (octets 0 - 127)> + // CTL = <any US-ASCII control character (octets 0 - 31) and DEL (127)> + // CR = <US-ASCII CR, carriage return (13)> + // LF = <US-ASCII LF, linefeed (10)> + // SP = <US-ASCII SP, space (32)> + // HT = <US-ASCII HT, horizontal-tab (9)> + // <"> = <US-ASCII double-quote mark (34)> + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = <any OCTET except CTLs, but including LWS> + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1*<any CHAR except CTLs or separators> + // qdtext = <any TEXT except <">> + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +// Copy returns a shallow copy of the header. +func Copy(header http.Header) http.Header { + h := make(http.Header) + for k, vs := range header { + h[k] = vs + } + return h +} + +var timeLayouts = []string{"Mon, 02 Jan 2006 15:04:05 GMT", time.RFC850, time.ANSIC} + +// ParseTime parses the header as time. The zero value is returned if the +// header is not present or there is an error parsing the +// header. +func ParseTime(header http.Header, key string) time.Time { + if s := header.Get(key); s != "" { + for _, layout := range timeLayouts { + if t, err := time.Parse(layout, s); err == nil { + return t.UTC() + } + } + } + return time.Time{} +} + +// ParseList parses a comma separated list of values. Commas are ignored in +// quoted strings. Quoted values are not unescaped or unquoted. Whitespace is +// trimmed. +func ParseList(header http.Header, key string) []string { + var result []string + for _, s := range header[http.CanonicalHeaderKey(key)] { + begin := 0 + end := 0 + escape := false + quote := false + for i := 0; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + end = i + 1 + case quote: + switch b { + case '\\': + escape = true + case '"': + quote = false + } + end = i + 1 + case b == '"': + quote = true + end = i + 1 + case octetTypes[b]&isSpace != 0: + if begin == end { + begin = i + 1 + end = begin + } + case b == ',': + if begin < end { + result = append(result, s[begin:end]) + } + begin = i + 1 + end = begin + default: + end = i + 1 + } + } + if begin < end { + result = append(result, s[begin:end]) + } + } + return result +} + +// ParseValueAndParams parses a comma separated list of values with optional +// semicolon separated name-value pairs. Content-Type and Content-Disposition +// headers are in this format. +func ParseValueAndParams(header http.Header, key string) (value string, params map[string]string) { + params = make(map[string]string) + s := header.Get(key) + value, s = expectTokenSlash(s) + if value == "" { + return + } + value = strings.ToLower(value) + s = skipSpace(s) + for strings.HasPrefix(s, ";") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +// AcceptSpec describes an Accept* header. +type AcceptSpec struct { + Value string + Q float64 +} + +// ParseAccept parses Accept* headers. +func ParseAccept(header http.Header, key string) (specs []AcceptSpec) { +loop: + for _, s := range header[key] { + for { + var spec AcceptSpec + spec.Value, s = expectTokenSlash(s) + if spec.Value == "" { + continue loop + } + spec.Q = 1.0 + s = skipSpace(s) + if strings.HasPrefix(s, ";") { + s = skipSpace(s[1:]) + if !strings.HasPrefix(s, "q=") { + continue loop + } + spec.Q, s = expectQuality(s[2:]) + if spec.Q < 0.0 { + continue loop + } + } + specs = append(specs, spec) + s = skipSpace(s) + if !strings.HasPrefix(s, ",") { + continue loop + } + s = skipSpace(s[1:]) + } + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenSlash(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + b := s[i] + if (octetTypes[b]&isToken == 0) && b != '/' { + break + } + } + return s[:i], s[i:] +} + +func expectQuality(s string) (q float64, rest string) { + switch { + case len(s) == 0: + return -1, "" + case s[0] == '0': + q = 0 + case s[0] == '1': + q = 1 + default: + return -1, "" + } + s = s[1:] + if !strings.HasPrefix(s, ".") { + return q, s + } + s = s[1:] + i := 0 + n := 0 + d := 1 + for ; i < len(s); i++ { + b := s[i] + if b < '0' || b > '9' { + break + } + n = n*10 + int(b) - '0' + d *= 10 + } + return q + float64(n)/float64(d), s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + 1; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/internal/httputil/negotiate.go b/internal/httputil/negotiate.go new file mode 100644 index 00000000..a25e3ed1 --- /dev/null +++ b/internal/httputil/negotiate.go @@ -0,0 +1,80 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd. + +package httputil + +import ( + "net/http" + "strings" + + "gitlab.com/gitlab-org/gitlab-pages/internal/httputil/header" +) + +// NegotiateContentEncoding returns the best offered content encoding for the +// request's Accept-Encoding header. If two offers match with equal weight and +// then the offer earlier in the list is preferred. If no offers are +// acceptable, then "" is returned. +func NegotiateContentEncoding(r *http.Request, offers []string) string { + bestOffer := "identity" + bestQ := -1.0 + specs := header.ParseAccept(r.Header, "Accept-Encoding") + for _, offer := range offers { + for _, spec := range specs { + if spec.Q > bestQ && + (spec.Value == "*" || spec.Value == offer) { + bestQ = spec.Q + bestOffer = offer + } + } + } + if bestQ == 0 { + bestOffer = "" + } + return bestOffer +} + +// NegotiateContentType returns the best offered content type for the request's +// Accept header. If two offers match with equal weight, then the more specific +// offer is preferred. For example, text/* trumps */*. If two offers match +// with equal weight and specificity, then the offer earlier in the list is +// preferred. If no offers match, then defaultOffer is returned. +func NegotiateContentType(r *http.Request, offers []string, defaultOffer string) string { + bestOffer := defaultOffer + bestQ := -1.0 + bestWild := 3 + specs := header.ParseAccept(r.Header, "Accept") + for _, offer := range offers { + for _, spec := range specs { + switch { + case spec.Q == 0.0: + // ignore + case spec.Q < bestQ: + // better match found + case spec.Value == "*/*": + if spec.Q > bestQ || bestWild > 2 { + bestQ = spec.Q + bestWild = 2 + bestOffer = offer + } + case strings.HasSuffix(spec.Value, "/*"): + if strings.HasPrefix(offer, spec.Value[:len(spec.Value)-1]) && + (spec.Q > bestQ || bestWild > 1) { + bestQ = spec.Q + bestWild = 1 + bestOffer = offer + } + default: + if spec.Value == offer && + (spec.Q > bestQ || bestWild > 0) { + bestQ = spec.Q + bestWild = 0 + bestOffer = offer + } + } + } + } + return bestOffer +} diff --git a/shared/pages/group/group.test.io/public/index.html.gz b/shared/pages/group/group.test.io/public/index.html.gz Binary files differnew file mode 100644 index 00000000..73fadf1d --- /dev/null +++ b/shared/pages/group/group.test.io/public/index.html.gz |