diff options
author | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2018-03-29 17:43:39 +0300 |
---|---|---|
committer | Nick Thomas <nick@gitlab.com> | 2018-03-29 17:43:39 +0300 |
commit | e7b9a2c510c47f53346f2402eecfec92849e613f (patch) | |
tree | bd78ed49680b09eae808ec381dd3202637fa7ca4 /internal | |
parent | e51175062c0fada8fadc37f6fc96531ff750221b (diff) |
Put domain code in a separate package
Diffstat (limited to 'internal')
-rw-r--r-- | internal/domain/domain.go | 322 | ||||
-rw-r--r-- | internal/domain/domain_config.go | 42 | ||||
-rw-r--r-- | internal/domain/domain_config_test.go | 65 | ||||
-rw-r--r-- | internal/domain/domain_test.go | 281 | ||||
-rw-r--r-- | internal/domain/map.go | 251 | ||||
-rw-r--r-- | internal/domain/map_test.go | 148 | ||||
-rw-r--r-- | internal/fixture/fixtures.go | 56 |
7 files changed, 1165 insertions, 0 deletions
diff --git a/internal/domain/domain.go b/internal/domain/domain.go new file mode 100644 index 00000000..0333cebe --- /dev/null +++ b/internal/domain/domain.go @@ -0,0 +1,322 @@ +package domain + +import ( + "crypto/tls" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" + "gitlab.com/gitlab-org/gitlab-pages/internal/httputil" +) + +type locationDirectoryError struct { + FullPath string + RelativePath string +} + +type project struct { + HTTPSOnly bool +} + +type projects map[string]*project + +// D is a domain that gitlab-pages can serve. +type D struct { + group string + + // custom domains: + projectName string + config *domainConfig + certificate *tls.Certificate + certificateError error + + // group domains: + projects projects +} + +// String implements Stringer. +func (d *D) String() string { + if d.group != "" && d.projectName != "" { + return d.group + "/" + d.projectName + } + + if d.group != "" { + return d.group + } + + return d.projectName +} + +func (l *locationDirectoryError) Error() string { + return "location error accessing directory where file expected" +} + +func acceptsGZip(r *http.Request) bool { + if r.Header.Get("Range") != "" { + return false + } + + offers := []string{"gzip", "identity"} + acceptedEncoding := httputil.NegotiateContentEncoding(r, offers) + return acceptedEncoding == "gzip" +} + +func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string { + if !acceptsGZip(r) { + return fullPath + } + + gzipPath := fullPath + ".gz" + + // Ensure the .gz file is not a symlink + if fi, err := os.Lstat(gzipPath); err != nil || !fi.Mode().IsRegular() { + return fullPath + } + + w.Header().Set("Content-Encoding", "gzip") + + return gzipPath +} + +func setContentType(w http.ResponseWriter, fullPath string) { + ext := filepath.Ext(fullPath) + ctype := mime.TypeByExtension(ext) + if ctype != "" { + w.Header().Set("Content-Type", ctype) + } +} + +// IsHTTPSOnly figures out if the request should be handled with HTTPS +// only by looking at group and project level config. +func (d *D) IsHTTPSOnly(r *http.Request) bool { + if d.config != nil { + return d.config.HTTPSOnly + } + + split := strings.SplitN(r.URL.Path, "/", 3) + if len(split) < 2 { + return false + } + + project := d.projects[split[1]] + + if project != nil { + return project.HTTPSOnly + } + + return false +} + +func (d *D) serveFile(w http.ResponseWriter, r *http.Request, origPath string) error { + fullPath := handleGZip(w, r, origPath) + + file, err := os.Open(fullPath) + if err != nil { + return err + } + defer file.Close() + + fi, err := file.Stat() + if err != nil { + return err + } + + // Set caching headers + w.Header().Set("Cache-Control", "max-age=600") + w.Header().Set("Expires", time.Now().Add(10*time.Minute).Format(time.RFC1123)) + + // ServeContent sets Content-Type for us + http.ServeContent(w, r, origPath, fi.ModTime(), file) + return nil +} + +func (d *D) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, origPath string) error { + fullPath := handleGZip(w, r, origPath) + + // Open and serve content of file + file, err := os.Open(fullPath) + if err != nil { + return err + } + defer file.Close() + + fi, err := file.Stat() + if err != nil { + return err + } + + setContentType(w, origPath) + w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) + w.WriteHeader(code) + + if r.Method != "HEAD" { + _, err := io.CopyN(w, file, fi.Size()) + return err + } + + return nil +} + +// Resolve the HTTP request to a path on disk, converting requests for +// directories to requests for index.html inside the directory if appropriate. +func (d *D) resolvePath(projectName string, subPath ...string) (string, error) { + publicPath := filepath.Join(d.group, projectName, "public") + + // Don't use filepath.Join as cleans the path, + // where we want to traverse full path as supplied by user + // (including ..) + testPath := publicPath + "/" + strings.Join(subPath, "/") + fullPath, err := filepath.EvalSymlinks(testPath) + if err != nil { + return "", err + } + + // The requested path resolved to somewhere outside of the public/ directory + if !strings.HasPrefix(fullPath, publicPath+"/") && fullPath != publicPath { + return "", fmt.Errorf("%q should be in %q", fullPath, publicPath) + } + + fi, err := os.Lstat(fullPath) + if err != nil { + return "", err + } + + // The requested path is a directory, so try index.html via recursion + if fi.IsDir() { + return "", &locationDirectoryError{ + FullPath: fullPath, + RelativePath: strings.TrimPrefix(fullPath, publicPath), + } + } + + // The file exists, but is not a supported type to serve. Perhaps a block + // special device or something else that may be a security risk. + if !fi.Mode().IsRegular() { + return "", fmt.Errorf("%s: is not a regular file", fullPath) + } + + return fullPath, nil +} + +func (d *D) tryNotFound(w http.ResponseWriter, r *http.Request, projectName string) error { + page404, err := d.resolvePath(projectName, "404.html") + if err != nil { + return err + } + + err = d.serveCustomFile(w, r, http.StatusNotFound, page404) + if err != nil { + return err + } + return nil +} + +func (d *D) tryFile(w http.ResponseWriter, r *http.Request, projectName, pathSuffix string, subPath ...string) error { + fullPath, err := d.resolvePath(projectName, subPath...) + + if locationError, _ := err.(*locationDirectoryError); locationError != nil { + if endsWithSlash(r.URL.Path) { + fullPath, err = d.resolvePath(projectName, filepath.Join(subPath...), "index.html") + } else { + redirectPath := "//" + r.Host + "/" + if pathSuffix != "" { + redirectPath += pathSuffix + "/" + } + if locationError.RelativePath != "" { + redirectPath += strings.TrimPrefix(locationError.RelativePath, "/") + "/" + } + http.Redirect(w, r, redirectPath, 302) + return nil + } + } + + if err != nil { + return err + } + + return d.serveFile(w, r, fullPath) +} + +func (d *D) serveFromGroup(w http.ResponseWriter, r *http.Request) { + // The Path always contains "/" at the beginning + split := strings.SplitN(r.URL.Path, "/", 3) + + // Try to serve file for http://group.example.com/subpath/... => /group/subpath/... + if len(split) >= 2 && d.tryFile(w, r, split[1], split[1], split[2:]...) == nil { + return + } + + // Try to serve file for http://group.example.com/... => /group/group.example.com/... + if r.Host != "" && d.tryFile(w, r, strings.ToLower(r.Host), "", r.URL.Path) == nil { + return + } + + // Try serving not found page for http://group.example.com/subpath/ => /group/subpath/404.html + if len(split) >= 2 && d.tryNotFound(w, r, split[1]) == nil { + return + } + + // Try serving not found page for http://group.example.com/ => /group/group.example.com/404.html + if r.Host != "" && d.tryNotFound(w, r, strings.ToLower(r.Host)) == nil { + return + } + + // Serve generic not found + httperrors.Serve404(w) +} + +func (d *D) serveFromConfig(w http.ResponseWriter, r *http.Request) { + // Try to serve file for http://host/... => /group/project/... + if d.tryFile(w, r, d.projectName, "", r.URL.Path) == nil { + return + } + + // Try serving not found page for http://host/ => /group/project/404.html + if d.tryNotFound(w, r, d.projectName) == nil { + return + } + + // Serve generic not found + httperrors.Serve404(w) +} + +// EnsureCertificate parses the PEM-encoded certificate for the domain +func (d *D) EnsureCertificate() (*tls.Certificate, error) { + if d.config == nil { + return nil, errors.New("tls certificates can be loaded only for pages with configuration") + } + + if d.certificate != nil || d.certificateError != nil { + return d.certificate, d.certificateError + } + + tls, err := tls.X509KeyPair([]byte(d.config.Certificate), []byte(d.config.Key)) + if err != nil { + d.certificateError = err + return nil, err + } + + d.certificate = &tls + return d.certificate, nil +} + +// ServeHTTP implements http.Handler. +func (d *D) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if d.config != nil { + d.serveFromConfig(w, r) + } else { + d.serveFromGroup(w, r) + } +} + +func endsWithSlash(path string) bool { + return strings.HasSuffix(path, "/") +} diff --git a/internal/domain/domain_config.go b/internal/domain/domain_config.go new file mode 100644 index 00000000..ed7e0820 --- /dev/null +++ b/internal/domain/domain_config.go @@ -0,0 +1,42 @@ +package domain + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" +) + +type domainConfig struct { + Domain string + Certificate string + Key string + HTTPSOnly bool `json:"https_only"` +} + +type domainsConfig struct { + Domains []domainConfig + HTTPSOnly bool `json:"https_only"` +} + +func (c *domainConfig) Valid(rootDomain string) bool { + if c.Domain == "" { + return false + } + + // TODO: better sanitize domain + domain := strings.ToLower(c.Domain) + rootDomain = "." + rootDomain + return !strings.HasSuffix(domain, rootDomain) +} + +func (c *domainsConfig) Read(group, project string) (err error) { + configFile, err := os.Open(filepath.Join(group, project, "config.json")) + if err != nil { + return err + } + defer configFile.Close() + + err = json.NewDecoder(configFile).Decode(c) + return +} diff --git a/internal/domain/domain_config_test.go b/internal/domain/domain_config_test.go new file mode 100644 index 00000000..05db0aa3 --- /dev/null +++ b/internal/domain/domain_config_test.go @@ -0,0 +1,65 @@ +package domain + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const configFile = "test-group/test-project/config.json" +const invalidConfig = `{"Domains":{}}` +const validConfig = `{"Domains":[{"Domain":"test"}]}` + +func TestDomainConfigValidness(t *testing.T) { + d := domainConfig{} + assert.False(t, d.Valid("gitlab.io")) + + d = domainConfig{Domain: "test"} + assert.True(t, d.Valid("gitlab.io")) + + d = domainConfig{Domain: "test"} + assert.True(t, d.Valid("gitlab.io")) + + d = domainConfig{Domain: "test.gitlab.io"} + assert.False(t, d.Valid("gitlab.io")) + + d = domainConfig{Domain: "test.test.gitlab.io"} + assert.False(t, d.Valid("gitlab.io")) + + d = domainConfig{Domain: "test.testgitlab.io"} + assert.True(t, d.Valid("gitlab.io")) + + d = domainConfig{Domain: "test.GitLab.Io"} + assert.False(t, d.Valid("gitlab.io")) +} + +func TestDomainConfigRead(t *testing.T) { + setUpTests() + + d := domainsConfig{} + err := d.Read("test-group", "test-project") + assert.Error(t, err) + + os.MkdirAll(filepath.Dir(configFile), 0700) + defer os.RemoveAll("test-group") + + d = domainsConfig{} + err = d.Read("test-group", "test-project") + assert.Error(t, err) + + err = ioutil.WriteFile(configFile, []byte(invalidConfig), 0600) + require.NoError(t, err) + d = domainsConfig{} + err = d.Read("test-group", "test-project") + assert.Error(t, err) + + err = ioutil.WriteFile(configFile, []byte(validConfig), 0600) + require.NoError(t, err) + d = domainsConfig{} + err = d.Read("test-group", "test-project") + require.NoError(t, err) +} diff --git a/internal/domain/domain_test.go b/internal/domain/domain_test.go new file mode 100644 index 00000000..130501d6 --- /dev/null +++ b/internal/domain/domain_test.go @@ -0,0 +1,281 @@ +package domain + +import ( + "compress/gzip" + "io/ioutil" + "mime" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitlab-pages/internal/fixture" +) + +func TestGroupServeHTTP(t *testing.T) { + setUpTests() + + testGroup := &D{ + group: "group", + projectName: "", + } + + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/", nil, "main-dir") + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/index.html", nil, "main-dir") + assert.HTTPRedirect(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project", nil) + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project", nil, + `<a href="//group.test.io/project/">Found</a>`) + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/", nil, "project-subdir") + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/index.html", nil, "project-subdir") + assert.HTTPRedirect(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/subdir", nil) + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/subdir", nil, + `<a href="//group.test.io/project/subdir/">Found</a>`) + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/subdir/", nil, "project-subsubdir") + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project2/", nil, "project2-main") + assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project2/index.html", nil, "project2-main") + assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io//about.gitlab.com/%2e%2e", nil) + assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/symlink", nil) + assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/symlink/index.html", nil) + assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/symlink/subdir/", nil) + assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/fifo", nil) + assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/not-existing-file", nil) + assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project//about.gitlab.com/%2e%2e", nil) +} + +func TestDomainServeHTTP(t *testing.T) { + setUpTests() + + testDomain := &D{ + group: "group", + projectName: "project2", + config: &domainConfig{ + Domain: "test.domain.com", + }, + } + + assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/", nil, "project2-main") + assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/index.html", nil, "project2-main") + assert.HTTPRedirect(t, testDomain.ServeHTTP, "GET", "/subdir", nil) + assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/subdir", nil, + `<a href="/subdir/">Found</a>`) + assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/subdir/", nil, "project2-subdir") + assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/subdir/index.html", nil, "project2-subdir") + assert.HTTPError(t, testDomain.ServeHTTP, "GET", "//about.gitlab.com/%2e%2e", nil) + assert.HTTPError(t, testDomain.ServeHTTP, "GET", "/not-existing-file", nil) +} + +func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, ungzip bool) { + w := httptest.NewRecorder() + req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) + require.NoError(t, err) + if acceptEncoding != "" { + req.Header.Add("Accept-Encoding", acceptEncoding) + } + handler(w, req) + + if ungzip { + reader, err := gzip.NewReader(w.Body) + require.NoError(t, err) + defer reader.Close() + + contentEncoding := w.Header().Get("Content-Encoding") + assert.Equal(t, "gzip", contentEncoding, "Content-Encoding") + + bytes, err := ioutil.ReadAll(reader) + require.NoError(t, err) + assert.Contains(t, string(bytes), str) + } else { + assert.Contains(t, w.Body.String(), str) + } +} + +func TestGroupServeHTTPGzip(t *testing.T) { + setUpTests() + + testGroup := &D{ + group: "group", + projectName: "", + } + + testSet := []struct { + mode string // HTTP mode + url string // Test URL + params url.Values // Test URL params + acceptEncoding string // Accept encoding header + body interface{} // Expected body at above URL + ungzip bool // Do we expect the request to require unzip? + }{ + // No gzip encoding requested + {"GET", "http://group.test.io/", nil, "", "main-dir", false}, + {"GET", "http://group.test.io/", nil, "identity", "main-dir", false}, + {"GET", "http://group.test.io/", nil, "gzip; q=0", "main-dir", false}, + // gzip encoding requeste}, + {"GET", "http://group.test.io/", nil, "*", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "identity, gzip", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip; q=1", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip; q=0.9", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip, deflate", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip; q=1, deflate", "main-dir", true}, + {"GET", "http://group.test.io/", nil, "gzip; q=0.9, deflate", "main-dir", true}, + // gzip encoding requested, but url does not have compressed content on disk + {"GET", "http://group.test.io/project2/", nil, "*", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "identity, gzip", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip; q=1", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip; q=0.9", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip, deflate", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip; q=1, deflate", "project2-main", false}, + {"GET", "http://group.test.io/project2/", nil, "gzip; q=0.9, deflate", "project2-main", false}, + // malformed headers + {"GET", "http://group.test.io/", nil, ";; gzip", "main-dir", false}, + {"GET", "http://group.test.io/", nil, "middle-out", "main-dir", false}, + {"GET", "http://group.test.io/", nil, "gzip; quality=1", "main-dir", false}, + // Symlinked .gz files are not supported + {"GET", "http://group.test.io/gz-symlink", nil, "*", "data", false}, + } + + for _, tt := range testSet { + testHTTPGzip(t, testGroup.ServeHTTP, tt.mode, tt.url, tt.params, tt.acceptEncoding, tt.body, tt.ungzip) + } +} + +func testHTTP404(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, str interface{}) { + w := httptest.NewRecorder() + req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) + require.NoError(t, err) + handler(w, req) + + contentType, _, _ := mime.ParseMediaType(w.Header().Get("Content-Type")) + assert.Equal(t, http.StatusNotFound, w.Code, "HTTP status") + assert.Equal(t, "text/html", contentType, "Content-Type") + assert.Contains(t, w.Body.String(), str) +} + +func TestGroup404ServeHTTP(t *testing.T) { + setUpTests() + + testGroup := &D{ + group: "group.404", + projectName: "", + } + + testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.404/not/existing-file", nil, "Custom 404 project page") + testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.404/", nil, "Custom 404 project page") + testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.no.404/not/existing-file", nil, "Custom 404 group page") + testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/not/existing-file", nil, "Custom 404 group page") + testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") + testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/", nil, "Custom 404 group page") + assert.HTTPBodyNotContains(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.404.symlink/not/existing-file", nil, "Custom 404 project page") +} + +func TestDomain404ServeHTTP(t *testing.T) { + setUpTests() + + testDomain := &D{ + group: "group.404", + projectName: "domain.404", + config: &domainConfig{ + Domain: "domain.404.com", + }, + } + + testHTTP404(t, testDomain.ServeHTTP, "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") + testHTTP404(t, testDomain.ServeHTTP, "GET", "http://group.404.test.io/", nil, "Custom 404 group page") +} + +func TestPredefined404ServeHTTP(t *testing.T) { + setUpTests() + + testDomain := &D{ + group: "group", + } + + testHTTP404(t, testDomain.ServeHTTP, "GET", "http://group.test.io/not-existing-file", nil, "The page you're looking for could not be found") +} + +func TestGroupCertificate(t *testing.T) { + testGroup := &D{ + group: "group", + projectName: "", + } + + tls, err := testGroup.EnsureCertificate() + assert.Nil(t, tls) + assert.Error(t, err) +} + +func TestDomainNoCertificate(t *testing.T) { + testDomain := &D{ + group: "group", + projectName: "project2", + config: &domainConfig{ + Domain: "test.domain.com", + }, + } + + tls, err := testDomain.EnsureCertificate() + assert.Nil(t, tls) + assert.Error(t, err) + + _, err2 := testDomain.EnsureCertificate() + assert.Error(t, err) + assert.Equal(t, err, err2) +} + +func TestDomainCertificate(t *testing.T) { + testDomain := &D{ + group: "group", + projectName: "project2", + config: &domainConfig{ + Domain: "test.domain.com", + Certificate: fixture.Certificate, + Key: fixture.Key, + }, + } + + tls, err := testDomain.EnsureCertificate() + assert.NotNil(t, tls) + require.NoError(t, err) +} + +func TestCacheControlHeaders(t *testing.T) { + testGroup := &D{group: "group"} + w := httptest.NewRecorder() + req, err := http.NewRequest("GET", "http://group.test.io/", nil) + require.NoError(t, err) + + now := time.Now() + testGroup.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "max-age=600", w.Header().Get("Cache-Control")) + + expires := w.Header().Get("Expires") + require.NotEmpty(t, expires) + + expiresTime, err := time.Parse(time.RFC1123, expires) + require.NoError(t, err) + + assert.WithinDuration(t, now.UTC().Add(10*time.Minute), expiresTime.UTC(), time.Minute) +} + +var chdirSet = false + +func setUpTests() { + if chdirSet { + return + } + + err := os.Chdir("../../shared/pages") + if err != nil { + log.WithError(err).Print("chdir") + } else { + chdirSet = true + } +} diff --git a/internal/domain/map.go b/internal/domain/map.go new file mode 100644 index 00000000..ad38e84b --- /dev/null +++ b/internal/domain/map.go @@ -0,0 +1,251 @@ +package domain + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/karrick/godirwalk" + log "github.com/sirupsen/logrus" + + "gitlab.com/gitlab-org/gitlab-pages/metrics" +) + +// Map maps domain names to D instances. +type Map map[string]*D + +type domainsUpdater func(Map) + +func (dm Map) addDomain(rootDomain, group, projectName string, config *domainConfig) { + newDomain := &D{ + group: group, + projectName: projectName, + config: config, + } + + var domainName string + domainName = strings.ToLower(config.Domain) + dm[domainName] = newDomain +} + +func (dm Map) updateGroupDomain(rootDomain, group, projectName string, httpsOnly bool) { + domainName := strings.ToLower(group + "." + rootDomain) + groupDomain := dm[domainName] + + if groupDomain == nil { + groupDomain = &D{ + group: group, + projects: make(projects), + } + } + + groupDomain.projects[projectName] = &project{ + HTTPSOnly: httpsOnly, + } + + dm[domainName] = groupDomain +} + +func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *domainsConfig) { + if config == nil { + // This is necessary to preserve the previous behaviour where a + // group domain is created even if no config.json files are + // loaded successfully. Is it safe to remove this? + dm.updateGroupDomain(rootDomain, group, projectName, false) + return + } + + dm.updateGroupDomain(rootDomain, group, projectName, config.HTTPSOnly) + + for _, domainConfig := range config.Domains { + config := domainConfig // domainConfig is reused for each loop iteration + if domainConfig.Valid(rootDomain) { + dm.addDomain(rootDomain, group, projectName, &config) + } + } +} + +func readProject(group, projectName string, fanIn chan<- jobResult) { + if strings.HasPrefix(projectName, ".") { + return + } + + // Ignore projects that have .deleted in name + if strings.HasSuffix(projectName, ".deleted") { + return + } + + if _, err := os.Lstat(filepath.Join(group, projectName, "public")); err != nil { + return + } + + // We read the config.json file _before_ fanning in, because it does disk + // IO and it does not need access to the domains map. + config := &domainsConfig{} + if err := config.Read(group, projectName); err != nil { + config = nil + } + + fanIn <- jobResult{group: group, project: projectName, config: config} +} + +func readProjects(group string, buf []byte, fanIn chan<- jobResult) { + fis, err := godirwalk.ReadDirents(group, buf) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "group": group, + }).Print("readdir failed") + return + } + + for _, project := range fis { + // Ignore non directories + if !project.IsDir() { + continue + } + + readProject(group, project.Name(), fanIn) + } +} + +type jobResult struct { + group string + project string + config *domainsConfig +} + +// ReadGroups walks the pages directory and populates dm with all the domains it finds. +func (dm Map) ReadGroups(rootDomain string) error { + fis, err := godirwalk.ReadDirents(".", nil) + if err != nil { + return err + } + + fanOutGroups := make(chan string) + fanIn := make(chan jobResult) + wg := &sync.WaitGroup{} + for i := 0; i < 4; i++ { + wg.Add(1) + + go func() { + buf := make([]byte, 2*os.Getpagesize()) + + for group := range fanOutGroups { + started := time.Now() + + readProjects(group, buf, fanIn) + + log.WithFields(log.Fields{ + "group": group, + "duration": time.Since(started).Seconds(), + }).Debug("Loaded projects for group") + } + + wg.Done() + }() + } + + go func() { + wg.Wait() + close(fanIn) + }() + + done := make(chan struct{}) + go func() { + for result := range fanIn { + dm.readProjectConfig(rootDomain, result.group, result.project, result.config) + } + + close(done) + }() + + for _, group := range fis { + if !group.IsDir() { + continue + } + if strings.HasPrefix(group.Name(), ".") { + continue + } + fanOutGroups <- group.Name() + } + close(fanOutGroups) + + <-done + return nil +} + +const ( + updateFile = ".update" +) + +// Watch polls the filesystem and kicks off a new domain directory scan when needed. +func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) { + lastUpdate := []byte("no-update") + + for { + // Read the update file + update, err := ioutil.ReadFile(updateFile) + if err != nil && !os.IsNotExist(err) { + log.WithError(err).Print("failed to read update timestamp") + time.Sleep(interval) + continue + } + + // If it's the same ignore + if bytes.Equal(lastUpdate, update) { + time.Sleep(interval) + continue + } + lastUpdate = update + + started := time.Now() + dm := make(Map) + if err := dm.ReadGroups(rootDomain); err != nil { + log.WithError(err).Warn("domain scan failed") + } + duration := time.Since(started).Seconds() + + var hash string + if len(update) < 1 { + hash = "<empty>" + } else { + hash = strings.TrimSpace(string(update)) + } + + logConfiguredDomains(dm) + + log.WithFields(log.Fields{ + "count(domains)": len(dm), + "duration": duration, + "hash": hash, + }).Info("Updated all domains") + + if updater != nil { + updater(dm) + } + + // Update prometheus metrics + metrics.DomainLastUpdateTime.Set(float64(time.Now().UTC().Unix())) + metrics.DomainsServed.Set(float64(len(dm))) + metrics.DomainUpdates.Inc() + + time.Sleep(interval) + } +} + +func logConfiguredDomains(dm Map) { + if log.GetLevel() == log.DebugLevel { + return + } + + for h, d := range dm { + log.WithFields(log.Fields{ + "domain": d, + "host": h, + }).Debug("Configured domain") + } +} diff --git a/internal/domain/map_test.go b/internal/domain/map_test.go new file mode 100644 index 00000000..f20f98bd --- /dev/null +++ b/internal/domain/map_test.go @@ -0,0 +1,148 @@ +package domain + +import ( + "crypto/rand" + "fmt" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadProjects(t *testing.T) { + setUpTests() + + dm := make(Map) + err := dm.ReadGroups("test.io") + require.NoError(t, err) + + var domains []string + for d := range dm { + domains = append(domains, d) + } + + expectedDomains := []string{ + "group.test.io", + "group.internal.test.io", + "test.domain.com", // from config.json + "other.domain.com", + "domain.404.com", + "group.404.test.io", + "group.https-only.test.io", + "test.my-domain.com", + "test2.my-domain.com", + "no.cert.com", + } + + for _, expected := range domains { + assert.Contains(t, domains, expected) + } + + for _, actual := range domains { + assert.Contains(t, expectedDomains, actual) + } + + // Check that multiple domains in the same project are recorded faithfully + exp1 := &domainConfig{Domain: "test.domain.com"} + assert.Equal(t, exp1, dm["test.domain.com"].config) + + exp2 := &domainConfig{Domain: "other.domain.com", Certificate: "test", Key: "key"} + assert.Equal(t, exp2, dm["other.domain.com"].config) +} + +// This write must be atomic, otherwise we cannot predict the state of the +// domain watcher goroutine. We cannot use ioutil.WriteFile because that +// has a race condition where the file is empty, which can get picked up +// by the domain watcher. +func writeRandomTimestamp(t *testing.T) { + b := make([]byte, 10) + n, _ := rand.Read(b) + require.True(t, n > 0, "read some random bytes") + + temp, err := ioutil.TempFile(".", "TestWatch") + require.NoError(t, err) + _, err = temp.Write(b) + require.NoError(t, err, "write to tempfile") + require.NoError(t, temp.Close(), "close tempfile") + + require.NoError(t, os.Rename(temp.Name(), updateFile), "rename tempfile") +} + +func TestWatch(t *testing.T) { + setUpTests() + + require.NoError(t, os.RemoveAll(updateFile)) + + update := make(chan Map) + go Watch("gitlab.io", func(dm Map) { + update <- dm + }, time.Microsecond*50) + + defer os.Remove(updateFile) + + domains := recvTimeout(t, update) + assert.NotNil(t, domains, "if the domains are fetched on start") + + writeRandomTimestamp(t) + domains = recvTimeout(t, update) + assert.NotNil(t, domains, "if the domains are updated after the creation") + + writeRandomTimestamp(t) + domains = recvTimeout(t, update) + assert.NotNil(t, domains, "if the domains are updated after the timestamp change") +} + +func recvTimeout(t *testing.T, ch <-chan Map) Map { + timeout := 5 * time.Second + + select { + case dm := <-ch: + return dm + case <-time.After(timeout): + t.Fatalf("timeout after %v waiting for domain update", timeout) + return nil + } +} + +func BenchmarkReadGroups(b *testing.B) { + testRoot, err := ioutil.TempDir("", "gitlab-pages-test") + require.NoError(b, err) + + cwd, err := os.Getwd() + require.NoError(b, err) + + defer func(oldWd, testWd string) { + os.Chdir(oldWd) + fmt.Printf("cleaning up test directory %s\n", testWd) + os.RemoveAll(testWd) + }(cwd, testRoot) + + require.NoError(b, os.Chdir(testRoot)) + + nGroups := 10000 + b.Logf("creating fake domains directory with %d groups", nGroups) + for i := 0; i < nGroups; i++ { + for j := 0; j < 5; j++ { + dir := fmt.Sprintf("%s/group-%d/project-%d", testRoot, i, j) + require.NoError(b, os.MkdirAll(dir+"/public", 0755)) + + fakeConfig := fmt.Sprintf(`{"Domains":[{"Domain":"foo.%d.%d.example.io","Certificate":"bar","Key":"baz"}]}`, i, j) + require.NoError(b, ioutil.WriteFile(dir+"/config.json", []byte(fakeConfig), 0644)) + } + if i%100 == 0 { + fmt.Print(".") + } + } + + b.Run("ReadGroups", func(b *testing.B) { + var dm Map + for i := 0; i < 2; i++ { + dm = make(Map) + require.NoError(b, dm.ReadGroups("example.com")) + } + b.Logf("found %d domains", len(dm)) + }) +} diff --git a/internal/fixture/fixtures.go b/internal/fixture/fixtures.go new file mode 100644 index 00000000..38bbd375 --- /dev/null +++ b/internal/fixture/fixtures.go @@ -0,0 +1,56 @@ +package fixture + +const ( + // Certificate is used for HTTPS tests + Certificate = `-----BEGIN CERTIFICATE----- +MIIDZDCCAkygAwIBAgIRAOtN9/zy+gFjdsgpKq3QRdQwDQYJKoZIhvcNAQELBQAw +MzEUMBIGA1UEChMLTG9nIENvdXJpZXIxGzAZBgNVBAMTEmdpdGxhYi1leGFtcGxl +LmNvbTAgFw0xODAzMjMxODMwMDZaGA8yMTE4MDIyNzE4MzAwNlowMzEUMBIGA1UE +ChMLTG9nIENvdXJpZXIxGzAZBgNVBAMTEmdpdGxhYi1leGFtcGxlLmNvbTCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKULsxpnazXX5RsVzayrAQB+lWwr +Wef5L5eDhSsIsBLbelYp5YB4TmVRt5x7bWKOOJSBsOfwHZHKJXdu+uuX2RenZlhk +3Qpq9XGaPZjYm/NHi8gBHPAtz5sG5VaKNvkfTzRGnO9CWA9TM1XtYiOBq94dO+H3 +c+5jP5Yw+mJ+hA+i2058zF8nRlUHArEno2ofrHwE0LMZ11VskpXtWnVfs3voLs8p +r76KXPBFkMJR4qkWrMDF5Y5MbsQ0zisn6KXrTyV0S4MQh4vSyPdFHnEzvJ07rm5x +4RTWrjgQeQ2DjZjQvRmaDzlVBK9kaMkJ1Si3agK+gpji6d6WZ/Mb2el1GK8CAwEA +AaNxMG8wDgYDVR0PAQH/BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1Ud +EwEB/wQFMAMBAf8wNwYDVR0RBDAwLoIUKi5naXRsYWItZXhhbXBsZS5jb22HBH8A +AAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggEBAJ0NM8apK0xI +YxMstP/dCQXtR0wyREGSD/eOpeY3bWlqCbpRgMFUGjQlrsEozcPZOCSCKX5p+tym +7GsnYtXkwbsuURoSz+5IlhRPVHcUlUeGRdv3/gCd8fDXiigALCsB6GrkMG5cUfh+ +x5p52AC3eQdWTDoxNou+2gzwkAl8iJc13Ykusst0YUqcsXKqTuei2quxFv0pEBSO +p8wEixoicLFNqPnIDmgx5894DAn0bccNXgRWtq8lLbdhGUlBbpatevvFMgNvFUbe +eeGb9D0EfpxmzxUl+L0xZtfg3f7cu5AgLG8tb6l4AK6NPVuXN8DmUgvnauWJjZME +fgStI+IRNVg= +-----END CERTIFICATE----- +` + + // Key is used for HTTPS tests + Key = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEApQuzGmdrNdflGxXNrKsBAH6VbCtZ5/kvl4OFKwiwEtt6Vinl +gHhOZVG3nHttYo44lIGw5/Adkcold27665fZF6dmWGTdCmr1cZo9mNib80eLyAEc +8C3PmwblVoo2+R9PNEac70JYD1MzVe1iI4Gr3h074fdz7mM/ljD6Yn6ED6LbTnzM +XydGVQcCsSejah+sfATQsxnXVWySle1adV+ze+guzymvvopc8EWQwlHiqRaswMXl +jkxuxDTOKyfopetPJXRLgxCHi9LI90UecTO8nTuubnHhFNauOBB5DYONmNC9GZoP +OVUEr2RoyQnVKLdqAr6CmOLp3pZn8xvZ6XUYrwIDAQABAoIBAHhP5QnUZeTkMtDh +vgKmzZ4sqIQnvexKTBUo/MR4GtJESBPTisdx68QUI8LgfsafYkNvnyQUd5m1QEam +Eif3k3uYvhSlwjQ78BwWEdz/2f8oIo9zsEKtQm+CQWAqdRR5bGVxLCmFtWfGgN+c +ojO77SuHKAX7OvmGQ+4aWgu+qkoyg/chIpPXMduAjLMtN3eg60ZqJ5KrKuIF63Bb +xkPQvzJueB9SfUurmKjUltDMx6G/9RZyS0OIRGyL9Qp8MZ8jE23cXOcDgm0HhkPq +W4LU++aWAOLYziTjnhjJ+4Iz9R7U8sCmk1wgnK/tapVcJf41R98WuGluyjXpsXgA +k7vmofECgYEAzuGun9lZ7xGwPifp6vGanWXnW+JiZgCTGuHWgQLIXWYcLfoI3kpH +eLBYINBwvjIQ7P6UxsGSSXd+T82t+8W2LLc2fiKFKE1LVySpH99+cfmIPXxrviOz +GBX9LTdSCdGkgb54m8aJCpNFnKw5wYgcW1L8CaXXly2Z/KNrGR9R/YUCgYEAzDs4 +19HqlutGLTC30/ziiiIfDaBbX9AzBdUfp9GdT53Mi/7bfxpW/sL4RjG2fGgmN6ua +fh5npT9AB1ldcEg2qfyOJPt1Ubdi6ek9lx8AB2RMhwdihgX+7bjVMFtjg4b8z5C1 +jQbEr1rhFdpaGyNehtAXDgCbDWQBYnBrmM0rCaMCgYBip1Qyfd9ZFcJJoZb2pofo +jvOo6Weq5JNBungjxUPu5gaCFj2sYxd6Af3EiCF7UTypBy3DKgOsbQMa4yYYbcvV +vviJZcTB1zoaMC1GObl+eFPzniVy4mtBDRtSOJMyg3pDNKUnA6HOHTSQ5cAU/ecn +1YbCwwbv3JsV0of7zue2UQKBgQCVc0j3dd9rLSQfcaUz9bx5RNrgh9YV2S9dN0aA +8f1iA6FpWMiazFWY/GfeRga6JyTAXE0juXAzFoPuXNDpl46Y+f2yxmhlsgMqFMpD +SiYlQppVvWu1k7GnmDg5uMarux5JbiXM24UWpTRNX4nMjidgE+qrDnpoZCQ3Ovkh +yhGSbQKBgD3VEnPiSUmXBo39kPcnPg93E3JfdAOiOwIB2qwfYzg9kpmuTWws+DFz +lKpMI27YkmnPqROQ2NTUfdxYmw3EHHMAsvnmHeMNGn3ijSUZVKmPfV436Qc8iVci +s4wKoCRhBUZ52sHki/ieb+5hycT3JnVXMDtbJxgXFW5a86usXEpO +-----END RSA PRIVATE KEY-----` +) |