Put domain code in a separate package

7 files changed, 1165 insertions, 0 deletions

diff --git a/internal/domain/domain.go b/internal/domain/domain.go
new file mode 100644
index 00000000..0333cebe
--- /dev/null
+++ b/internal/domain/domain.go
@@ -0,0 +1,322 @@
+package domain
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/httperrors"
+	"gitlab.com/gitlab-org/gitlab-pages/internal/httputil"
+)
+
+type locationDirectoryError struct {
+	FullPath     string
+	RelativePath string
+}
+
+type project struct {
+	HTTPSOnly bool
+}
+
+type projects map[string]*project
+
+// D is a domain that gitlab-pages can serve.
+type D struct {
+	group string
+
+	// custom domains:
+	projectName      string
+	config           *domainConfig
+	certificate      *tls.Certificate
+	certificateError error
+
+	// group domains:
+	projects projects
+}
+
+// String implements Stringer.
+func (d *D) String() string {
+	if d.group != "" && d.projectName != "" {
+		return d.group + "/" + d.projectName
+	}
+
+	if d.group != "" {
+		return d.group
+	}
+
+	return d.projectName
+}
+
+func (l *locationDirectoryError) Error() string {
+	return "location error accessing directory where file expected"
+}
+
+func acceptsGZip(r *http.Request) bool {
+	if r.Header.Get("Range") != "" {
+		return false
+	}
+
+	offers := []string{"gzip", "identity"}
+	acceptedEncoding := httputil.NegotiateContentEncoding(r, offers)
+	return acceptedEncoding == "gzip"
+}
+
+func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string {
+	if !acceptsGZip(r) {
+		return fullPath
+	}
+
+	gzipPath := fullPath + ".gz"
+
+	// Ensure the .gz file is not a symlink
+	if fi, err := os.Lstat(gzipPath); err != nil || !fi.Mode().IsRegular() {
+		return fullPath
+	}
+
+	w.Header().Set("Content-Encoding", "gzip")
+
+	return gzipPath
+}
+
+func setContentType(w http.ResponseWriter, fullPath string) {
+	ext := filepath.Ext(fullPath)
+	ctype := mime.TypeByExtension(ext)
+	if ctype != "" {
+		w.Header().Set("Content-Type", ctype)
+	}
+}
+
+// IsHTTPSOnly figures out if the request should be handled with HTTPS
+// only by looking at group and project level config.
+func (d *D) IsHTTPSOnly(r *http.Request) bool {
+	if d.config != nil {
+		return d.config.HTTPSOnly
+	}
+
+	split := strings.SplitN(r.URL.Path, "/", 3)
+	if len(split) < 2 {
+		return false
+	}
+
+	project := d.projects[split[1]]
+
+	if project != nil {
+		return project.HTTPSOnly
+	}
+
+	return false
+}
+
+func (d *D) serveFile(w http.ResponseWriter, r *http.Request, origPath string) error {
+	fullPath := handleGZip(w, r, origPath)
+
+	file, err := os.Open(fullPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	// Set caching headers
+	w.Header().Set("Cache-Control", "max-age=600")
+	w.Header().Set("Expires", time.Now().Add(10*time.Minute).Format(time.RFC1123))
+
+	// ServeContent sets Content-Type for us
+	http.ServeContent(w, r, origPath, fi.ModTime(), file)
+	return nil
+}
+
+func (d *D) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, origPath string) error {
+	fullPath := handleGZip(w, r, origPath)
+
+	// Open and serve content of file
+	file, err := os.Open(fullPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	setContentType(w, origPath)
+	w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10))
+	w.WriteHeader(code)
+
+	if r.Method != "HEAD" {
+		_, err := io.CopyN(w, file, fi.Size())
+		return err
+	}
+
+	return nil
+}
+
+// Resolve the HTTP request to a path on disk, converting requests for
+// directories to requests for index.html inside the directory if appropriate.
+func (d *D) resolvePath(projectName string, subPath ...string) (string, error) {
+	publicPath := filepath.Join(d.group, projectName, "public")
+
+	// Don't use filepath.Join as cleans the path,
+	// where we want to traverse full path as supplied by user
+	// (including ..)
+	testPath := publicPath + "/" + strings.Join(subPath, "/")
+	fullPath, err := filepath.EvalSymlinks(testPath)
+	if err != nil {
+		return "", err
+	}
+
+	// The requested path resolved to somewhere outside of the public/ directory
+	if !strings.HasPrefix(fullPath, publicPath+"/") && fullPath != publicPath {
+		return "", fmt.Errorf("%q should be in %q", fullPath, publicPath)
+	}
+
+	fi, err := os.Lstat(fullPath)
+	if err != nil {
+		return "", err
+	}
+
+	// The requested path is a directory, so try index.html via recursion
+	if fi.IsDir() {
+		return "", &locationDirectoryError{
+			FullPath:     fullPath,
+			RelativePath: strings.TrimPrefix(fullPath, publicPath),
+		}
+	}
+
+	// The file exists, but is not a supported type to serve. Perhaps a block
+	// special device or something else that may be a security risk.
+	if !fi.Mode().IsRegular() {
+		return "", fmt.Errorf("%s: is not a regular file", fullPath)
+	}
+
+	return fullPath, nil
+}
+
+func (d *D) tryNotFound(w http.ResponseWriter, r *http.Request, projectName string) error {
+	page404, err := d.resolvePath(projectName, "404.html")
+	if err != nil {
+		return err
+	}
+
+	err = d.serveCustomFile(w, r, http.StatusNotFound, page404)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *D) tryFile(w http.ResponseWriter, r *http.Request, projectName, pathSuffix string, subPath ...string) error {
+	fullPath, err := d.resolvePath(projectName, subPath...)
+
+	if locationError, _ := err.(*locationDirectoryError); locationError != nil {
+		if endsWithSlash(r.URL.Path) {
+			fullPath, err = d.resolvePath(projectName, filepath.Join(subPath...), "index.html")
+		} else {
+			redirectPath := "//" + r.Host + "/"
+			if pathSuffix != "" {
+				redirectPath += pathSuffix + "/"
+			}
+			if locationError.RelativePath != "" {
+				redirectPath += strings.TrimPrefix(locationError.RelativePath, "/") + "/"
+			}
+			http.Redirect(w, r, redirectPath, 302)
+			return nil
+		}
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return d.serveFile(w, r, fullPath)
+}
+
+func (d *D) serveFromGroup(w http.ResponseWriter, r *http.Request) {
+	// The Path always contains "/" at the beginning
+	split := strings.SplitN(r.URL.Path, "/", 3)
+
+	// Try to serve file for http://group.example.com/subpath/... => /group/subpath/...
+	if len(split) >= 2 && d.tryFile(w, r, split[1], split[1], split[2:]...) == nil {
+		return
+	}
+
+	// Try to serve file for http://group.example.com/... => /group/group.example.com/...
+	if r.Host != "" && d.tryFile(w, r, strings.ToLower(r.Host), "", r.URL.Path) == nil {
+		return
+	}
+
+	// Try serving not found page for http://group.example.com/subpath/ => /group/subpath/404.html
+	if len(split) >= 2 && d.tryNotFound(w, r, split[1]) == nil {
+		return
+	}
+
+	// Try serving not found page for http://group.example.com/ => /group/group.example.com/404.html
+	if r.Host != "" && d.tryNotFound(w, r, strings.ToLower(r.Host)) == nil {
+		return
+	}
+
+	// Serve generic not found
+	httperrors.Serve404(w)
+}
+
+func (d *D) serveFromConfig(w http.ResponseWriter, r *http.Request) {
+	// Try to serve file for http://host/... => /group/project/...
+	if d.tryFile(w, r, d.projectName, "", r.URL.Path) == nil {
+		return
+	}
+
+	// Try serving not found page for http://host/ => /group/project/404.html
+	if d.tryNotFound(w, r, d.projectName) == nil {
+		return
+	}
+
+	// Serve generic not found
+	httperrors.Serve404(w)
+}
+
+// EnsureCertificate parses the PEM-encoded certificate for the domain
+func (d *D) EnsureCertificate() (*tls.Certificate, error) {
+	if d.config == nil {
+		return nil, errors.New("tls certificates can be loaded only for pages with configuration")
+	}
+
+	if d.certificate != nil || d.certificateError != nil {
+		return d.certificate, d.certificateError
+	}
+
+	tls, err := tls.X509KeyPair([]byte(d.config.Certificate), []byte(d.config.Key))
+	if err != nil {
+		d.certificateError = err
+		return nil, err
+	}
+
+	d.certificate = &tls
+	return d.certificate, nil
+}
+
+// ServeHTTP implements http.Handler.
+func (d *D) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if d.config != nil {
+		d.serveFromConfig(w, r)
+	} else {
+		d.serveFromGroup(w, r)
+	}
+}
+
+func endsWithSlash(path string) bool {
+	return strings.HasSuffix(path, "/")
+}
diff --git a/internal/domain/domain_config.go b/internal/domain/domain_config.go
new file mode 100644
index 00000000..ed7e0820
--- /dev/null
+++ b/internal/domain/domain_config.go
@@ -0,0 +1,42 @@
+package domain
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type domainConfig struct {
+	Domain      string
+	Certificate string
+	Key         string
+	HTTPSOnly   bool `json:"https_only"`
+}
+
+type domainsConfig struct {
+	Domains   []domainConfig
+	HTTPSOnly bool `json:"https_only"`
+}
+
+func (c *domainConfig) Valid(rootDomain string) bool {
+	if c.Domain == "" {
+		return false
+	}
+
+	// TODO: better sanitize domain
+	domain := strings.ToLower(c.Domain)
+	rootDomain = "." + rootDomain
+	return !strings.HasSuffix(domain, rootDomain)
+}
+
+func (c *domainsConfig) Read(group, project string) (err error) {
+	configFile, err := os.Open(filepath.Join(group, project, "config.json"))
+	if err != nil {
+		return err
+	}
+	defer configFile.Close()
+
+	err = json.NewDecoder(configFile).Decode(c)
+	return
+}
diff --git a/internal/domain/domain_config_test.go b/internal/domain/domain_config_test.go
new file mode 100644
index 00000000..05db0aa3
--- /dev/null
+++ b/internal/domain/domain_config_test.go
@@ -0,0 +1,65 @@
+package domain
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const configFile = "test-group/test-project/config.json"
+const invalidConfig = `{"Domains":{}}`
+const validConfig = `{"Domains":[{"Domain":"test"}]}`
+
+func TestDomainConfigValidness(t *testing.T) {
+	d := domainConfig{}
+	assert.False(t, d.Valid("gitlab.io"))
+
+	d = domainConfig{Domain: "test"}
+	assert.True(t, d.Valid("gitlab.io"))
+
+	d = domainConfig{Domain: "test"}
+	assert.True(t, d.Valid("gitlab.io"))
+
+	d = domainConfig{Domain: "test.gitlab.io"}
+	assert.False(t, d.Valid("gitlab.io"))
+
+	d = domainConfig{Domain: "test.test.gitlab.io"}
+	assert.False(t, d.Valid("gitlab.io"))
+
+	d = domainConfig{Domain: "test.testgitlab.io"}
+	assert.True(t, d.Valid("gitlab.io"))
+
+	d = domainConfig{Domain: "test.GitLab.Io"}
+	assert.False(t, d.Valid("gitlab.io"))
+}
+
+func TestDomainConfigRead(t *testing.T) {
+	setUpTests()
+
+	d := domainsConfig{}
+	err := d.Read("test-group", "test-project")
+	assert.Error(t, err)
+
+	os.MkdirAll(filepath.Dir(configFile), 0700)
+	defer os.RemoveAll("test-group")
+
+	d = domainsConfig{}
+	err = d.Read("test-group", "test-project")
+	assert.Error(t, err)
+
+	err = ioutil.WriteFile(configFile, []byte(invalidConfig), 0600)
+	require.NoError(t, err)
+	d = domainsConfig{}
+	err = d.Read("test-group", "test-project")
+	assert.Error(t, err)
+
+	err = ioutil.WriteFile(configFile, []byte(validConfig), 0600)
+	require.NoError(t, err)
+	d = domainsConfig{}
+	err = d.Read("test-group", "test-project")
+	require.NoError(t, err)
+}
diff --git a/internal/domain/domain_test.go b/internal/domain/domain_test.go
new file mode 100644
index 00000000..130501d6
--- /dev/null
+++ b/internal/domain/domain_test.go
@@ -0,0 +1,281 @@
+package domain
+
+import (
+	"compress/gzip"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitlab-pages/internal/fixture"
+)
+
+func TestGroupServeHTTP(t *testing.T) {
+	setUpTests()
+
+	testGroup := &D{
+		group:       "group",
+		projectName: "",
+	}
+
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/", nil, "main-dir")
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/index.html", nil, "main-dir")
+	assert.HTTPRedirect(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project", nil)
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project", nil,
+		`<a href="//group.test.io/project/">Found</a>`)
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/", nil, "project-subdir")
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/index.html", nil, "project-subdir")
+	assert.HTTPRedirect(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/subdir", nil)
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/subdir", nil,
+		`<a href="//group.test.io/project/subdir/">Found</a>`)
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/subdir/", nil, "project-subsubdir")
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project2/", nil, "project2-main")
+	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project2/index.html", nil, "project2-main")
+	assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io//about.gitlab.com/%2e%2e", nil)
+	assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/symlink", nil)
+	assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/symlink/index.html", nil)
+	assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/symlink/subdir/", nil)
+	assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project/fifo", nil)
+	assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/not-existing-file", nil)
+	assert.HTTPError(t, testGroup.ServeHTTP, "GET", "http://group.test.io/project//about.gitlab.com/%2e%2e", nil)
+}
+
+func TestDomainServeHTTP(t *testing.T) {
+	setUpTests()
+
+	testDomain := &D{
+		group:       "group",
+		projectName: "project2",
+		config: &domainConfig{
+			Domain: "test.domain.com",
+		},
+	}
+
+	assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/", nil, "project2-main")
+	assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/index.html", nil, "project2-main")
+	assert.HTTPRedirect(t, testDomain.ServeHTTP, "GET", "/subdir", nil)
+	assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/subdir", nil,
+		`<a href="/subdir/">Found</a>`)
+	assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/subdir/", nil, "project2-subdir")
+	assert.HTTPBodyContains(t, testDomain.ServeHTTP, "GET", "/subdir/index.html", nil, "project2-subdir")
+	assert.HTTPError(t, testDomain.ServeHTTP, "GET", "//about.gitlab.com/%2e%2e", nil)
+	assert.HTTPError(t, testDomain.ServeHTTP, "GET", "/not-existing-file", nil)
+}
+
+func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, ungzip bool) {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil)
+	require.NoError(t, err)
+	if acceptEncoding != "" {
+		req.Header.Add("Accept-Encoding", acceptEncoding)
+	}
+	handler(w, req)
+
+	if ungzip {
+		reader, err := gzip.NewReader(w.Body)
+		require.NoError(t, err)
+		defer reader.Close()
+
+		contentEncoding := w.Header().Get("Content-Encoding")
+		assert.Equal(t, "gzip", contentEncoding, "Content-Encoding")
+
+		bytes, err := ioutil.ReadAll(reader)
+		require.NoError(t, err)
+		assert.Contains(t, string(bytes), str)
+	} else {
+		assert.Contains(t, w.Body.String(), str)
+	}
+}
+
+func TestGroupServeHTTPGzip(t *testing.T) {
+	setUpTests()
+
+	testGroup := &D{
+		group:       "group",
+		projectName: "",
+	}
+
+	testSet := []struct {
+		mode           string      // HTTP mode
+		url            string      // Test URL
+		params         url.Values  // Test URL params
+		acceptEncoding string      // Accept encoding header
+		body           interface{} // Expected body at above URL
+		ungzip         bool        // Do we expect the request to require unzip?
+	}{
+		// No gzip encoding requested
+		{"GET", "http://group.test.io/", nil, "", "main-dir", false},
+		{"GET", "http://group.test.io/", nil, "identity", "main-dir", false},
+		{"GET", "http://group.test.io/", nil, "gzip; q=0", "main-dir", false},
+		// gzip encoding requeste},
+		{"GET", "http://group.test.io/", nil, "*", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "identity, gzip", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip; q=1", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip; q=0.9", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip, deflate", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip; q=1, deflate", "main-dir", true},
+		{"GET", "http://group.test.io/", nil, "gzip; q=0.9, deflate", "main-dir", true},
+		// gzip encoding requested, but url does not have compressed content on disk
+		{"GET", "http://group.test.io/project2/", nil, "*", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "identity, gzip", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip; q=1", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip; q=0.9", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip, deflate", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip; q=1, deflate", "project2-main", false},
+		{"GET", "http://group.test.io/project2/", nil, "gzip; q=0.9, deflate", "project2-main", false},
+		// malformed headers
+		{"GET", "http://group.test.io/", nil, ";; gzip", "main-dir", false},
+		{"GET", "http://group.test.io/", nil, "middle-out", "main-dir", false},
+		{"GET", "http://group.test.io/", nil, "gzip; quality=1", "main-dir", false},
+		// Symlinked .gz files are not supported
+		{"GET", "http://group.test.io/gz-symlink", nil, "*", "data", false},
+	}
+
+	for _, tt := range testSet {
+		testHTTPGzip(t, testGroup.ServeHTTP, tt.mode, tt.url, tt.params, tt.acceptEncoding, tt.body, tt.ungzip)
+	}
+}
+
+func testHTTP404(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, str interface{}) {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil)
+	require.NoError(t, err)
+	handler(w, req)
+
+	contentType, _, _ := mime.ParseMediaType(w.Header().Get("Content-Type"))
+	assert.Equal(t, http.StatusNotFound, w.Code, "HTTP status")
+	assert.Equal(t, "text/html", contentType, "Content-Type")
+	assert.Contains(t, w.Body.String(), str)
+}
+
+func TestGroup404ServeHTTP(t *testing.T) {
+	setUpTests()
+
+	testGroup := &D{
+		group:       "group.404",
+		projectName: "",
+	}
+
+	testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.404/not/existing-file", nil, "Custom 404 project page")
+	testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.404/", nil, "Custom 404 project page")
+	testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.no.404/not/existing-file", nil, "Custom 404 group page")
+	testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/not/existing-file", nil, "Custom 404 group page")
+	testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page")
+	testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/", nil, "Custom 404 group page")
+	assert.HTTPBodyNotContains(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.404.symlink/not/existing-file", nil, "Custom 404 project page")
+}
+
+func TestDomain404ServeHTTP(t *testing.T) {
+	setUpTests()
+
+	testDomain := &D{
+		group:       "group.404",
+		projectName: "domain.404",
+		config: &domainConfig{
+			Domain: "domain.404.com",
+		},
+	}
+
+	testHTTP404(t, testDomain.ServeHTTP, "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page")
+	testHTTP404(t, testDomain.ServeHTTP, "GET", "http://group.404.test.io/", nil, "Custom 404 group page")
+}
+
+func TestPredefined404ServeHTTP(t *testing.T) {
+	setUpTests()
+
+	testDomain := &D{
+		group: "group",
+	}
+
+	testHTTP404(t, testDomain.ServeHTTP, "GET", "http://group.test.io/not-existing-file", nil, "The page you're looking for could not be found")
+}
+
+func TestGroupCertificate(t *testing.T) {
+	testGroup := &D{
+		group:       "group",
+		projectName: "",
+	}
+
+	tls, err := testGroup.EnsureCertificate()
+	assert.Nil(t, tls)
+	assert.Error(t, err)
+}
+
+func TestDomainNoCertificate(t *testing.T) {
+	testDomain := &D{
+		group:       "group",
+		projectName: "project2",
+		config: &domainConfig{
+			Domain: "test.domain.com",
+		},
+	}
+
+	tls, err := testDomain.EnsureCertificate()
+	assert.Nil(t, tls)
+	assert.Error(t, err)
+
+	_, err2 := testDomain.EnsureCertificate()
+	assert.Error(t, err)
+	assert.Equal(t, err, err2)
+}
+
+func TestDomainCertificate(t *testing.T) {
+	testDomain := &D{
+		group:       "group",
+		projectName: "project2",
+		config: &domainConfig{
+			Domain:      "test.domain.com",
+			Certificate: fixture.Certificate,
+			Key:         fixture.Key,
+		},
+	}
+
+	tls, err := testDomain.EnsureCertificate()
+	assert.NotNil(t, tls)
+	require.NoError(t, err)
+}
+
+func TestCacheControlHeaders(t *testing.T) {
+	testGroup := &D{group: "group"}
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest("GET", "http://group.test.io/", nil)
+	require.NoError(t, err)
+
+	now := time.Now()
+	testGroup.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+	assert.Equal(t, "max-age=600", w.Header().Get("Cache-Control"))
+
+	expires := w.Header().Get("Expires")
+	require.NotEmpty(t, expires)
+
+	expiresTime, err := time.Parse(time.RFC1123, expires)
+	require.NoError(t, err)
+
+	assert.WithinDuration(t, now.UTC().Add(10*time.Minute), expiresTime.UTC(), time.Minute)
+}
+
+var chdirSet = false
+
+func setUpTests() {
+	if chdirSet {
+		return
+	}
+
+	err := os.Chdir("../../shared/pages")
+	if err != nil {
+		log.WithError(err).Print("chdir")
+	} else {
+		chdirSet = true
+	}
+}
diff --git a/internal/domain/map.go b/internal/domain/map.go
new file mode 100644
index 00000000..ad38e84b
--- /dev/null
+++ b/internal/domain/map.go
@@ -0,0 +1,251 @@
+package domain
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/karrick/godirwalk"
+	log "github.com/sirupsen/logrus"
+
+	"gitlab.com/gitlab-org/gitlab-pages/metrics"
+)
+
+// Map maps domain names to D instances.
+type Map map[string]*D
+
+type domainsUpdater func(Map)
+
+func (dm Map) addDomain(rootDomain, group, projectName string, config *domainConfig) {
+	newDomain := &D{
+		group:       group,
+		projectName: projectName,
+		config:      config,
+	}
+
+	var domainName string
+	domainName = strings.ToLower(config.Domain)
+	dm[domainName] = newDomain
+}
+
+func (dm Map) updateGroupDomain(rootDomain, group, projectName string, httpsOnly bool) {
+	domainName := strings.ToLower(group + "." + rootDomain)
+	groupDomain := dm[domainName]
+
+	if groupDomain == nil {
+		groupDomain = &D{
+			group:    group,
+			projects: make(projects),
+		}
+	}
+
+	groupDomain.projects[projectName] = &project{
+		HTTPSOnly: httpsOnly,
+	}
+
+	dm[domainName] = groupDomain
+}
+
+func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *domainsConfig) {
+	if config == nil {
+		// This is necessary to preserve the previous behaviour where a
+		// group domain is created even if no config.json files are
+		// loaded successfully. Is it safe to remove this?
+		dm.updateGroupDomain(rootDomain, group, projectName, false)
+		return
+	}
+
+	dm.updateGroupDomain(rootDomain, group, projectName, config.HTTPSOnly)
+
+	for _, domainConfig := range config.Domains {
+		config := domainConfig // domainConfig is reused for each loop iteration
+		if domainConfig.Valid(rootDomain) {
+			dm.addDomain(rootDomain, group, projectName, &config)
+		}
+	}
+}
+
+func readProject(group, projectName string, fanIn chan<- jobResult) {
+	if strings.HasPrefix(projectName, ".") {
+		return
+	}
+
+	// Ignore projects that have .deleted in name
+	if strings.HasSuffix(projectName, ".deleted") {
+		return
+	}
+
+	if _, err := os.Lstat(filepath.Join(group, projectName, "public")); err != nil {
+		return
+	}
+
+	// We read the config.json file _before_ fanning in, because it does disk
+	// IO and it does not need access to the domains map.
+	config := &domainsConfig{}
+	if err := config.Read(group, projectName); err != nil {
+		config = nil
+	}
+
+	fanIn <- jobResult{group: group, project: projectName, config: config}
+}
+
+func readProjects(group string, buf []byte, fanIn chan<- jobResult) {
+	fis, err := godirwalk.ReadDirents(group, buf)
+	if err != nil {
+		log.WithError(err).WithFields(log.Fields{
+			"group": group,
+		}).Print("readdir failed")
+		return
+	}
+
+	for _, project := range fis {
+		// Ignore non directories
+		if !project.IsDir() {
+			continue
+		}
+
+		readProject(group, project.Name(), fanIn)
+	}
+}
+
+type jobResult struct {
+	group   string
+	project string
+	config  *domainsConfig
+}
+
+// ReadGroups walks the pages directory and populates dm with all the domains it finds.
+func (dm Map) ReadGroups(rootDomain string) error {
+	fis, err := godirwalk.ReadDirents(".", nil)
+	if err != nil {
+		return err
+	}
+
+	fanOutGroups := make(chan string)
+	fanIn := make(chan jobResult)
+	wg := &sync.WaitGroup{}
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+
+		go func() {
+			buf := make([]byte, 2*os.Getpagesize())
+
+			for group := range fanOutGroups {
+				started := time.Now()
+
+				readProjects(group, buf, fanIn)
+
+				log.WithFields(log.Fields{
+					"group":    group,
+					"duration": time.Since(started).Seconds(),
+				}).Debug("Loaded projects for group")
+			}
+
+			wg.Done()
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		close(fanIn)
+	}()
+
+	done := make(chan struct{})
+	go func() {
+		for result := range fanIn {
+			dm.readProjectConfig(rootDomain, result.group, result.project, result.config)
+		}
+
+		close(done)
+	}()
+
+	for _, group := range fis {
+		if !group.IsDir() {
+			continue
+		}
+		if strings.HasPrefix(group.Name(), ".") {
+			continue
+		}
+		fanOutGroups <- group.Name()
+	}
+	close(fanOutGroups)
+
+	<-done
+	return nil
+}
+
+const (
+	updateFile = ".update"
+)
+
+// Watch polls the filesystem and kicks off a new domain directory scan when needed.
+func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) {
+	lastUpdate := []byte("no-update")
+
+	for {
+		// Read the update file
+		update, err := ioutil.ReadFile(updateFile)
+		if err != nil && !os.IsNotExist(err) {
+			log.WithError(err).Print("failed to read update timestamp")
+			time.Sleep(interval)
+			continue
+		}
+
+		// If it's the same ignore
+		if bytes.Equal(lastUpdate, update) {
+			time.Sleep(interval)
+			continue
+		}
+		lastUpdate = update
+
+		started := time.Now()
+		dm := make(Map)
+		if err := dm.ReadGroups(rootDomain); err != nil {
+			log.WithError(err).Warn("domain scan failed")
+		}
+		duration := time.Since(started).Seconds()
+
+		var hash string
+		if len(update) < 1 {
+			hash = "<empty>"
+		} else {
+			hash = strings.TrimSpace(string(update))
+		}
+
+		logConfiguredDomains(dm)
+
+		log.WithFields(log.Fields{
+			"count(domains)": len(dm),
+			"duration":       duration,
+			"hash":           hash,
+		}).Info("Updated all domains")
+
+		if updater != nil {
+			updater(dm)
+		}
+
+		// Update prometheus metrics
+		metrics.DomainLastUpdateTime.Set(float64(time.Now().UTC().Unix()))
+		metrics.DomainsServed.Set(float64(len(dm)))
+		metrics.DomainUpdates.Inc()
+
+		time.Sleep(interval)
+	}
+}
+
+func logConfiguredDomains(dm Map) {
+	if log.GetLevel() == log.DebugLevel {
+		return
+	}
+
+	for h, d := range dm {
+		log.WithFields(log.Fields{
+			"domain": d,
+			"host":   h,
+		}).Debug("Configured domain")
+	}
+}
diff --git a/internal/domain/map_test.go b/internal/domain/map_test.go
new file mode 100644
index 00000000..f20f98bd
--- /dev/null
+++ b/internal/domain/map_test.go
@@ -0,0 +1,148 @@
+package domain
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadProjects(t *testing.T) {
+	setUpTests()
+
+	dm := make(Map)
+	err := dm.ReadGroups("test.io")
+	require.NoError(t, err)
+
+	var domains []string
+	for d := range dm {
+		domains = append(domains, d)
+	}
+
+	expectedDomains := []string{
+		"group.test.io",
+		"group.internal.test.io",
+		"test.domain.com", // from config.json
+		"other.domain.com",
+		"domain.404.com",
+		"group.404.test.io",
+		"group.https-only.test.io",
+		"test.my-domain.com",
+		"test2.my-domain.com",
+		"no.cert.com",
+	}
+
+	for _, expected := range domains {
+		assert.Contains(t, domains, expected)
+	}
+
+	for _, actual := range domains {
+		assert.Contains(t, expectedDomains, actual)
+	}
+
+	// Check that multiple domains in the same project are recorded faithfully
+	exp1 := &domainConfig{Domain: "test.domain.com"}
+	assert.Equal(t, exp1, dm["test.domain.com"].config)
+
+	exp2 := &domainConfig{Domain: "other.domain.com", Certificate: "test", Key: "key"}
+	assert.Equal(t, exp2, dm["other.domain.com"].config)
+}
+
+// This write must be atomic, otherwise we cannot predict the state of the
+// domain watcher goroutine. We cannot use ioutil.WriteFile because that
+// has a race condition where the file is empty, which can get picked up
+// by the domain watcher.
+func writeRandomTimestamp(t *testing.T) {
+	b := make([]byte, 10)
+	n, _ := rand.Read(b)
+	require.True(t, n > 0, "read some random bytes")
+
+	temp, err := ioutil.TempFile(".", "TestWatch")
+	require.NoError(t, err)
+	_, err = temp.Write(b)
+	require.NoError(t, err, "write to tempfile")
+	require.NoError(t, temp.Close(), "close tempfile")
+
+	require.NoError(t, os.Rename(temp.Name(), updateFile), "rename tempfile")
+}
+
+func TestWatch(t *testing.T) {
+	setUpTests()
+
+	require.NoError(t, os.RemoveAll(updateFile))
+
+	update := make(chan Map)
+	go Watch("gitlab.io", func(dm Map) {
+		update <- dm
+	}, time.Microsecond*50)
+
+	defer os.Remove(updateFile)
+
+	domains := recvTimeout(t, update)
+	assert.NotNil(t, domains, "if the domains are fetched on start")
+
+	writeRandomTimestamp(t)
+	domains = recvTimeout(t, update)
+	assert.NotNil(t, domains, "if the domains are updated after the creation")
+
+	writeRandomTimestamp(t)
+	domains = recvTimeout(t, update)
+	assert.NotNil(t, domains, "if the domains are updated after the timestamp change")
+}
+
+func recvTimeout(t *testing.T, ch <-chan Map) Map {
+	timeout := 5 * time.Second
+
+	select {
+	case dm := <-ch:
+		return dm
+	case <-time.After(timeout):
+		t.Fatalf("timeout after %v waiting for domain update", timeout)
+		return nil
+	}
+}
+
+func BenchmarkReadGroups(b *testing.B) {
+	testRoot, err := ioutil.TempDir("", "gitlab-pages-test")
+	require.NoError(b, err)
+
+	cwd, err := os.Getwd()
+	require.NoError(b, err)
+
+	defer func(oldWd, testWd string) {
+		os.Chdir(oldWd)
+		fmt.Printf("cleaning up test directory %s\n", testWd)
+		os.RemoveAll(testWd)
+	}(cwd, testRoot)
+
+	require.NoError(b, os.Chdir(testRoot))
+
+	nGroups := 10000
+	b.Logf("creating fake domains directory with %d groups", nGroups)
+	for i := 0; i < nGroups; i++ {
+		for j := 0; j < 5; j++ {
+			dir := fmt.Sprintf("%s/group-%d/project-%d", testRoot, i, j)
+			require.NoError(b, os.MkdirAll(dir+"/public", 0755))
+
+			fakeConfig := fmt.Sprintf(`{"Domains":[{"Domain":"foo.%d.%d.example.io","Certificate":"bar","Key":"baz"}]}`, i, j)
+			require.NoError(b, ioutil.WriteFile(dir+"/config.json", []byte(fakeConfig), 0644))
+		}
+		if i%100 == 0 {
+			fmt.Print(".")
+		}
+	}
+
+	b.Run("ReadGroups", func(b *testing.B) {
+		var dm Map
+		for i := 0; i < 2; i++ {
+			dm = make(Map)
+			require.NoError(b, dm.ReadGroups("example.com"))
+		}
+		b.Logf("found %d domains", len(dm))
+	})
+}
diff --git a/internal/fixture/fixtures.go b/internal/fixture/fixtures.go
new file mode 100644
index 00000000..38bbd375
--- /dev/null
+++ b/internal/fixture/fixtures.go
@@ -0,0 +1,56 @@
+package fixture
+
+const (
+	// Certificate is used for HTTPS tests
+	Certificate = `-----BEGIN CERTIFICATE-----
+MIIDZDCCAkygAwIBAgIRAOtN9/zy+gFjdsgpKq3QRdQwDQYJKoZIhvcNAQELBQAw
+MzEUMBIGA1UEChMLTG9nIENvdXJpZXIxGzAZBgNVBAMTEmdpdGxhYi1leGFtcGxl
+LmNvbTAgFw0xODAzMjMxODMwMDZaGA8yMTE4MDIyNzE4MzAwNlowMzEUMBIGA1UE
+ChMLTG9nIENvdXJpZXIxGzAZBgNVBAMTEmdpdGxhYi1leGFtcGxlLmNvbTCCASIw
+DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKULsxpnazXX5RsVzayrAQB+lWwr
+Wef5L5eDhSsIsBLbelYp5YB4TmVRt5x7bWKOOJSBsOfwHZHKJXdu+uuX2RenZlhk
+3Qpq9XGaPZjYm/NHi8gBHPAtz5sG5VaKNvkfTzRGnO9CWA9TM1XtYiOBq94dO+H3
+c+5jP5Yw+mJ+hA+i2058zF8nRlUHArEno2ofrHwE0LMZ11VskpXtWnVfs3voLs8p
+r76KXPBFkMJR4qkWrMDF5Y5MbsQ0zisn6KXrTyV0S4MQh4vSyPdFHnEzvJ07rm5x
+4RTWrjgQeQ2DjZjQvRmaDzlVBK9kaMkJ1Si3agK+gpji6d6WZ/Mb2el1GK8CAwEA
+AaNxMG8wDgYDVR0PAQH/BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1Ud
+EwEB/wQFMAMBAf8wNwYDVR0RBDAwLoIUKi5naXRsYWItZXhhbXBsZS5jb22HBH8A
+AAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggEBAJ0NM8apK0xI
+YxMstP/dCQXtR0wyREGSD/eOpeY3bWlqCbpRgMFUGjQlrsEozcPZOCSCKX5p+tym
+7GsnYtXkwbsuURoSz+5IlhRPVHcUlUeGRdv3/gCd8fDXiigALCsB6GrkMG5cUfh+
+x5p52AC3eQdWTDoxNou+2gzwkAl8iJc13Ykusst0YUqcsXKqTuei2quxFv0pEBSO
+p8wEixoicLFNqPnIDmgx5894DAn0bccNXgRWtq8lLbdhGUlBbpatevvFMgNvFUbe
+eeGb9D0EfpxmzxUl+L0xZtfg3f7cu5AgLG8tb6l4AK6NPVuXN8DmUgvnauWJjZME
+fgStI+IRNVg=
+-----END CERTIFICATE-----
+`
+
+	// Key is used for HTTPS tests
+	Key = `-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEApQuzGmdrNdflGxXNrKsBAH6VbCtZ5/kvl4OFKwiwEtt6Vinl
+gHhOZVG3nHttYo44lIGw5/Adkcold27665fZF6dmWGTdCmr1cZo9mNib80eLyAEc
+8C3PmwblVoo2+R9PNEac70JYD1MzVe1iI4Gr3h074fdz7mM/ljD6Yn6ED6LbTnzM
+XydGVQcCsSejah+sfATQsxnXVWySle1adV+ze+guzymvvopc8EWQwlHiqRaswMXl
+jkxuxDTOKyfopetPJXRLgxCHi9LI90UecTO8nTuubnHhFNauOBB5DYONmNC9GZoP
+OVUEr2RoyQnVKLdqAr6CmOLp3pZn8xvZ6XUYrwIDAQABAoIBAHhP5QnUZeTkMtDh
+vgKmzZ4sqIQnvexKTBUo/MR4GtJESBPTisdx68QUI8LgfsafYkNvnyQUd5m1QEam
+Eif3k3uYvhSlwjQ78BwWEdz/2f8oIo9zsEKtQm+CQWAqdRR5bGVxLCmFtWfGgN+c
+ojO77SuHKAX7OvmGQ+4aWgu+qkoyg/chIpPXMduAjLMtN3eg60ZqJ5KrKuIF63Bb
+xkPQvzJueB9SfUurmKjUltDMx6G/9RZyS0OIRGyL9Qp8MZ8jE23cXOcDgm0HhkPq
+W4LU++aWAOLYziTjnhjJ+4Iz9R7U8sCmk1wgnK/tapVcJf41R98WuGluyjXpsXgA
+k7vmofECgYEAzuGun9lZ7xGwPifp6vGanWXnW+JiZgCTGuHWgQLIXWYcLfoI3kpH
+eLBYINBwvjIQ7P6UxsGSSXd+T82t+8W2LLc2fiKFKE1LVySpH99+cfmIPXxrviOz
+GBX9LTdSCdGkgb54m8aJCpNFnKw5wYgcW1L8CaXXly2Z/KNrGR9R/YUCgYEAzDs4
+19HqlutGLTC30/ziiiIfDaBbX9AzBdUfp9GdT53Mi/7bfxpW/sL4RjG2fGgmN6ua
+fh5npT9AB1ldcEg2qfyOJPt1Ubdi6ek9lx8AB2RMhwdihgX+7bjVMFtjg4b8z5C1
+jQbEr1rhFdpaGyNehtAXDgCbDWQBYnBrmM0rCaMCgYBip1Qyfd9ZFcJJoZb2pofo
+jvOo6Weq5JNBungjxUPu5gaCFj2sYxd6Af3EiCF7UTypBy3DKgOsbQMa4yYYbcvV
+vviJZcTB1zoaMC1GObl+eFPzniVy4mtBDRtSOJMyg3pDNKUnA6HOHTSQ5cAU/ecn
+1YbCwwbv3JsV0of7zue2UQKBgQCVc0j3dd9rLSQfcaUz9bx5RNrgh9YV2S9dN0aA
+8f1iA6FpWMiazFWY/GfeRga6JyTAXE0juXAzFoPuXNDpl46Y+f2yxmhlsgMqFMpD
+SiYlQppVvWu1k7GnmDg5uMarux5JbiXM24UWpTRNX4nMjidgE+qrDnpoZCQ3Ovkh
+yhGSbQKBgD3VEnPiSUmXBo39kPcnPg93E3JfdAOiOwIB2qwfYzg9kpmuTWws+DFz
+lKpMI27YkmnPqROQ2NTUfdxYmw3EHHMAsvnmHeMNGn3ijSUZVKmPfV436Qc8iVci
+s4wKoCRhBUZ52sHki/ieb+5hycT3JnVXMDtbJxgXFW5a86usXEpO
+-----END RSA PRIVATE KEY-----`
+)