Implement HTTPS-only pages

- Check `config.json` for `httpsonly` attribute
- Store value against custom domain or group/project pair
- Respond with 301 redirect to HTTP requests to these domains/projects

Re: https://gitlab.com/gitlab-org/gitlab-ce/issues/28857

18 files changed, 249 insertions, 76 deletions

diff --git a/README.md b/README.md
index a99f8510..d93fe2c7 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,24 @@ current requests.
    a `Content-Encoding: gzip` header. This allows compressed versions of the
    files to be precalculated, saving CPU time and network bandwidth.
 
+### HTTPS only domains
+
+Users have the option to enable "HTTPS only pages" on a per-project basis.
+This option is also enabled by default for all newly-created projects.
+
+When the option is enabled, a project's `config.json` will contain an
+`https_only` attribute.
+
+When the `https_only` attribute is found in the root context, any project pages
+served over HTTP via the group domain (i.e. `username.gitlab.io`) will be 301
+redirected to HTTPS.
+
+When the attribute is found in a custom domain's configuration, any HTTP
+requests to this domain will likewise be redirected.
+
+If the attribute's value is false, or the attribute is missing, then
+the content will be served to the client over HTTP.
+
 ### How it should be run?
 
 Ideally the GitLab Pages should run without any load balancer in front of it.
diff --git a/acceptance_test.go b/acceptance_test.go
index 058b4994..91fce558 100644
--- a/acceptance_test.go
+++ b/acceptance_test.go
@@ -48,7 +48,7 @@ func skipUnlessEnabled(t *testing.T) {
 
 func TestUnknownHostReturnsNotFound(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-redirect-http=false")
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "")
 	defer teardown()
 
 	for _, spec := range listeners {
@@ -152,7 +152,7 @@ func TestKnownHostWithPortReturns200(t *testing.T) {
 
 func TestHttpToHttpsRedirectDisabled(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-redirect-http=false")
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "")
 	defer teardown()
 
 	rsp, err := GetRedirectPage(t, httpListener, "group.gitlab-example.com", "project/")
@@ -184,6 +184,61 @@ func TestHttpToHttpsRedirectEnabled(t *testing.T) {
 	assert.Equal(t, http.StatusOK, rsp.StatusCode)
 }
 
+func TestHttpsOnlyGroupEnabled(t *testing.T) {
+	skipUnlessEnabled(t)
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "")
+	defer teardown()
+
+	rsp, err := GetRedirectPage(t, httpListener, "group.https-only.gitlab-example.com", "project1/")
+	assert.NoError(t, err)
+	defer rsp.Body.Close()
+	assert.Equal(t, http.StatusMovedPermanently, rsp.StatusCode)
+}
+
+func TestHttpsOnlyGroupDisabled(t *testing.T) {
+	skipUnlessEnabled(t)
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "")
+	defer teardown()
+
+	rsp, err := GetPageFromListener(t, httpListener, "group.https-only.gitlab-example.com", "project2/")
+	assert.NoError(t, err)
+	defer rsp.Body.Close()
+	assert.Equal(t, http.StatusOK, rsp.StatusCode)
+}
+
+func TestHttpsOnlyProjectEnabled(t *testing.T) {
+	skipUnlessEnabled(t)
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "")
+	defer teardown()
+
+	rsp, err := GetRedirectPage(t, httpListener, "test.my-domain.com", "/index.html")
+	assert.NoError(t, err)
+	defer rsp.Body.Close()
+	assert.Equal(t, http.StatusMovedPermanently, rsp.StatusCode)
+}
+
+func TestHttpsOnlyProjectDisabled(t *testing.T) {
+	skipUnlessEnabled(t)
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "")
+	defer teardown()
+
+	rsp, err := GetPageFromListener(t, httpListener, "test2.my-domain.com", "/")
+	assert.NoError(t, err)
+	defer rsp.Body.Close()
+	assert.Equal(t, http.StatusOK, rsp.StatusCode)
+}
+
+func TestHttpsOnlyDomainDisabled(t *testing.T) {
+	skipUnlessEnabled(t)
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "")
+	defer teardown()
+
+	rsp, err := GetPageFromListener(t, httpListener, "no.cert.com", "/")
+	assert.NoError(t, err)
+	defer rsp.Body.Close()
+	assert.Equal(t, http.StatusOK, rsp.StatusCode)
+}
+
 func TestPrometheusMetricsCanBeScraped(t *testing.T) {
 	skipUnlessEnabled(t)
 	listener := []ListenSpec{{"http", "127.0.0.1", "37003"}}
@@ -197,13 +252,13 @@ func TestPrometheusMetricsCanBeScraped(t *testing.T) {
 		body, _ := ioutil.ReadAll(resp.Body)
 
 		assert.Contains(t, string(body), "gitlab_pages_http_sessions_active 0")
-		assert.Contains(t, string(body), "gitlab_pages_domains_served_total 7")
+		assert.Contains(t, string(body), "gitlab_pages_domains_served_total 11")
 	}
 }
 
 func TestStatusPage(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-redirect-http=false", "-pages-status=/@statuscheck")
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-pages-status=/@statuscheck")
 	defer teardown()
 
 	rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "@statuscheck")
@@ -214,7 +269,7 @@ func TestStatusPage(t *testing.T) {
 
 func TestStatusNotYetReady(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-redirect-http=false", "-pages-status=/@statuscheck", "-pages-root=shared/invalid-pages")
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-pages-status=/@statuscheck", "-pages-root=shared/invalid-pages")
 	defer teardown()
 
 	rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "@statuscheck")
@@ -225,7 +280,7 @@ func TestStatusNotYetReady(t *testing.T) {
 
 func TestPageNotAvailableIfNotLoaded(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-redirect-http=false", "-pages-root=shared/invalid-pages")
+	teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-pages-root=shared/invalid-pages")
 	defer teardown()
 
 	rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "index.html")
diff --git a/app.go b/app.go
index afc43466..f7b07976 100644
--- a/app.go
+++ b/app.go
@@ -66,49 +66,71 @@ func (a *theApp) healthCheck(w http.ResponseWriter, r *http.Request, https bool)
 	}
 }
 
-func (a *theApp) serveContent(ww http.ResponseWriter, r *http.Request, https bool) {
-	w := newLoggingResponseWriter(ww)
-	defer w.Log(r)
+func (a *theApp) redirectToHTTPS(w http.ResponseWriter, r *http.Request, statusCode int) {
+	u := *r.URL
+	u.Scheme = "https"
+	u.Host = r.Host
+	u.User = nil
 
-	metrics.SessionsActive.Inc()
-	defer metrics.SessionsActive.Dec()
+	http.Redirect(w, r, u.String(), statusCode)
+}
 
+func (a *theApp) getHostAndDomain(r *http.Request) (host string, domain *domain) {
+	host, _, err := net.SplitHostPort(r.Host)
+	if err != nil {
+		host = r.Host
+	}
+
+	return host, a.domain(host)
+}
+
+func (a *theApp) tryAuxiliaryHandlers(w http.ResponseWriter, r *http.Request, https bool, host string, domain *domain) bool {
 	// short circuit content serving to check for a status page
 	if r.RequestURI == a.appConfig.StatusPath {
-		a.healthCheck(&w, r, https)
-		return
+		a.healthCheck(w, r, https)
+		return true
 	}
 
 	// Add auto redirect
 	if !https && a.RedirectHTTP {
-		u := *r.URL
-		u.Scheme = "https"
-		u.Host = r.Host
-		u.User = nil
-
-		http.Redirect(&w, r, u.String(), 307)
-		return
-	}
-
-	host, _, err := net.SplitHostPort(r.Host)
-	if err != nil {
-		host = r.Host
+		a.redirectToHTTPS(w, r, http.StatusTemporaryRedirect)
+		return true
 	}
 
 	// In the event a host is prefixed with the artifact prefix an artifact
 	// value is created, and an attempt to proxy the request is made
-	if a.Artifact.TryMakeRequest(host, &w, r) {
-		return
+	if a.Artifact.TryMakeRequest(host, w, r) {
+		return true
 	}
 
 	if !a.isReady() {
-		httperrors.Serve503(&w)
-		return
+		httperrors.Serve503(w)
+		return true
 	}
 
-	domain := a.domain(host)
 	if domain == nil {
-		httperrors.Serve404(&w)
+		httperrors.Serve404(w)
+		return true
+	}
+
+	if !https && domain.isHTTPSOnly(r) {
+		a.redirectToHTTPS(w, r, http.StatusMovedPermanently)
+		return true
+	}
+
+	return false
+}
+
+func (a *theApp) serveContent(ww http.ResponseWriter, r *http.Request, https bool) {
+	w := newLoggingResponseWriter(ww)
+	defer w.Log(r)
+
+	metrics.SessionsActive.Inc()
+	defer metrics.SessionsActive.Dec()
+
+	host, domain := a.getHostAndDomain(r)
+
+	if a.tryAuxiliaryHandlers(&w, r, https, host, domain) {
 		return
 	}
 
diff --git a/domain.go b/domain.go
index 5d406fdc..aca027ee 100644
--- a/domain.go
+++ b/domain.go
@@ -22,16 +22,27 @@ type locationDirectoryError struct {
 	RelativePath string
 }
 
-func (l *locationDirectoryError) Error() string {
-	return "location error accessing directory where file expected"
+type project struct {
+	HTTPSOnly bool
 }
 
+type projects map[string]*project
+
 type domain struct {
-	Group            string
-	Project          string
+	Group string
+
+	// custom domains:
+	ProjectName      string
 	Config           *domainConfig
 	certificate      *tls.Certificate
 	certificateError error
+
+	// group domains:
+	Projects projects
+}
+
+func (l *locationDirectoryError) Error() string {
+	return "location error accessing directory where file expected"
 }
 
 func acceptsGZip(r *http.Request) bool {
@@ -69,6 +80,20 @@ func setContentType(w http.ResponseWriter, fullPath string) {
 	}
 }
 
+func (d *domain) isHTTPSOnly(r *http.Request) bool {
+	if d.Config != nil {
+		return d.Config.HTTPSOnly
+	}
+
+	split := strings.SplitN(r.URL.Path, "/", 3)
+	if len(split) < 2 {
+		return false
+	}
+
+	project := d.Projects[split[1]]
+	return project.HTTPSOnly
+}
+
 func (d *domain) serveFile(w http.ResponseWriter, r *http.Request, origPath string) error {
 	fullPath := handleGZip(w, r, origPath)
 
@@ -234,12 +259,12 @@ func (d *domain) serveFromGroup(w http.ResponseWriter, r *http.Request) {
 
 func (d *domain) serveFromConfig(w http.ResponseWriter, r *http.Request) {
 	// Try to serve file for http://host/... => /group/project/...
-	if d.tryFile(w, r, d.Project, "", r.URL.Path) == nil {
+	if d.tryFile(w, r, d.ProjectName, "", r.URL.Path) == nil {
 		return
 	}
 
 	// Try serving not found page for http://host/ => /group/project/404.html
-	if d.tryNotFound(w, r, d.Project) == nil {
+	if d.tryNotFound(w, r, d.ProjectName) == nil {
 		return
 	}
 
diff --git a/domain_config.go b/domain_config.go
index e0f00cc4..f5c3db79 100644
--- a/domain_config.go
+++ b/domain_config.go
@@ -11,10 +11,12 @@ type domainConfig struct {
 	Domain      string
 	Certificate string
 	Key         string
+	HTTPSOnly   bool `json:"https_only"`
 }
 
 type domainsConfig struct {
-	Domains []domainConfig
+	Domains   []domainConfig
+	HTTPSOnly bool `json:"https_only"`
 }
 
 func (c *domainConfig) Valid(rootDomain string) bool {
diff --git a/domain_test.go b/domain_test.go
index 26be21eb..454148ae 100644
--- a/domain_test.go
+++ b/domain_test.go
@@ -18,8 +18,8 @@ func TestGroupServeHTTP(t *testing.T) {
 	setUpTests()
 
 	testGroup := &domain{
-		Group:   "group",
-		Project: "",
+		Group:       "group",
+		ProjectName: "",
 	}
 
 	assert.HTTPBodyContains(t, testGroup.ServeHTTP, "GET", "http://group.test.io/", nil, "main-dir")
@@ -48,8 +48,8 @@ func TestDomainServeHTTP(t *testing.T) {
 	setUpTests()
 
 	testDomain := &domain{
-		Group:   "group",
-		Project: "project2",
+		Group:       "group",
+		ProjectName: "project2",
 		Config: &domainConfig{
 			Domain: "test.domain.com",
 		},
@@ -95,8 +95,8 @@ func TestGroupServeHTTPGzip(t *testing.T) {
 	setUpTests()
 
 	testGroup := &domain{
-		Group:   "group",
-		Project: "",
+		Group:       "group",
+		ProjectName: "",
 	}
 
 	testSet := []struct {
@@ -158,8 +158,8 @@ func TestGroup404ServeHTTP(t *testing.T) {
 	setUpTests()
 
 	testGroup := &domain{
-		Group:   "group.404",
-		Project: "",
+		Group:       "group.404",
+		ProjectName: "",
 	}
 
 	testHTTP404(t, testGroup.ServeHTTP, "GET", "http://group.404.test.io/project.404/not/existing-file", nil, "Custom 404 project page")
@@ -175,8 +175,8 @@ func TestDomain404ServeHTTP(t *testing.T) {
 	setUpTests()
 
 	testDomain := &domain{
-		Group:   "group.404",
-		Project: "domain.404",
+		Group:       "group.404",
+		ProjectName: "domain.404",
 		Config: &domainConfig{
 			Domain: "domain.404.com",
 		},
@@ -198,8 +198,8 @@ func TestPredefined404ServeHTTP(t *testing.T) {
 
 func TestGroupCertificate(t *testing.T) {
 	testGroup := &domain{
-		Group:   "group",
-		Project: "",
+		Group:       "group",
+		ProjectName: "",
 	}
 
 	tls, err := testGroup.ensureCertificate()
@@ -209,8 +209,8 @@ func TestGroupCertificate(t *testing.T) {
 
 func TestDomainNoCertificate(t *testing.T) {
 	testDomain := &domain{
-		Group:   "group",
-		Project: "project2",
+		Group:       "group",
+		ProjectName: "project2",
 		Config: &domainConfig{
 			Domain: "test.domain.com",
 		},
@@ -227,8 +227,8 @@ func TestDomainNoCertificate(t *testing.T) {
 
 func TestDomainCertificate(t *testing.T) {
 	testDomain := &domain{
-		Group:   "group",
-		Project: "project2",
+		Group:       "group",
+		ProjectName: "project2",
 		Config: &domainConfig{
 			Domain:      "test.domain.com",
 			Certificate: CertificateFixture,
diff --git a/domains.go b/domains.go
index 000aa42e..0eaca1c3 100644
--- a/domains.go
+++ b/domains.go
@@ -17,57 +17,77 @@ type domains map[string]*domain
 
 type domainsUpdater func(domains domains)
 
-func (d domains) addDomain(rootDomain, group, project string, config *domainConfig) error {
+func (d domains) addDomain(rootDomain, group, projectName string, config *domainConfig) error {
 	newDomain := &domain{
-		Group:   group,
-		Project: project,
-		Config:  config,
+		Group:       group,
+		ProjectName: projectName,
+		Config:      config,
 	}
 
 	var domainName string
-	if config != nil {
-		domainName = config.Domain
-	} else {
-		domainName = group + "." + rootDomain
-	}
-	domainName = strings.ToLower(domainName)
+	domainName = strings.ToLower(config.Domain)
 	d[domainName] = newDomain
 
 	return nil
 }
 
-func (d domains) readProjectConfig(rootDomain, group, project string) (err error) {
+func (d domains) updateGroupDomain(rootDomain, group, projectName string, httpsOnly bool) error {
+	domainName := strings.ToLower(group + "." + rootDomain)
+	groupDomain := d[domainName]
+
+	if groupDomain == nil {
+		groupDomain = &domain{
+			Group:    group,
+			Projects: make(projects),
+		}
+	}
+
+	groupDomain.Projects[projectName] = &project{
+		HTTPSOnly: httpsOnly,
+	}
+	d[domainName] = groupDomain
+
+	return nil
+}
+
+func (d domains) readProjectConfig(rootDomain, group, projectName string) (err error) {
 	var config domainsConfig
-	err = config.Read(group, project)
+	err = config.Read(group, projectName)
 	if err != nil {
+		// This is necessary to preserve the previous behaviour where a
+		// group domain is created even if no config.json files are
+		// loaded successfully. Is it safe to remove this?
+		d.updateGroupDomain(rootDomain, group, projectName, false)
 		return
 	}
 
+	d.updateGroupDomain(rootDomain, group, projectName, config.HTTPSOnly)
+
 	for _, domainConfig := range config.Domains {
 		config := domainConfig // domainConfig is reused for each loop iteration
 		if domainConfig.Valid(rootDomain) {
-			d.addDomain(rootDomain, group, project, &config)
+			d.addDomain(rootDomain, group, projectName, &config)
 		}
 	}
 	return
 }
 
-func (d domains) readProject(rootDomain, group, project string) error {
-	if strings.HasPrefix(project, ".") {
+func (d domains) readProject(rootDomain, group, projectName string) error {
+	if strings.HasPrefix(projectName, ".") {
 		return errors.New("hidden project")
 	}
 
 	// Ignore projects that have .deleted in name
-	if strings.HasSuffix(project, ".deleted") {
+	if strings.HasSuffix(projectName, ".deleted") {
 		return errors.New("deleted project")
 	}
 
-	_, err := os.Lstat(filepath.Join(group, project, "public"))
+	_, err := os.Lstat(filepath.Join(group, projectName, "public"))
 	if err != nil {
 		return errors.New("missing public/ in project")
 	}
 
-	d.readProjectConfig(rootDomain, group, project)
+	d.readProjectConfig(rootDomain, group, projectName)
 	return nil
 }
 
@@ -117,10 +137,7 @@ func (d domains) ReadGroups(rootDomain string) error {
 			continue
 		}
 
-		count := d.readProjects(rootDomain, group.Name())
-		if count > 0 {
-			d.addDomain(rootDomain, group.Name(), "", nil)
-		}
+		d.readProjects(rootDomain, group.Name())
 	}
 	return nil
 }
diff --git a/domains_test.go b/domains_test.go
index 51bafc9f..56901088 100644
--- a/domains_test.go
+++ b/domains_test.go
@@ -32,6 +32,10 @@ func TestReadProjects(t *testing.T) {
 		"other.domain.com",
 		"domain.404.com",
 		"group.404.test.io",
+		"group.https-only.test.io",
+		"test.my-domain.com",
+		"test2.my-domain.com",
+		"no.cert.com",
 	}
 
 	for _, expected := range domains {
diff --git a/shared/pages/group.https-only/project1/config.json b/shared/pages/group.https-only/project1/config.json
new file mode 100644
index 00000000..88f61b80
--- /dev/null
+++ b/shared/pages/group.https-only/project1/config.json
@@ -0,0 +1 @@
+{"https_only":true,"domains":[]}
diff --git a/shared/pages/group.https-only/project1/public/index.html b/shared/pages/group.https-only/project1/public/index.html
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/shared/pages/group.https-only/project1/public/index.html
@@ -0,0 +1 @@
+
diff --git a/shared/pages/group.https-only/project2/config.json b/shared/pages/group.https-only/project2/config.json
new file mode 100644
index 00000000..6a3d66c4
--- /dev/null
+++ b/shared/pages/group.https-only/project2/config.json
@@ -0,0 +1 @@
+{"https_only":false,"domains":[]}
diff --git a/shared/pages/group.https-only/project2/public/index.html b/shared/pages/group.https-only/project2/public/index.html
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/shared/pages/group.https-only/project2/public/index.html
diff --git a/shared/pages/group.https-only/project3/config.json b/shared/pages/group.https-only/project3/config.json
new file mode 100644
index 00000000..0dacdbdb
--- /dev/null
+++ b/shared/pages/group.https-only/project3/config.json
@@ -0,0 +1,8 @@
+{
+  "domains": [
+    {
+      "domain": "test.my-domain.com",
+      "https_only": true
+    }
+  ]
+}
diff --git a/shared/pages/group.https-only/project3/public/index.html b/shared/pages/group.https-only/project3/public/index.html
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/shared/pages/group.https-only/project3/public/index.html
diff --git a/shared/pages/group.https-only/project4/config.json b/shared/pages/group.https-only/project4/config.json
new file mode 100644
index 00000000..5ef48cef
--- /dev/null
+++ b/shared/pages/group.https-only/project4/config.json
@@ -0,0 +1,8 @@
+{
+  "domains": [
+    {
+      "domain": "test2.my-domain.com",
+      "https_only": false
+    }
+  ]
+}
diff --git a/shared/pages/group.https-only/project4/public/index.html b/shared/pages/group.https-only/project4/public/index.html
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/shared/pages/group.https-only/project4/public/index.html
diff --git a/shared/pages/group.https-only/project5/config.json b/shared/pages/group.https-only/project5/config.json
new file mode 100644
index 00000000..5813bb85
--- /dev/null
+++ b/shared/pages/group.https-only/project5/config.json
@@ -0,0 +1,11 @@
+{
+  "httpsonly": true,
+  "domains": [
+    {
+      "domain": "no.cert.com",
+      "certificate": "test",
+      "key": "test",
+      "httpsonly": false
+    }
+  ]
+}
diff --git a/shared/pages/group.https-only/project5/public/index.html b/shared/pages/group.https-only/project5/public/index.html
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/shared/pages/group.https-only/project5/public/index.html