Add simple GitLab Pages daemon with custom CNAME and TLS support

7 files changed, 460 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..bfa6a22a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+# Created by .ignore support plugin (hsz.mobi)
diff --git a/README.md b/README.md
new file mode 100644
index 00000000..0060538e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,48 @@
+## GitLab Pages Daemon
+
+This is simple HTTP server written in Go made to serve GitLab Pages with CNAMEs and SNI using HTTP/HTTP2.
+
+This is made to work in small-to-medium scale environments.
+In large environment it can be time consuming to list all directories, and CNAMEs.
+
+### How it generates routes
+
+1. It reads the `pages-root` directory to list all groups
+2. It looks for `CNAME` files in `pages-root/group/project` directory, reads them and creates mapping for custom CNAMEs.
+3. It generates virtual-host from these data.
+4. Periodically (every second) it checks the `pages-root` directory if it was modified to reload all mappings.
+
+To force route refresh, CNAME reload or TLS certificate reload: `touch pages-root`.
+It will be done asynchronously, not interrupting current requests. 
+
+### How it serves content
+
+1. When client initiates the TLS connection, the GitLab-Pages daemon looks in hash map for virtual hosts and tries to load TLS certificate from:
+`pages-root/group/project/domain.{crt,key}`.
+
+2. When client asks HTTP server the GitLab-Pages daemon looks in hash map for registered virtual hosts.
+
+3. The URL.Path is split into `/<project>/<subpath>` and we daemon tries to load: `pages-root/group/project/public/subpath`.
+
+4. If file was not found it will try to load `pages-root/group/<host>/public/<URL.Path>`.
+
+5. If requested path is directory, the `index.html` will be served.
+
+### How it should be run?
+
+Ideally the GitLab Pages should run without load balancer.
+
+If load balancer is required, the HTTP can be served in HTTP mode.
+For HTTPS traffic load balancer should be run in TCP-mode.
+If load balancer is run in SSL-offloading mode the custom TLS certificate will not work.
+
+### How to run it
+
+```
+go build
+./gitlab-pages -listen-https "" -listen-http ":8090" -pages-root path/to/gitlab/shared/pages
+```
+
+### License
+
+MIT
diff --git a/domain.go b/domain.go
new file mode 100644
index 00000000..d306b936
--- /dev/null
+++ b/domain.go
@@ -0,0 +1,122 @@
+package main
+
+import (
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"crypto/tls"
+	"errors"
+)
+
+type domain struct {
+	Group       string
+	Project     string
+	CNAME       bool
+	certificate *tls.Certificate
+}
+
+func (d *domain) notFound(w http.ResponseWriter, r *http.Request) {
+	http.NotFound(w, r)
+}
+
+func (d *domain) tryFile(w http.ResponseWriter, r *http.Request, projectName, subPath string) bool {
+	publicPath := filepath.Join(*pagesRoot, d.Group, projectName, "public")
+	fullPath := filepath.Join(publicPath, subPath)
+	fullPath = filepath.Clean(fullPath)
+	if !strings.HasPrefix(fullPath, publicPath) {
+		return false
+	}
+
+	fi, err := os.Lstat(fullPath)
+	if err != nil {
+		return false
+	}
+
+	// If this file is directory, open the index.html
+	if fi.IsDir() {
+		fullPath = filepath.Join(fullPath, "index.html")
+		fi, err = os.Lstat(fullPath)
+		if err != nil {
+			return false
+		}
+	}
+
+	// We don't allow to open non-regular files
+	if !fi.Mode().IsRegular() {
+		return false
+	}
+
+	// Open and serve content of file
+	file, err := os.Open(fullPath)
+	if err != nil {
+		return false
+	}
+	defer file.Close()
+
+	fi, err = file.Stat()
+	if err != nil {
+		return false
+	}
+
+	http.ServeContent(w, r, filepath.Base(file.Name()), fi.ModTime(), file)
+	return true
+}
+
+func (d *domain) serverGroup(w http.ResponseWriter, r *http.Request) {
+	// The Path always contains "/" at the beggining
+	split := strings.SplitN(r.URL.Path, "/", 3)
+
+	if len(split) >= 2 {
+		subPath := ""
+		if len(split) >= 3 {
+			subPath = split[2]
+		}
+		if d.tryFile(w, r, split[1], subPath) {
+			return
+		}
+	}
+
+	if d.tryFile(w, r, strings.ToLower(r.Host), r.URL.Path) {
+		return
+	}
+
+	d.notFound(w, r)
+}
+
+func (d *domain) serveCNAME(w http.ResponseWriter, r *http.Request) {
+	if d.tryFile(w, r, d.Project, r.URL.Path) {
+		return
+	}
+
+	d.notFound(w, r)
+}
+
+func (d *domain) ensureCertificate() (*tls.Certificate, error) {
+	if !d.CNAME {
+		return nil, errors.New("tls certificates can be loaded only for pages with CNAME")
+	}
+
+	if d.certificate != nil {
+		return d.certificate, nil
+	}
+
+	// Load keypair from shared/pages/group/project/domain.{crt,key}
+	certificateFile := filepath.Join(*pagesRoot, d.Group, d.Project, "domain.crt")
+	keyFile := filepath.Join(*pagesRoot, d.Group, d.Project, "domain.key")
+	tls, err := tls.LoadX509KeyPair(certificateFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	d.certificate = &tls
+	return d.certificate, nil
+}
+
+func (d *domain) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if d.CNAME {
+		d.serveCNAME(w, r)
+	} else {
+		d.serverGroup(w, r)
+	}
+}
diff --git a/domains.go b/domains.go
new file mode 100644
index 00000000..cddffdf8
--- /dev/null
+++ b/domains.go
@@ -0,0 +1,90 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type domains map[string]domain
+
+type domainsUpdater func(domains domains)
+
+func readGroups(domains domains) error {
+	groups, err := filepath.Glob(filepath.Join(*pagesRoot, "*/"))
+	if err != nil {
+		return err
+	}
+
+	for _, groupDir := range groups {
+		group := filepath.Base(groupDir)
+		groupName := strings.ToLower(group)
+		domains[groupName+"."+*pagesDomain] = domain{
+			Group: group,
+			CNAME: false,
+		}
+	}
+	return nil
+}
+
+func readCnames(domains domains) error {
+	cnames, err := filepath.Glob(filepath.Join(*pagesRoot, "*/*/CNAME"))
+	if err != nil {
+		return err
+	}
+
+	for _, cnamePath := range cnames {
+		cnameData, err := ioutil.ReadFile(cnamePath)
+		if err != nil {
+			continue
+		}
+
+		for _, cname := range strings.Fields(string(cnameData)) {
+			cname := strings.ToLower(cname)
+			if strings.HasSuffix(cname, "."+*pagesDomain) {
+				continue
+			}
+
+			domains[cname] = domain{
+				// TODO: make it nicer
+				Group:   filepath.Base(filepath.Dir(filepath.Dir(cnamePath))),
+				Project: filepath.Base(filepath.Dir(cnamePath)),
+				CNAME:   true,
+			}
+		}
+	}
+	return nil
+}
+
+func watchDomains(updater domainsUpdater) {
+	var lastModified time.Time
+
+	for {
+		fi, err := os.Stat(*pagesRoot)
+		if err != nil || !fi.IsDir() {
+			log.Println("Failed to read domains from", *pagesRoot, "due to:", err, fi.IsDir())
+			time.Sleep(time.Second)
+			continue
+		}
+
+		// If directory did not get modified we will reload
+		if !lastModified.Before(fi.ModTime()) {
+			time.Sleep(time.Second)
+			continue
+		}
+		lastModified = fi.ModTime()
+
+		started := time.Now()
+		domains := make(domains)
+		readGroups(domains)
+		readCnames(domains)
+		duration := time.Since(started)
+		log.Println("Updated", len(domains), "domains in", duration)
+		if updater != nil {
+			updater(domains)
+		}
+	}
+}
diff --git a/logging.go b/logging.go
new file mode 100644
index 00000000..bee171b2
--- /dev/null
+++ b/logging.go
@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+)
+
+type loggingResponseWriter struct {
+	rw      http.ResponseWriter
+	status  int
+	written int64
+	started time.Time
+}
+
+func newLoggingResponseWriter(rw http.ResponseWriter) loggingResponseWriter {
+	return loggingResponseWriter{
+		rw:      rw,
+		started: time.Now(),
+	}
+}
+
+func (l *loggingResponseWriter) Header() http.Header {
+	return l.rw.Header()
+}
+
+func (l *loggingResponseWriter) Write(data []byte) (n int, err error) {
+	if l.status == 0 {
+		l.WriteHeader(http.StatusOK)
+	}
+	n, err = l.rw.Write(data)
+	l.written += int64(n)
+	return
+}
+
+func (l *loggingResponseWriter) WriteHeader(status int) {
+	if l.status != 0 {
+		return
+	}
+
+	l.status = status
+	l.rw.WriteHeader(status)
+}
+
+func (l *loggingResponseWriter) Log(r *http.Request) {
+	duration := time.Since(l.started)
+	fmt.Printf("%s %s - - [%s] %q %d %d %q %q %f\n",
+		r.Host, r.RemoteAddr, l.started,
+		fmt.Sprintf("%s %s %s", r.Method, r.RequestURI, r.Proto),
+		l.status, l.written, r.Referer(), r.UserAgent(), duration.Seconds(),
+	)
+}
diff --git a/main.go b/main.go
index e69de29b..5e5c6a90 100644
--- a/main.go
+++ b/main.go
@@ -0,0 +1,105 @@
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+)
+
+var listenHTTP = flag.String("listen-http", ":80", "The address to listen for HTTP requests")
+var listenHTTPS = flag.String("listen-https", "", "The address to listen for HTTPS requests")
+var pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages")
+var pagesRootCert = flag.String("root-cert", "", "The default certificate to serve static pages")
+var pagesRootKey = flag.String("root-key", "", "The default certificate to serve static pages")
+var serverHTTP = flag.Bool("serve-http", true, "Serve the pages under HTTP")
+var http2proto = flag.Bool("http2", true, "Enable HTTP2 support")
+var pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored")
+
+type theApp struct {
+	domains domains
+}
+
+func (a *theApp) ServeTLS(ch *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	if ch.ServerName == "" {
+		return nil, nil
+	}
+
+	host := strings.ToLower(ch.ServerName)
+	if domain, ok := a.domains[host]; ok {
+		tls, _ := domain.ensureCertificate()
+		return tls, nil
+	}
+
+	return nil, nil
+}
+
+func (a *theApp) ServeHTTP(ww http.ResponseWriter, r *http.Request) {
+	w := newLoggingResponseWriter(ww)
+	defer w.Log(r)
+
+	// Add auto redirect
+	if r.TLS == nil && !*serverHTTP {
+		u := *r.URL
+		u.Scheme = "https"
+		u.Host = r.Host
+		u.User = nil
+
+		http.Redirect(&w, r, u.String(), 307)
+		return
+	}
+
+	host := strings.ToLower(r.Host)
+	domain, ok := a.domains[host]
+
+	if !ok {
+		http.NotFound(&w, r)
+		return
+	}
+
+	// Serve static file
+	domain.ServeHTTP(&w, r)
+}
+
+func (a *theApp) UpdateDomains(domains domains) {
+	fmt.Printf("Domains: %v", domains)
+	a.domains = domains
+}
+
+func main() {
+	var wg sync.WaitGroup
+	var app theApp
+
+	flag.Parse()
+
+	// Listen for HTTP
+	if *listenHTTP != "" {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err := ListenAndServe(*listenHTTP, &app)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}()
+	}
+
+	// Listen for HTTPS
+	if *listenHTTPS != "" {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err := ListenAndServeTLS(*listenHTTPS, *pagesRootCert, *pagesRootKey, &app)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}()
+	}
+
+	go watchDomains(app.UpdateDomains)
+
+	wg.Wait()
+}
diff --git a/server.go b/server.go
new file mode 100644
index 00000000..62647c89
--- /dev/null
+++ b/server.go
@@ -0,0 +1,42 @@
+package main
+
+import (
+	"crypto/tls"
+	"golang.org/x/net/http2"
+	"net/http"
+)
+
+type TLSHandler interface {
+	http.Handler
+	ServeTLS(*tls.ClientHelloInfo) (*tls.Certificate, error)
+}
+
+func ListenAndServe(addr string, handler http.Handler) error {
+	// create server
+	server := &http.Server{Addr: addr, Handler: handler}
+
+	if *http2proto {
+		err := http2.ConfigureServer(server, &http2.Server{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return server.ListenAndServe()
+}
+
+func ListenAndServeTLS(addr string, certFile, keyFile string, handler TLSHandler) error {
+	// create server
+	server := &http.Server{Addr: addr, Handler: handler}
+	server.TLSConfig = &tls.Config{}
+	server.TLSConfig.GetCertificate = handler.ServeTLS
+
+	if *http2proto {
+		err := http2.ConfigureServer(server, &http2.Server{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return server.ListenAndServeTLS(certFile, keyFile)
+}