diff options
author | Kamil Trzcinski <ayufan@ayufan.eu> | 2016-01-07 19:37:59 +0300 |
---|---|---|
committer | Kamil Trzcinski <ayufan@ayufan.eu> | 2016-01-07 19:37:59 +0300 |
commit | d7dc7ed7201437a3f450bd783800386e01d62661 (patch) | |
tree | 8053829c7ba64208917aaf599a912483a4509b95 | |
parent | 61debe70052f5da9e32acdb4d695ccf56991df9f (diff) |
Add simple GitLab Pages daemon with custom CNAME and TLS support
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | README.md | 48 | ||||
-rw-r--r-- | domain.go | 122 | ||||
-rw-r--r-- | domains.go | 90 | ||||
-rw-r--r-- | logging.go | 52 | ||||
-rw-r--r-- | main.go | 105 | ||||
-rw-r--r-- | server.go | 42 |
7 files changed, 460 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..bfa6a22a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +# Created by .ignore support plugin (hsz.mobi) diff --git a/README.md b/README.md new file mode 100644 index 00000000..0060538e --- /dev/null +++ b/README.md @@ -0,0 +1,48 @@ +## GitLab Pages Daemon + +This is simple HTTP server written in Go made to serve GitLab Pages with CNAMEs and SNI using HTTP/HTTP2. + +This is made to work in small-to-medium scale environments. +In large environment it can be time consuming to list all directories, and CNAMEs. + +### How it generates routes + +1. It reads the `pages-root` directory to list all groups +2. It looks for `CNAME` files in `pages-root/group/project` directory, reads them and creates mapping for custom CNAMEs. +3. It generates virtual-host from these data. +4. Periodically (every second) it checks the `pages-root` directory if it was modified to reload all mappings. + +To force route refresh, CNAME reload or TLS certificate reload: `touch pages-root`. +It will be done asynchronously, not interrupting current requests. + +### How it serves content + +1. When client initiates the TLS connection, the GitLab-Pages daemon looks in hash map for virtual hosts and tries to load TLS certificate from: +`pages-root/group/project/domain.{crt,key}`. + +2. When client asks HTTP server the GitLab-Pages daemon looks in hash map for registered virtual hosts. + +3. The URL.Path is split into `/<project>/<subpath>` and we daemon tries to load: `pages-root/group/project/public/subpath`. + +4. If file was not found it will try to load `pages-root/group/<host>/public/<URL.Path>`. + +5. If requested path is directory, the `index.html` will be served. + +### How it should be run? + +Ideally the GitLab Pages should run without load balancer. + +If load balancer is required, the HTTP can be served in HTTP mode. +For HTTPS traffic load balancer should be run in TCP-mode. +If load balancer is run in SSL-offloading mode the custom TLS certificate will not work. + +### How to run it + +``` +go build +./gitlab-pages -listen-https "" -listen-http ":8090" -pages-root path/to/gitlab/shared/pages +``` + +### License + +MIT diff --git a/domain.go b/domain.go new file mode 100644 index 00000000..d306b936 --- /dev/null +++ b/domain.go @@ -0,0 +1,122 @@ +package main + +import ( + "net/http" + "os" + "path/filepath" + "strings" + "crypto/tls" + "errors" +) + +type domain struct { + Group string + Project string + CNAME bool + certificate *tls.Certificate +} + +func (d *domain) notFound(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) +} + +func (d *domain) tryFile(w http.ResponseWriter, r *http.Request, projectName, subPath string) bool { + publicPath := filepath.Join(*pagesRoot, d.Group, projectName, "public") + fullPath := filepath.Join(publicPath, subPath) + fullPath = filepath.Clean(fullPath) + if !strings.HasPrefix(fullPath, publicPath) { + return false + } + + fi, err := os.Lstat(fullPath) + if err != nil { + return false + } + + // If this file is directory, open the index.html + if fi.IsDir() { + fullPath = filepath.Join(fullPath, "index.html") + fi, err = os.Lstat(fullPath) + if err != nil { + return false + } + } + + // We don't allow to open non-regular files + if !fi.Mode().IsRegular() { + return false + } + + // Open and serve content of file + file, err := os.Open(fullPath) + if err != nil { + return false + } + defer file.Close() + + fi, err = file.Stat() + if err != nil { + return false + } + + http.ServeContent(w, r, filepath.Base(file.Name()), fi.ModTime(), file) + return true +} + +func (d *domain) serverGroup(w http.ResponseWriter, r *http.Request) { + // The Path always contains "/" at the beggining + split := strings.SplitN(r.URL.Path, "/", 3) + + if len(split) >= 2 { + subPath := "" + if len(split) >= 3 { + subPath = split[2] + } + if d.tryFile(w, r, split[1], subPath) { + return + } + } + + if d.tryFile(w, r, strings.ToLower(r.Host), r.URL.Path) { + return + } + + d.notFound(w, r) +} + +func (d *domain) serveCNAME(w http.ResponseWriter, r *http.Request) { + if d.tryFile(w, r, d.Project, r.URL.Path) { + return + } + + d.notFound(w, r) +} + +func (d *domain) ensureCertificate() (*tls.Certificate, error) { + if !d.CNAME { + return nil, errors.New("tls certificates can be loaded only for pages with CNAME") + } + + if d.certificate != nil { + return d.certificate, nil + } + + // Load keypair from shared/pages/group/project/domain.{crt,key} + certificateFile := filepath.Join(*pagesRoot, d.Group, d.Project, "domain.crt") + keyFile := filepath.Join(*pagesRoot, d.Group, d.Project, "domain.key") + tls, err := tls.LoadX509KeyPair(certificateFile, keyFile) + if err != nil { + return nil, err + } + + d.certificate = &tls + return d.certificate, nil +} + +func (d *domain) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if d.CNAME { + d.serveCNAME(w, r) + } else { + d.serverGroup(w, r) + } +} diff --git a/domains.go b/domains.go new file mode 100644 index 00000000..cddffdf8 --- /dev/null +++ b/domains.go @@ -0,0 +1,90 @@ +package main + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "time" +) + +type domains map[string]domain + +type domainsUpdater func(domains domains) + +func readGroups(domains domains) error { + groups, err := filepath.Glob(filepath.Join(*pagesRoot, "*/")) + if err != nil { + return err + } + + for _, groupDir := range groups { + group := filepath.Base(groupDir) + groupName := strings.ToLower(group) + domains[groupName+"."+*pagesDomain] = domain{ + Group: group, + CNAME: false, + } + } + return nil +} + +func readCnames(domains domains) error { + cnames, err := filepath.Glob(filepath.Join(*pagesRoot, "*/*/CNAME")) + if err != nil { + return err + } + + for _, cnamePath := range cnames { + cnameData, err := ioutil.ReadFile(cnamePath) + if err != nil { + continue + } + + for _, cname := range strings.Fields(string(cnameData)) { + cname := strings.ToLower(cname) + if strings.HasSuffix(cname, "."+*pagesDomain) { + continue + } + + domains[cname] = domain{ + // TODO: make it nicer + Group: filepath.Base(filepath.Dir(filepath.Dir(cnamePath))), + Project: filepath.Base(filepath.Dir(cnamePath)), + CNAME: true, + } + } + } + return nil +} + +func watchDomains(updater domainsUpdater) { + var lastModified time.Time + + for { + fi, err := os.Stat(*pagesRoot) + if err != nil || !fi.IsDir() { + log.Println("Failed to read domains from", *pagesRoot, "due to:", err, fi.IsDir()) + time.Sleep(time.Second) + continue + } + + // If directory did not get modified we will reload + if !lastModified.Before(fi.ModTime()) { + time.Sleep(time.Second) + continue + } + lastModified = fi.ModTime() + + started := time.Now() + domains := make(domains) + readGroups(domains) + readCnames(domains) + duration := time.Since(started) + log.Println("Updated", len(domains), "domains in", duration) + if updater != nil { + updater(domains) + } + } +} diff --git a/logging.go b/logging.go new file mode 100644 index 00000000..bee171b2 --- /dev/null +++ b/logging.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + "net/http" + "time" +) + +type loggingResponseWriter struct { + rw http.ResponseWriter + status int + written int64 + started time.Time +} + +func newLoggingResponseWriter(rw http.ResponseWriter) loggingResponseWriter { + return loggingResponseWriter{ + rw: rw, + started: time.Now(), + } +} + +func (l *loggingResponseWriter) Header() http.Header { + return l.rw.Header() +} + +func (l *loggingResponseWriter) Write(data []byte) (n int, err error) { + if l.status == 0 { + l.WriteHeader(http.StatusOK) + } + n, err = l.rw.Write(data) + l.written += int64(n) + return +} + +func (l *loggingResponseWriter) WriteHeader(status int) { + if l.status != 0 { + return + } + + l.status = status + l.rw.WriteHeader(status) +} + +func (l *loggingResponseWriter) Log(r *http.Request) { + duration := time.Since(l.started) + fmt.Printf("%s %s - - [%s] %q %d %d %q %q %f\n", + r.Host, r.RemoteAddr, l.started, + fmt.Sprintf("%s %s %s", r.Method, r.RequestURI, r.Proto), + l.status, l.written, r.Referer(), r.UserAgent(), duration.Seconds(), + ) +} @@ -0,0 +1,105 @@ +package main + +import ( + "crypto/tls" + "flag" + "fmt" + "log" + "net/http" + "strings" + "sync" +) + +var listenHTTP = flag.String("listen-http", ":80", "The address to listen for HTTP requests") +var listenHTTPS = flag.String("listen-https", "", "The address to listen for HTTPS requests") +var pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages") +var pagesRootCert = flag.String("root-cert", "", "The default certificate to serve static pages") +var pagesRootKey = flag.String("root-key", "", "The default certificate to serve static pages") +var serverHTTP = flag.Bool("serve-http", true, "Serve the pages under HTTP") +var http2proto = flag.Bool("http2", true, "Enable HTTP2 support") +var pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored") + +type theApp struct { + domains domains +} + +func (a *theApp) ServeTLS(ch *tls.ClientHelloInfo) (*tls.Certificate, error) { + if ch.ServerName == "" { + return nil, nil + } + + host := strings.ToLower(ch.ServerName) + if domain, ok := a.domains[host]; ok { + tls, _ := domain.ensureCertificate() + return tls, nil + } + + return nil, nil +} + +func (a *theApp) ServeHTTP(ww http.ResponseWriter, r *http.Request) { + w := newLoggingResponseWriter(ww) + defer w.Log(r) + + // Add auto redirect + if r.TLS == nil && !*serverHTTP { + u := *r.URL + u.Scheme = "https" + u.Host = r.Host + u.User = nil + + http.Redirect(&w, r, u.String(), 307) + return + } + + host := strings.ToLower(r.Host) + domain, ok := a.domains[host] + + if !ok { + http.NotFound(&w, r) + return + } + + // Serve static file + domain.ServeHTTP(&w, r) +} + +func (a *theApp) UpdateDomains(domains domains) { + fmt.Printf("Domains: %v", domains) + a.domains = domains +} + +func main() { + var wg sync.WaitGroup + var app theApp + + flag.Parse() + + // Listen for HTTP + if *listenHTTP != "" { + wg.Add(1) + go func() { + defer wg.Done() + err := ListenAndServe(*listenHTTP, &app) + if err != nil { + log.Fatal(err) + } + }() + } + + // Listen for HTTPS + if *listenHTTPS != "" { + wg.Add(1) + go func() { + defer wg.Done() + err := ListenAndServeTLS(*listenHTTPS, *pagesRootCert, *pagesRootKey, &app) + if err != nil { + log.Fatal(err) + } + }() + } + + go watchDomains(app.UpdateDomains) + + wg.Wait() +} diff --git a/server.go b/server.go new file mode 100644 index 00000000..62647c89 --- /dev/null +++ b/server.go @@ -0,0 +1,42 @@ +package main + +import ( + "crypto/tls" + "golang.org/x/net/http2" + "net/http" +) + +type TLSHandler interface { + http.Handler + ServeTLS(*tls.ClientHelloInfo) (*tls.Certificate, error) +} + +func ListenAndServe(addr string, handler http.Handler) error { + // create server + server := &http.Server{Addr: addr, Handler: handler} + + if *http2proto { + err := http2.ConfigureServer(server, &http2.Server{}) + if err != nil { + return err + } + } + + return server.ListenAndServe() +} + +func ListenAndServeTLS(addr string, certFile, keyFile string, handler TLSHandler) error { + // create server + server := &http.Server{Addr: addr, Handler: handler} + server.TLSConfig = &tls.Config{} + server.TLSConfig.GetCertificate = handler.ServeTLS + + if *http2proto { + err := http2.ConfigureServer(server, &http2.Server{}) + if err != nil { + return err + } + } + + return server.ListenAndServeTLS(certFile, keyFile) +} |