diff options
| -rw-r--r-- | Godeps/Godeps.json | 4 | ||||
| -rw-r--r-- | README.md | 11 | ||||
| -rw-r--r-- | acceptance_test.go | 6 | ||||
| -rw-r--r-- | app.go | 24 | ||||
| -rw-r--r-- | app_config.go | 8 | ||||
| -rw-r--r-- | domains.go | 9 | ||||
| -rw-r--r-- | helpers_test.go | 60 | ||||
| -rw-r--r-- | main.go | 60 | ||||
| -rw-r--r-- | metrics/metrics.go (renamed from metrics.go) | 24 | ||||
| -rw-r--r-- | vendor/golang.org/x/crypto/LICENSE | 27 | ||||
| -rw-r--r-- | vendor/golang.org/x/crypto/PATENTS | 22 |
11 files changed, 121 insertions, 134 deletions
diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ca81fa69..4a9fbf8e 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "gitlab.com/gitlab-org/gitlab-pages", - "GoVersion": "go1.7", - "GodepVersion": "v77", + "GoVersion": "go1.5", + "GodepVersion": "v79", "Packages": [ "./..." ], @@ -72,6 +72,17 @@ go build This is most useful in dual-stack environments (IPv4+IPv6) where both Gitlab Pages and another HTTP server have to co-exist on the same server. +### Enable Prometheus Metrics + +For monitoring purposes, one could pass the `-metrics-address` flag when +starting. This will expose general metrics about the Go runtime and pages +application for [Prometheus](https://prometheus.io/) to scrape. + +Example: +``` +./gitlab-pages -listen-http ":8090" -metrics-address ":9101" -pages-root path/to/gitlab/shared/pages -pages-domain example.com +``` + ### License MIT diff --git a/acceptance_test.go b/acceptance_test.go index e603e392..1cfd133a 100644 --- a/acceptance_test.go +++ b/acceptance_test.go @@ -2,10 +2,8 @@ package main import ( "flag" - "fmt" "io/ioutil" "net/http" - "regexp" "testing" "github.com/stretchr/testify/assert" @@ -68,7 +66,6 @@ func TestKnownHostReturns200(t *testing.T) { func TestPrometheusMetricsCanBeScraped(t *testing.T) { skipUnlessEnabled(t) listener := []ListenSpec{{"http", "127.0.0.1", "37003"}} - fmt.Println("Start pages process") teardown := RunPagesProcess(t, *pagesBinary, listener, ":42345") defer teardown() @@ -78,6 +75,7 @@ func TestPrometheusMetricsCanBeScraped(t *testing.T) { defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) - assert.Regexp(t, regexp.MustCompile("gitlab_pages_http_sessions_active 0"), string(body)) + assert.Contains(t, string(body), "gitlab_pages_http_sessions_active 0") + assert.Contains(t, string(body), "gitlab_pages_domains_served_total 7") } } @@ -9,7 +9,9 @@ import ( "sync" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "gitlab.com/gitlab-org/gitlab-pages/metrics" ) const xForwardedProto = "X-Forwarded-Proto" @@ -46,8 +48,8 @@ func (a *theApp) serveContent(ww http.ResponseWriter, r *http.Request, https boo w := newLoggingResponseWriter(ww) defer w.Log(r) - sessionsActive.Inc() - defer sessionsActive.Dec() + metrics.SessionsActive.Inc() + defer metrics.SessionsActive.Dec() // Add auto redirect if https && !a.RedirectHTTP { @@ -68,7 +70,7 @@ func (a *theApp) serveContent(ww http.ResponseWriter, r *http.Request, https boo // Serve static file domain.ServeHTTP(&w, r) - processedRequests.WithLabelValues(strconv.Itoa(w.status), r.Method).Inc() + metrics.ProcessedRequests.WithLabelValues(strconv.Itoa(w.status), r.Method).Inc() } func (a *theApp) ServeHTTP(ww http.ResponseWriter, r *http.Request) { @@ -129,11 +131,17 @@ func (a *theApp) Run() { } // Serve metrics for Prometheus - if a.MetricsAddress != "" { - go func() { - http.Handle("/metrics", promhttp.Handler()) - log.Fatal(http.ListenAndServe(a.MetricsAddress, nil)) - }() + if a.ListenMetrics != 0 { + wg.Add(1) + go func(fd uintptr) { + defer wg.Done() + + handler := promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}).ServeHTTP + err := listenAndServe(fd, handler, false, nil) + if err != nil { + log.Fatal(err) + } + }(a.ListenMetrics) } go watchDomains(a.Domain, a.UpdateDomains, time.Second) diff --git a/app_config.go b/app_config.go index 8220bb87..04c787e1 100644 --- a/app_config.go +++ b/app_config.go @@ -6,10 +6,10 @@ type appConfig struct { RootCertificate []byte RootKey []byte - ListenHTTP []uintptr - ListenHTTPS []uintptr - ListenProxy []uintptr - MetricsAddress string + ListenHTTP []uintptr + ListenHTTPS []uintptr + ListenProxy []uintptr + ListenMetrics uintptr HTTP2 bool RedirectHTTP bool @@ -9,6 +9,8 @@ import ( "path/filepath" "strings" "time" + + "gitlab.com/gitlab-org/gitlab-pages/metrics" ) type domains map[string]*domain @@ -31,7 +33,6 @@ func (d domains) addDomain(rootDomain, group, project string, config *domainConf domainName = strings.ToLower(domainName) d[domainName] = newDomain - domainsServed.Inc() return nil } @@ -151,9 +152,9 @@ func watchDomains(rootDomain string, updater domainsUpdater, interval time.Durat } // Update prometheus metrics - domainLastUpdateTime.Set(float64(time.Now().UTC().Unix())) - domainsServed.Set(float64(len(domains))) - domainUpdates.Inc() + metrics.DomainLastUpdateTime.Set(float64(time.Now().UTC().Unix())) + metrics.DomainsServed.Set(float64(len(domains))) + metrics.DomainUpdates.Inc() time.Sleep(interval) } diff --git a/helpers_test.go b/helpers_test.go index 4e5c3d93..e7fd744e 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -3,7 +3,6 @@ package main import ( "crypto/tls" "fmt" - "github.com/stretchr/testify/assert" "io/ioutil" "log" "net" @@ -12,6 +11,8 @@ import ( "os/exec" "testing" "time" + + "github.com/stretchr/testify/assert" ) var chdirSet = false @@ -125,13 +126,38 @@ func (l ListenSpec) JoinHostPort() string { // // If run as root via sudo, the gitlab-pages process will drop privileges func RunPagesProcess(t *testing.T, pagesPath string, listeners []ListenSpec, promPort string) (teardown func()) { - var tempfiles []string - var args []string - var hasHTTPS bool - _, err := os.Stat(pagesPath) assert.NoError(t, err) + args, tempfiles := getPagesArgs(t, listeners, promPort) + cmd := exec.Command(pagesPath, args...) + cmd.Start() + t.Logf("Running %s %v", pagesPath, args) + + // Wait for all TCP servers to be open. Even with this, gitlab-pages + // will sometimes return 404 if a HTTP request comes in before it has + // updated its set of domains. This usually takes < 1ms, hence the sleep + // for now. Without it, intermittent failures occur. + // + // TODO: replace this with explicit status from the pages binary + // TODO: fix the first-request race + for _, spec := range listeners { + spec.WaitUntilListening() + } + time.Sleep(50 * time.Millisecond) + + return func() { + cmd.Process.Kill() + cmd.Process.Wait() + for _, tempfile := range tempfiles { + os.Remove(tempfile) + } + } +} + +func getPagesArgs(t *testing.T, listeners []ListenSpec, promPort string) (args, tempfiles []string) { + var hasHTTPS bool + for _, spec := range listeners { args = append(args, "-listen-"+spec.Type, spec.JoinHostPort()) @@ -155,29 +181,7 @@ func RunPagesProcess(t *testing.T, pagesPath string, listeners []ListenSpec, pro args = append(args, "-daemon-uid", os.Getenv("SUDO_UID"), "-daemon-gid", os.Getenv("SUDO_GID")) } - cmd := exec.Command(pagesPath, args...) - cmd.Start() - fmt.Println("Running %s %v", pagesPath, args) - - // Wait for all TCP servers to be open. Even with this, gitlab-pages - // will sometimes return 404 if a HTTP request comes in before it has - // updated its set of domains. This usually takes < 1ms, hence the sleep - // for now. Without it, intermittent failures occur. - // - // TODO: replace this with explicit status from the pages binary - // TODO: fix the first-request race - for _, spec := range listeners { - spec.WaitUntilListening() - } - time.Sleep(50 * time.Millisecond) - - return func() { - cmd.Process.Kill() - cmd.Process.Wait() - for _, tempfile := range tempfiles { - os.Remove(tempfile) - } - } + return } // Does an insecure HTTP GET against the listener specified, setting a fake @@ -13,6 +13,36 @@ var VERSION = "dev" // REVISION stores the information about the git revision of application var REVISION = "HEAD" +var ( + pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages") + pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages") + redirectHTTP = flag.Bool("redirect-http", true, "Serve the pages under HTTP") + useHTTP2 = flag.Bool("use-http2", true, "Enable HTTP2 support") + pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored") + pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages") + metricsAddress = flag.String("metrics-address", "", "The address to listen on for metrics requests") + daemonUID = flag.Uint("daemon-uid", 0, "Drop privileges to this user") + daemonGID = flag.Uint("daemon-gid", 0, "Drop privileges to this group") +) + +func configFromFlags() appConfig { + var config appConfig + + config.Domain = strings.ToLower(*pagesDomain) + config.RedirectHTTP = *redirectHTTP + config.HTTP2 = *useHTTP2 + + if *pagesRootCert != "" { + config.RootCertificate = readFile(*pagesRootCert) + } + + if *pagesRootKey != "" { + config.RootKey = readFile(*pagesRootKey) + } + + return config +} + func appMain() { var showVersion = flag.Bool("version", false, "Show version") var listenHTTP, listenHTTPS, listenProxy MultiStringFlag @@ -21,16 +51,6 @@ func appMain() { flag.Var(&listenHTTPS, "listen-https", "The address(es) to listen on for HTTPS requests") flag.Var(&listenProxy, "listen-proxy", "The address(es) to listen on for proxy requests") - var pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages") - var pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages") - var redirectHTTP = flag.Bool("redirect-http", true, "Serve the pages under HTTP") - var useHTTP2 = flag.Bool("use-http2", true, "Enable HTTP2 support") - var pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored") - var pagesDomain = flag.String("pages-domain", "gitlab-example.com", "The domain to serve static pages") - var metricsAdress = flag.String("metrics-address", "", "The adress to server metrics to") - var daemonUID = flag.Uint("daemon-uid", 0, "Drop privileges to this user") - var daemonGID = flag.Uint("daemon-gid", 0, "Drop privileges to this group") - flag.Parse() printVersion(*showVersion, VERSION) @@ -43,19 +63,7 @@ func appMain() { log.Fatalln(err) } - var config appConfig - config.Domain = strings.ToLower(*pagesDomain) - config.RedirectHTTP = *redirectHTTP - config.HTTP2 = *useHTTP2 - config.MetricsAddress = *metricsAdress - - if *pagesRootCert != "" { - config.RootCertificate = readFile(*pagesRootCert) - } - - if *pagesRootKey != "" { - config.RootKey = readFile(*pagesRootKey) - } + config := configFromFlags() for _, addr := range listenHTTP { l, fd := createSocket(addr) @@ -75,6 +83,12 @@ func appMain() { config.ListenProxy = append(config.ListenProxy, fd) } + if *metricsAddress != "" { + l, fd := createSocket(*metricsAddress) + defer l.Close() + config.ListenMetrics = fd + } + if *daemonUID != 0 || *daemonGID != 0 { daemonize(config, *daemonUID, *daemonGID) return diff --git a/metrics.go b/metrics/metrics.go index 8250fe3f..00e39460 100644 --- a/metrics.go +++ b/metrics/metrics.go @@ -1,42 +1,42 @@ -package main +package metrics import ( "github.com/prometheus/client_golang/prometheus" ) var ( - domainsServed = prometheus.NewGauge(prometheus.GaugeOpts{ + DomainsServed = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "gitlab_pages_domains_served_total", Help: "The total number of sites served by this Pages app", }) - domainUpdates = prometheus.NewCounter(prometheus.CounterOpts{ + DomainUpdates = prometheus.NewCounter(prometheus.CounterOpts{ Name: "gitlab_pages_domains_updated_total", Help: "The total number of site updates processed since daemon start", }) - domainLastUpdateTime = prometheus.NewCounter(prometheus.CounterOpts{ + DomainLastUpdateTime = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "gitlab_pages_last_domain_update_seconds", - Help: "Seconds since Unix Epoc to the last update for all domains served", + Help: "UNIX timestamp of the last update", }) - processedRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ + ProcessedRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "gitlab_pages_http_requests_total", Help: "Total number of HTTP requests done serving", }, []string{"code", "method"}, ) - sessionsActive = prometheus.NewGauge(prometheus.GaugeOpts{ + SessionsActive = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "gitlab_pages_http_sessions_active", Help: "The number of HTTP requests currently being processed", }) ) func init() { - prometheus.MustRegister(domainsServed) - prometheus.MustRegister(domainUpdates) - prometheus.MustRegister(domainLastUpdateTime) - prometheus.MustRegister(processedRequests) - prometheus.MustRegister(sessionsActive) + prometheus.MustRegister(DomainsServed) + prometheus.MustRegister(DomainUpdates) + prometheus.MustRegister(DomainLastUpdateTime) + prometheus.MustRegister(ProcessedRequests) + prometheus.MustRegister(SessionsActive) } diff --git a/vendor/golang.org/x/crypto/LICENSE b/vendor/golang.org/x/crypto/LICENSE deleted file mode 100644 index 6a66aea5..00000000 --- a/vendor/golang.org/x/crypto/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/crypto/PATENTS b/vendor/golang.org/x/crypto/PATENTS deleted file mode 100644 index 73309904..00000000 --- a/vendor/golang.org/x/crypto/PATENTS +++ /dev/null @@ -1,22 +0,0 @@ -Additional IP Rights Grant (Patents) - -"This implementation" means the copyrightable works distributed by -Google as part of the Go project. - -Google hereby grants to You a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable (except as stated in this section) -patent license to make, have made, use, offer to sell, sell, import, -transfer and otherwise run, modify and propagate the contents of this -implementation of Go, where such license applies only to those patent -claims, both currently owned or controlled by Google and acquired in -the future, licensable by Google that are necessarily infringed by this -implementation of Go. This grant does not include claims that would be -infringed only as a consequence of further modification of this -implementation. If you or your agent or exclusive licensee institute or -order or agree to the institution of patent litigation against any -entity (including a cross-claim or counterclaim in a lawsuit) alleging -that this implementation of Go or any code incorporated within this -implementation of Go constitutes direct or contributory patent -infringement, or inducement of patent infringement, then any patent -rights granted to you under this License for this implementation of Go -shall terminate as of the date such litigation is filed. |