diff options
author | Jaime Martinez <jmartinez@gitlab.com> | 2020-02-11 11:39:25 +0300 |
---|---|---|
committer | Alessio Caiazza <acaiazza@gitlab.com> | 2020-02-11 11:39:25 +0300 |
commit | 2b0661b4ff4fe112798f03f62861096104b9e373 (patch) | |
tree | 6a6fe4dc6f0d499fbce4ae2b6e5a151cfef6eea1 | |
parent | 81db00b64dd2d4c9aec85fc58e92374e806732d9 (diff) |
Add prometheus metrics for GitLab API client
Refactor metrics initialization removing init function
from the metrics package.
-rw-r--r-- | internal/httptransport/transport.go | 39 | ||||
-rw-r--r-- | internal/httptransport/transport_test.go | 91 | ||||
-rw-r--r-- | internal/source/disk/map.go | 2 | ||||
-rw-r--r-- | internal/source/gitlab/client/client.go | 11 | ||||
-rw-r--r-- | main.go | 3 | ||||
-rw-r--r-- | metrics/metrics.go | 41 |
6 files changed, 169 insertions, 18 deletions
diff --git a/internal/httptransport/transport.go b/internal/httptransport/transport.go index 6d946ae0..ba2aa5eb 100644 --- a/internal/httptransport/transport.go +++ b/internal/httptransport/transport.go @@ -7,8 +7,11 @@ import ( "net" "net/http" "os" + "strconv" "sync" + "time" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -16,7 +19,7 @@ var ( sysPoolOnce = &sync.Once{} sysPool *x509.CertPool - // Transport can be used with httpclient with TLS and certificates + // Transport can be used with http.Client with TLS and certificates Transport = &http.Transport{ DialTLS: func(network, addr string) (net.Conn, error) { return tls.Dial(network, addr, &tls.Config{RootCAs: pool()}) @@ -25,6 +28,22 @@ var ( } ) +type meteredRoundTripper struct { + next http.RoundTripper + durations *prometheus.GaugeVec + counter *prometheus.CounterVec +} + +// NewTransportWithMetrics will create a custom http.RoundTripper that can be used with an http.Client. +// The RoundTripper will report metrics based on the collectors passed. +func NewTransportWithMetrics(gaugeVec *prometheus.GaugeVec, counterVec *prometheus.CounterVec) http.RoundTripper { + return &meteredRoundTripper{ + next: Transport, + durations: gaugeVec, + counter: counterVec, + } +} + // This is here because macOS does not support the SSL_CERT_FILE // environment variable. We have arrange things to read SSL_CERT_FILE as // late as possible to avoid conflicts with file descriptor passing at @@ -55,3 +74,21 @@ func loadPool() { sysPool.AppendCertsFromPEM(certPem) } + +// withRoundTripper takes an original RoundTripper, reports metrics based on the +// gauge and counter collectors passed +func (mrt *meteredRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + start := time.Now() + + resp, err := mrt.next.RoundTrip(r) + if err != nil { + mrt.counter.WithLabelValues("error").Inc() + return nil, err + } + + statusCode := strconv.Itoa(resp.StatusCode) + mrt.durations.WithLabelValues(statusCode).Set(time.Since(start).Seconds()) + mrt.counter.WithLabelValues(statusCode).Inc() + + return resp, nil +} diff --git a/internal/httptransport/transport_test.go b/internal/httptransport/transport_test.go new file mode 100644 index 00000000..cfb8d708 --- /dev/null +++ b/internal/httptransport/transport_test.go @@ -0,0 +1,91 @@ +package httptransport + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func Test_withRoundTripper(t *testing.T) { + + tests := []struct { + name string + statusCode int + err error + }{ + { + name: "successful_response", + statusCode: http.StatusNoContent, + }, + { + name: "error_response", + statusCode: http.StatusForbidden, + }, + { + name: "internal_error_response", + statusCode: http.StatusInternalServerError, + }, + { + name: "unhandled_status_response", + statusCode: http.StatusPermanentRedirect, + }, + { + name: "client_error", + err: fmt.Errorf("something went wrong"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: t.Name(), + }, []string{"status_code"}) + + counterVec := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: t.Name(), + }, []string{"status_code"}) + + next := &mockRoundTripper{ + res: &http.Response{ + StatusCode: tt.statusCode, + }, + err: tt.err, + } + + mtr := &meteredRoundTripper{next, gaugeVec, counterVec} + r := httptest.NewRequest("GET", "/", nil) + + res, err := mtr.RoundTrip(r) + if tt.err != nil { + counterCount := testutil.ToFloat64(counterVec.WithLabelValues("error")) + require.Equal(t, float64(1), counterCount, "error") + + return + } + require.NoError(t, err) + require.NotNil(t, res) + + statusCode := strconv.Itoa(res.StatusCode) + gaugeValue := testutil.ToFloat64(gaugeVec.WithLabelValues(statusCode)) + require.Greater(t, gaugeValue, float64(0)) + + counterCount := testutil.ToFloat64(counterVec.WithLabelValues(statusCode)) + require.Equal(t, float64(1), counterCount, statusCode) + }) + } +} + +type mockRoundTripper struct { + res *http.Response + err error +} + +func (mrt *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + return mrt.res, mrt.err +} diff --git a/internal/source/disk/map.go b/internal/source/disk/map.go index bae4b764..5c053fcb 100644 --- a/internal/source/disk/map.go +++ b/internal/source/disk/map.go @@ -252,7 +252,7 @@ func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) { fis, err := godirwalk.ReadDirents(".", nil) if err != nil { log.WithError(err).Warn("domain scan failed") - metrics.FailedDomainUpdates.Inc() + metrics.DomainFailedUpdates.Inc() continue } diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go index 59776a8f..afe9da6f 100644 --- a/internal/source/gitlab/client/client.go +++ b/internal/source/gitlab/client/client.go @@ -11,10 +11,11 @@ import ( "net/url" "time" - jwt "github.com/dgrijalva/jwt-go" + "github.com/dgrijalva/jwt-go" "gitlab.com/gitlab-org/gitlab-pages/internal/httptransport" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api" + "gitlab.com/gitlab-org/gitlab-pages/metrics" ) // Client is a HTTP client to access Pages internal API @@ -32,7 +33,7 @@ func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpi return nil, errors.New("GitLab API URL or API secret has not been provided") } - url, err := url.Parse(baseURL) + parsedURL, err := url.Parse(baseURL) if err != nil { return nil, err } @@ -47,10 +48,10 @@ func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpi return &Client{ secretKey: secretKey, - baseURL: url, + baseURL: parsedURL, httpClient: &http.Client{ Timeout: connectionTimeout, - Transport: httptransport.Transport, + Transport: httptransport.NewTransportWithMetrics(metrics.DomainsSourceAPICallDuration, metrics.DomainsSourceAPIReqTotal), }, jwtTokenExpiry: jwtTokenExpiry, }, nil @@ -115,6 +116,8 @@ func (gc *Client) get(ctx context.Context, path string, params url.Values) (*htt return resp, nil } + // nolint: errcheck + // best effort to discard and close the response body io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() @@ -19,6 +19,7 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/host" "gitlab.com/gitlab-org/gitlab-pages/internal/logging" "gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig" + "gitlab.com/gitlab-org/gitlab-pages/metrics" ) // VERSION stores the information about the semantic version of application @@ -397,6 +398,8 @@ func main() { rand.Seed(time.Now().UnixNano()) + metrics.MustRegister() + daemonMain() appMain() } diff --git a/metrics/metrics.go b/metrics/metrics.go index 1ae527a8..7ae50c81 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,8 +11,8 @@ var ( Help: "The number of sites served by this Pages app", }) - // FailedDomainUpdates counts the number of failed site updates - FailedDomainUpdates = prometheus.NewCounter(prometheus.CounterOpts{ + // DomainFailedUpdates counts the number of failed site updates + DomainFailedUpdates = prometheus.NewCounter(prometheus.CounterOpts{ Name: "gitlab_pages_domains_failed_total", Help: "The total number of site updates that have failed since daemon start", }) @@ -58,16 +58,33 @@ var ( Name: "gitlab_pages_serverless_latency", Help: "Serverless serving roundtrip duration", }) + + // DomainsSourceAPIReqTotal is the number of calls made to the GitLab API that returned a 4XX error + DomainsSourceAPIReqTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "gitlab_pages_domains_source_api_requests_total", + Help: "The number of GitLab domains API calls with different status codes", + }, []string{"status_code"}) + + // DomainsSourceAPICallDuration is the time it takes to get a response from the GitLab API in seconds + DomainsSourceAPICallDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gitlab_pages_domains_source_api_call_duration", + Help: "The time (in seconds) it takes to get a response from the GitLab domains API", + }, []string{"status_code"}) ) -func init() { - prometheus.MustRegister(DomainsServed) - prometheus.MustRegister(FailedDomainUpdates) - prometheus.MustRegister(DomainUpdates) - prometheus.MustRegister(DomainLastUpdateTime) - prometheus.MustRegister(DomainsConfigurationUpdateDuration) - prometheus.MustRegister(DomainsSourceCacheHit) - prometheus.MustRegister(DomainsSourceCacheMiss) - prometheus.MustRegister(ServerlessRequests) - prometheus.MustRegister(ServerlessLatency) +// MustRegister collectors with the Prometheus client +func MustRegister() { + prometheus.MustRegister( + DomainsServed, + DomainFailedUpdates, + DomainUpdates, + DomainLastUpdateTime, + DomainsConfigurationUpdateDuration, + DomainsSourceCacheHit, + DomainsSourceCacheMiss, + DomainsSourceAPIReqTotal, + DomainsSourceAPICallDuration, + ServerlessRequests, + ServerlessLatency, + ) } |