Merge branch '267-track-api-calls' into 'master'

Provide a way to track and measure API calls

Closes #267

See merge request gitlab-org/gitlab-pages!229

6 files changed, 169 insertions, 18 deletions

diff --git a/internal/httptransport/transport.go b/internal/httptransport/transport.go
index 6d946ae0..ba2aa5eb 100644
--- a/internal/httptransport/transport.go
+++ b/internal/httptransport/transport.go
@@ -7,8 +7,11 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"sync"
+	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -16,7 +19,7 @@ var (
 	sysPoolOnce = &sync.Once{}
 	sysPool     *x509.CertPool
 
-	// Transport can be used with httpclient with TLS and certificates
+	// Transport can be used with http.Client with TLS and certificates
 	Transport = &http.Transport{
 		DialTLS: func(network, addr string) (net.Conn, error) {
 			return tls.Dial(network, addr, &tls.Config{RootCAs: pool()})
@@ -25,6 +28,22 @@ var (
 	}
 )
 
+type meteredRoundTripper struct {
+	next      http.RoundTripper
+	durations *prometheus.GaugeVec
+	counter   *prometheus.CounterVec
+}
+
+// NewTransportWithMetrics will create a custom http.RoundTripper that can be used with an http.Client.
+// The RoundTripper will report metrics based on the collectors passed.
+func NewTransportWithMetrics(gaugeVec *prometheus.GaugeVec, counterVec *prometheus.CounterVec) http.RoundTripper {
+	return &meteredRoundTripper{
+		next:      Transport,
+		durations: gaugeVec,
+		counter:   counterVec,
+	}
+}
+
 // This is here because macOS does not support the SSL_CERT_FILE
 // environment variable. We have arrange things to read SSL_CERT_FILE as
 // late as possible to avoid conflicts with file descriptor passing at
@@ -55,3 +74,21 @@ func loadPool() {
 
 	sysPool.AppendCertsFromPEM(certPem)
 }
+
+// withRoundTripper takes an original RoundTripper, reports metrics based on the
+// gauge and counter collectors passed
+func (mrt *meteredRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	start := time.Now()
+
+	resp, err := mrt.next.RoundTrip(r)
+	if err != nil {
+		mrt.counter.WithLabelValues("error").Inc()
+		return nil, err
+	}
+
+	statusCode := strconv.Itoa(resp.StatusCode)
+	mrt.durations.WithLabelValues(statusCode).Set(time.Since(start).Seconds())
+	mrt.counter.WithLabelValues(statusCode).Inc()
+
+	return resp, nil
+}
diff --git a/internal/httptransport/transport_test.go b/internal/httptransport/transport_test.go
new file mode 100644
index 00000000..cfb8d708
--- /dev/null
+++ b/internal/httptransport/transport_test.go
@@ -0,0 +1,91 @@
+package httptransport
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_withRoundTripper(t *testing.T) {
+
+	tests := []struct {
+		name       string
+		statusCode int
+		err        error
+	}{
+		{
+			name:       "successful_response",
+			statusCode: http.StatusNoContent,
+		},
+		{
+			name:       "error_response",
+			statusCode: http.StatusForbidden,
+		},
+		{
+			name:       "internal_error_response",
+			statusCode: http.StatusInternalServerError,
+		},
+		{
+			name:       "unhandled_status_response",
+			statusCode: http.StatusPermanentRedirect,
+		},
+		{
+			name: "client_error",
+			err:  fmt.Errorf("something went wrong"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+				Name: t.Name(),
+			}, []string{"status_code"})
+
+			counterVec := prometheus.NewCounterVec(prometheus.CounterOpts{
+				Name: t.Name(),
+			}, []string{"status_code"})
+
+			next := &mockRoundTripper{
+				res: &http.Response{
+					StatusCode: tt.statusCode,
+				},
+				err: tt.err,
+			}
+
+			mtr := &meteredRoundTripper{next, gaugeVec, counterVec}
+			r := httptest.NewRequest("GET", "/", nil)
+
+			res, err := mtr.RoundTrip(r)
+			if tt.err != nil {
+				counterCount := testutil.ToFloat64(counterVec.WithLabelValues("error"))
+				require.Equal(t, float64(1), counterCount, "error")
+
+				return
+			}
+			require.NoError(t, err)
+			require.NotNil(t, res)
+
+			statusCode := strconv.Itoa(res.StatusCode)
+			gaugeValue := testutil.ToFloat64(gaugeVec.WithLabelValues(statusCode))
+			require.Greater(t, gaugeValue, float64(0))
+
+			counterCount := testutil.ToFloat64(counterVec.WithLabelValues(statusCode))
+			require.Equal(t, float64(1), counterCount, statusCode)
+		})
+	}
+}
+
+type mockRoundTripper struct {
+	res *http.Response
+	err error
+}
+
+func (mrt *mockRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	return mrt.res, mrt.err
+}
diff --git a/internal/source/disk/map.go b/internal/source/disk/map.go
index bae4b764..5c053fcb 100644
--- a/internal/source/disk/map.go
+++ b/internal/source/disk/map.go
@@ -252,7 +252,7 @@ func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) {
 		fis, err := godirwalk.ReadDirents(".", nil)
 		if err != nil {
 			log.WithError(err).Warn("domain scan failed")
-			metrics.FailedDomainUpdates.Inc()
+			metrics.DomainFailedUpdates.Inc()
 			continue
 		}
 
diff --git a/internal/source/gitlab/client/client.go b/internal/source/gitlab/client/client.go
index 59776a8f..afe9da6f 100644
--- a/internal/source/gitlab/client/client.go
+++ b/internal/source/gitlab/client/client.go
@@ -11,10 +11,11 @@ import (
 	"net/url"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/dgrijalva/jwt-go"
 
 	"gitlab.com/gitlab-org/gitlab-pages/internal/httptransport"
 	"gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api"
+	"gitlab.com/gitlab-org/gitlab-pages/metrics"
 )
 
 // Client is a HTTP client to access Pages internal API
@@ -32,7 +33,7 @@ func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpi
 		return nil, errors.New("GitLab API URL or API secret has not been provided")
 	}
 
-	url, err := url.Parse(baseURL)
+	parsedURL, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
 	}
@@ -47,10 +48,10 @@ func NewClient(baseURL string, secretKey []byte, connectionTimeout, jwtTokenExpi
 
 	return &Client{
 		secretKey: secretKey,
-		baseURL:   url,
+		baseURL:   parsedURL,
 		httpClient: &http.Client{
 			Timeout:   connectionTimeout,
-			Transport: httptransport.Transport,
+			Transport: httptransport.NewTransportWithMetrics(metrics.DomainsSourceAPICallDuration, metrics.DomainsSourceAPIReqTotal),
 		},
 		jwtTokenExpiry: jwtTokenExpiry,
 	}, nil
@@ -115,6 +116,8 @@ func (gc *Client) get(ctx context.Context, path string, params url.Values) (*htt
 		return resp, nil
 	}
 
+	// nolint: errcheck
+	// best effort to discard and close the response body
 	io.Copy(ioutil.Discard, resp.Body)
 	resp.Body.Close()
 
diff --git a/main.go b/main.go
index ea96a096..9a316c5e 100644
--- a/main.go
+++ b/main.go
@@ -19,6 +19,7 @@ import (
 	"gitlab.com/gitlab-org/gitlab-pages/internal/host"
 	"gitlab.com/gitlab-org/gitlab-pages/internal/logging"
 	"gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig"
+	"gitlab.com/gitlab-org/gitlab-pages/metrics"
 )
 
 // VERSION stores the information about the semantic version of application
@@ -397,6 +398,8 @@ func main() {
 
 	rand.Seed(time.Now().UnixNano())
 
+	metrics.MustRegister()
+
 	daemonMain()
 	appMain()
 }
diff --git a/metrics/metrics.go b/metrics/metrics.go
index 1ae527a8..7ae50c81 100644
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -11,8 +11,8 @@ var (
 		Help: "The number of sites served by this Pages app",
 	})
 
-	// FailedDomainUpdates counts the number of failed site updates
-	FailedDomainUpdates = prometheus.NewCounter(prometheus.CounterOpts{
+	// DomainFailedUpdates counts the number of failed site updates
+	DomainFailedUpdates = prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "gitlab_pages_domains_failed_total",
 		Help: "The total number of site updates that have failed since daemon start",
 	})
@@ -58,16 +58,33 @@ var (
 		Name: "gitlab_pages_serverless_latency",
 		Help: "Serverless serving roundtrip duration",
 	})
+
+	// DomainsSourceAPIReqTotal is the number of calls made to the GitLab API that returned a 4XX error
+	DomainsSourceAPIReqTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "gitlab_pages_domains_source_api_requests_total",
+		Help: "The number of GitLab domains API calls with different status codes",
+	}, []string{"status_code"})
+
+	// DomainsSourceAPICallDuration is the time it takes to get a response from the GitLab API in seconds
+	DomainsSourceAPICallDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "gitlab_pages_domains_source_api_call_duration",
+		Help: "The time (in seconds) it takes to get a response from the GitLab domains API",
+	}, []string{"status_code"})
 )
 
-func init() {
-	prometheus.MustRegister(DomainsServed)
-	prometheus.MustRegister(FailedDomainUpdates)
-	prometheus.MustRegister(DomainUpdates)
-	prometheus.MustRegister(DomainLastUpdateTime)
-	prometheus.MustRegister(DomainsConfigurationUpdateDuration)
-	prometheus.MustRegister(DomainsSourceCacheHit)
-	prometheus.MustRegister(DomainsSourceCacheMiss)
-	prometheus.MustRegister(ServerlessRequests)
-	prometheus.MustRegister(ServerlessLatency)
+// MustRegister collectors with the Prometheus client
+func MustRegister() {
+	prometheus.MustRegister(
+		DomainsServed,
+		DomainFailedUpdates,
+		DomainUpdates,
+		DomainLastUpdateTime,
+		DomainsConfigurationUpdateDuration,
+		DomainsSourceCacheHit,
+		DomainsSourceCacheMiss,
+		DomainsSourceAPIReqTotal,
+		DomainsSourceAPICallDuration,
+		ServerlessRequests,
+		ServerlessLatency,
+	)
 }