Add gitaly-blackbox prometheus exporter

4 files changed, 230 insertions, 0 deletions

diff --git a/internal/blackbox/blackbox.go b/internal/blackbox/blackbox.go
new file mode 100644
index 000000000..b3efcfa68
--- /dev/null
+++ b/internal/blackbox/blackbox.go
@@ -0,0 +1,73 @@
+package blackbox
+
+import (
+	"context"
+	"net"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"gitlab.com/gitlab-org/gitaly/internal/git/stats"
+	"gitlab.com/gitlab-org/gitaly/internal/version"
+	"gitlab.com/gitlab-org/labkit/monitoring"
+)
+
+func Run(cfg *Config) error {
+	listener, err := net.Listen("tcp", cfg.PrometheusListenAddr)
+	if err != nil {
+		return err
+	}
+
+	go runProbes(cfg)
+
+	return servePrometheus(listener)
+}
+
+func runProbes(cfg *Config) {
+	for ; ; time.Sleep(cfg.SleepDuration) {
+		for _, probe := range cfg.Probes {
+			doProbe(probe)
+		}
+	}
+}
+
+func servePrometheus(l net.Listener) error {
+	return monitoring.Serve(
+		monitoring.WithListener(l),
+		monitoring.WithBuildInformation(version.GetVersion(), version.GetBuildTime()),
+	)
+}
+
+func doProbe(probe Probe) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	entry := log.WithField("probe", probe.Name)
+	entry.Info("starting probe")
+
+	clone := &stats.Clone{
+		URL:      probe.URL,
+		User:     probe.User,
+		Password: probe.Password,
+	}
+
+	if err := clone.Perform(ctx); err != nil {
+		entry.WithError(err).Error("probe failed")
+		return
+	}
+
+	entry.Info("finished probe")
+
+	setGauge := func(gv *prometheus.GaugeVec, value float64) {
+		gv.WithLabelValues(probe.Name).Set(value)
+	}
+
+	setGauge(getFirstPacket, clone.Get.FirstGitPacket().Seconds())
+	setGauge(getTotalTime, clone.Get.ResponseBody().Seconds())
+	setGauge(getAdvertisedRefs, float64(len(clone.Get.Refs())))
+	setGauge(wantedRefs, float64(clone.RefsWanted()))
+	setGauge(postTotalTime, clone.Post.ResponseBody().Seconds())
+	setGauge(postFirstProgressPacket, clone.Post.BandFirstPacket("progress").Seconds())
+	setGauge(postFirstPackPacket, clone.Post.BandFirstPacket("pack").Seconds())
+	setGauge(postPackBytes, float64(clone.Post.BandPayloadSize("pack")))
+}
diff --git a/internal/blackbox/config.go b/internal/blackbox/config.go
new file mode 100644
index 000000000..3074f6ee4
--- /dev/null
+++ b/internal/blackbox/config.go
@@ -0,0 +1,65 @@
+package blackbox
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	logconfig "gitlab.com/gitlab-org/gitaly/internal/config/log"
+)
+
+type Config struct {
+	PrometheusListenAddr string `toml:"prometheus_listen_addr"`
+	Sleep                int    `toml:"sleep"`
+	SleepDuration        time.Duration
+	Logging              logconfig.Config `toml:"logging"`
+	Probes               []Probe          `toml:"probe"`
+}
+
+type Probe struct {
+	Name     string `toml:"name"`
+	URL      string `toml:"url"`
+	User     string `toml:"user"`
+	Password string `toml:"password"`
+}
+
+func ParseConfig(raw string) (*Config, error) {
+	config := &Config{}
+	if _, err := toml.Decode(raw, config); err != nil {
+		return nil, err
+	}
+
+	if config.PrometheusListenAddr == "" {
+		return nil, fmt.Errorf("missing prometheus_listen_addr")
+	}
+
+	if config.Sleep < 0 {
+		return nil, fmt.Errorf("sleep time is less than 0")
+	}
+	if config.Sleep == 0 {
+		config.Sleep = 15 * 60
+	}
+	config.SleepDuration = time.Duration(config.Sleep) * time.Second
+
+	if len(config.Probes) == 0 {
+		return nil, fmt.Errorf("must define at least one probe")
+	}
+
+	for _, probe := range config.Probes {
+		if len(probe.Name) == 0 {
+			return nil, fmt.Errorf("all probes must have a 'name' attribute")
+		}
+
+		parsedURL, err := url.Parse(probe.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		if s := parsedURL.Scheme; s != "http" && s != "https" {
+			return nil, fmt.Errorf("unsupported probe URL scheme: %v", probe.URL)
+		}
+	}
+
+	return config, nil
+}
diff --git a/internal/blackbox/config_test.go b/internal/blackbox/config_test.go
new file mode 100644
index 000000000..fb41a0f25
--- /dev/null
+++ b/internal/blackbox/config_test.go
@@ -0,0 +1,63 @@
+package blackbox
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfigParseFailures(t *testing.T) {
+	testCases := []struct {
+		desc string
+		in   string
+	}{
+		{desc: "empty config"},
+		{desc: "probe without name", in: "[[probe]]\n"},
+		{desc: "unsupported probe url", in: "[[probe]]\nname='foo'\nurl='ssh://not:supported'"},
+		{desc: "missing probe url", in: "[[probe]]\nname='foo'\n"},
+		{desc: "negative sleep", in: "sleep=-1\n[[probe]]\nname='foo'\nurl='http://foo/bar'"},
+		{desc: "no listen addr", in: "[[probe]]\nname='foo'\nurl='http://foo/bar'"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			_, err := ParseConfig(tc.in)
+
+			require.Error(t, err, "expect parse error")
+		})
+	}
+}
+
+func TestConfigSleep(t *testing.T) {
+	testCases := []struct {
+		desc string
+		in   string
+		out  time.Duration
+	}{
+		{
+			desc: "default sleep time",
+			out:  15 * time.Minute,
+		},
+		{
+			desc: "1 second",
+			in:   "sleep = 1\n",
+			out:  time.Second,
+		},
+	}
+
+	const validConfig = `
+prometheus_listen_addr = ':9687'
+[[probe]]
+name = 'foo'
+url = 'http://foo/bar'
+`
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			cfg, err := ParseConfig(tc.in + validConfig)
+			require.NoError(t, err, "parse config")
+
+			require.Equal(t, tc.out, cfg.SleepDuration, "parsed sleep time")
+		})
+	}
+}
diff --git a/internal/blackbox/prometheus.go b/internal/blackbox/prometheus.go
new file mode 100644
index 000000000..6eacbb7c4
--- /dev/null
+++ b/internal/blackbox/prometheus.go
@@ -0,0 +1,29 @@
+package blackbox
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	getFirstPacket          = newGauge("get_first_packet_seconds", "Time to first Git packet in GET /info/refs reponse")
+	getTotalTime            = newGauge("get_total_time_seconds", "Time to receive entire GET /info/refs reponse")
+	getAdvertisedRefs       = newGauge("get_advertised_refs", "Number of Git refs advertised in GET /info/refs")
+	wantedRefs              = newGauge("wanted_refs", "Number of Git refs selected for (fake) Git clone (branches + tags)")
+	postTotalTime           = newGauge("post_total_time_seconds", "Time to receive entire POST /upload-pack reponse")
+	postFirstProgressPacket = newGauge("post_first_progress_packet_seconds", "Time to first progress band Git packet in POST /upload-pack response")
+	postFirstPackPacket     = newGauge("post_first_pack_packet_seconds", "Time to first pack band Git packet in POST /upload-pack response")
+	postPackBytes           = newGauge("post_pack_bytes", "Number of pack band bytes in POST /upload-pack response")
+)
+
+func newGauge(name string, help string) *prometheus.GaugeVec {
+	return promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "gitaly_blackbox",
+			Subsystem: "git_http",
+			Name:      name,
+			Help:      help,
+		},
+		[]string{"probe"},
+	)
+}