Merge branch 'instrumented-cluster' into 'master'

Docker-compose cluster useful for profiling and perfomance testing

See merge request gitlab-org/gitaly!464

7 files changed, 192 insertions, 0 deletions

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 47d053181..a2bf0daf9 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -200,3 +200,42 @@ DEBU[0037] 13:04:31.074736 git.c:322               trace: built-in: git 'worktre
 DEBU[0037] 13:04:31.076135 run-command.c:626       trace: run_command: 'rerere' 'gc'  grpc.method=GarbageCollect grpc.request.repoPath="gitlab/gitlab-design.git" grpc.request.repoStorage=default grpc.request.topLevelGroup=gitlab grpc.service=gitaly.RepositoryService peer.address= span.kind=server system=grpc
 DEBU[0037] 13:04:31.079286 git.c:322               trace: built-in: git 'rerere' 'gc'  grpc.method=GarbageCollect grpc.request.repoPath="gitlab/gitlab-design.git" grpc.request.repoStorage=default grpc.request.topLevelGroup=gitlab grpc.service=gitaly.RepositoryService peer.address= span.kind=server system=grpc
 ```
+
+## Testing with Instrumentation
+
+If you would like to test with instrumentation and prometheus metrics, use the `instrumented-cluster` docker compose configuration in
+`_support/instrumented-cluster`. This cluster will create several services:
+
+|*Service*|*Endpoint*|
+|---------|------|
+| Gitaly | [http://localhost:9999](http://localhost:9999) |
+| Gitaly Metrics and pprof | [http://localhost:9236](http://localhost:9236) |
+| Prometheus | [http://localhost:9090](http://localhost:9090) |
+| cAdvisor | [http://localhost:8080](http://localhost:8080) |
+| Grafana | [http://localhost:3000](http://localhost:3000) use default login `admin`/`admin` |
+
+The gitaly service uses the `gitlab/gitaly:latest` image, which you need to build using `make docker` before starting the cluster.
+
+Once you have the `gitlab/gitaly:latest` image, start the cluster from the `_support/instrumented-cluster` directory using:
+
+```shell
+docker-compose up --remove-orphans
+```
+
+Enter `^C` to kill the cluster.
+
+Note that the Gitaly service is intentionally limited to 50% CPU and 200MB of memory. This can be adjusted in the `docker-compose.yml` file.
+
+Once the cluster has started, it will clone the `gitlab-org/gitlab-ce` repository, for testing purposes.
+
+This can then be used for testing, using tools like [gitaly-bench](https://gitlab.com/gitlab-org/gitaly-bench):
+
+```shell
+gitaly-bench -concurrency 100 -repo gitlab-org/gitlab-ce.git find-all-branches
+```
+
+It can also be used with profiling tools, for example [go-torch](https://github.com/uber/go-torch) for generating flame graphs, as follows:
+
+```shell
+go-torch --url http://localhost:9236 -p > flamegraph.svg
+```
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 000000000..8e38b6c6c
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,27 @@
+
+#
+# This will create a docker image for Gitaly that is suitable for testing, but
+# is not expected to be used in a production environment, yet.
+#
+# See the _support/load-cluster/docker-compose.yml for an example of how to use
+# this image
+#
+FROM registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.3-golang-1.8-git-2.13
+
+RUN mkdir -p /app/ruby
+
+RUN git clone https://gitlab.com/gitlab-org/gitlab-shell.git /app/gitlab-shell
+
+COPY ./ruby/Gemfile /app/ruby/
+COPY ./ruby/Gemfile.lock /app/ruby/
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update -qq && \
+    apt-get install -qq -y rubygems bundler cmake build-essential libicu-dev && \
+    cd /app/ruby && bundle install --path vendor/bundle && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY . /app
+
+CMD ["/app/bin/gitaly", "/app/config/config.toml"]
+
diff --git a/Makefile b/Makefile
index 81b39db55..a50f44ca8 100644
--- a/Makefile
+++ b/Makefile
@@ -78,6 +78,19 @@ assemble: force-ruby-bundle build
 	cp -r ruby $(ASSEMBLY_ROOT)/ruby
 	install $(foreach cmd,$(COMMANDS),$(BIN_BUILD_DIR)/$(cmd)) $(ASSEMBLY_ROOT)/bin
 
+docker: $(TARGET_SETUP)
+	rm -rf $(TARGET_DIR)/docker/
+	mkdir -p $(TARGET_DIR)/docker/bin/
+	cp -r ruby $(TARGET_DIR)/docker/ruby/
+	rm -rf $(TARGET_DIR)/docker/ruby/vendor/bundle
+
+	for cmd in $(COMMAND_PACKAGES); do \
+		GOOS=linux GOARCH=amd64 go build $(GO_LDFLAGS) -o "$(TARGET_DIR)/docker/bin/$$(basename $$cmd)" $$cmd; \
+	done
+
+	cp Dockerfile $(TARGET_DIR)/docker/
+	docker build -t gitlab/gitaly:$(VERSION_PREFIXED) -t gitlab/gitaly:latest $(TARGET_DIR)/docker/
+
 .PHONY: verify
 verify: lint check-formatting megacheck govendor-status notice-up-to-date
 
diff --git a/_support/instrumented-cluster/docker-compose.yml b/_support/instrumented-cluster/docker-compose.yml
new file mode 100644
index 000000000..d8b217a50
--- /dev/null
+++ b/_support/instrumented-cluster/docker-compose.yml
@@ -0,0 +1,66 @@
+version: "2.3"
+
+services:
+  gitaly1repos:
+    image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.3-golang-1.8-git-2.13
+    volumes:
+      - gitalydata1:/repositories
+      - ./gitaly1repos/setup.sh:/setup.sh
+    restart: 'no'
+    command: bash /setup.sh
+
+  gitaly1:
+    image: gitlab/gitaly:latest
+    ports:
+      - "9999:9999"
+      - "9236:9236"
+    environment:
+      - TZ=UTC
+    volumes:
+      - ./gitaly1/config.toml:/app/config/config.toml
+    volumes_from:
+      - gitaly1repos
+    cpus: 0.5
+    mem_limit: 200m
+    restart: unless-stopped
+
+  cadvisor:
+    image: google/cadvisor:latest
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    ports:
+      - 8080:8080
+    restart: unless-stopped
+
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - promdata:/prometheus
+      - ./prom/prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - 9090:9090
+    links:
+      - gitaly1
+      - cadvisor
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana
+    volumes:
+      - grafdata:/var/lib/grafana
+    ports:
+      - 3000:3000
+    links:
+      - prometheus
+    restart: unless-stopped
+
+volumes:
+  gitalydata1:
+    driver: local
+  grafdata:
+    driver: local
+  promdata:
+    driver: local
diff --git a/_support/instrumented-cluster/gitaly1/config.toml b/_support/instrumented-cluster/gitaly1/config.toml
new file mode 100644
index 000000000..73a4ed7c7
--- /dev/null
+++ b/_support/instrumented-cluster/gitaly1/config.toml
@@ -0,0 +1,29 @@
+listen_addr = ":9999"
+prometheus_listen_addr = ":9236"
+bin_dir = "/app/bin"
+
+# # Git executable settings
+[git]
+bin_path = "/usr/local/bin/git"
+
+[[storage]]
+name = "default"
+path = "/repositories/"
+
+# # You can optionally configure Gitaly to record histogram latencies on GRPC method calls
+[prometheus]
+grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0]
+
+[gitaly-ruby]
+# The directory where gitaly-ruby is installed
+dir = "/app/ruby"
+
+[gitlab-shell]
+# The directory where gitlab-shell is installed
+dir = "/app/gitlab-shell"
+
+[[concurrency]]
+rpc = "/gitaly.RepositoryService/GarbageCollect"
+max_per_repo = 1
+
+
diff --git a/_support/instrumented-cluster/gitaly1repos/setup.sh b/_support/instrumented-cluster/gitaly1repos/setup.sh
new file mode 100644
index 000000000..101b1bee9
--- /dev/null
+++ b/_support/instrumented-cluster/gitaly1repos/setup.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+git clone --bare https://gitlab.com/gitlab-org/gitlab-ce.git /repositories/gitlab-org/gitlab-ce.git
diff --git a/_support/instrumented-cluster/prom/prometheus.yml b/_support/instrumented-cluster/prom/prometheus.yml
new file mode 100644
index 000000000..5cb6c61d8
--- /dev/null
+++ b/_support/instrumented-cluster/prom/prometheus.yml
@@ -0,0 +1,15 @@
+scrape_configs:
+  - job_name: 'cadvisor'
+    scrape_interval: 10s
+    scrape_timeout: 5s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['cadvisor:8080']
+  - job_name: 'gitaly1'
+    scrape_interval: 10s
+    scrape_timeout: 5s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['gitaly1:9236']