benchmarking: Build profiling tools

Output from `ghz` provides raw latency numbers, but understanding where
Gitaly is spending its time requires more detail. To provide this, we
will profile it with Linux `perf` and `libbpf-tools`. The latter are
C versions of the older `bcc` BPF monitoring tools and are extremely
lightweight to run in terms of both memory and CPU. `bcc` uses a
python wrapper which can add significant load, particularly if multiple
tools are run at once.

`perf` output will be converted into flamegraphs.

3 files changed, 83 insertions, 0 deletions

diff --git a/_support/benchmarking/roles/gitaly/tasks/main.yml b/_support/benchmarking/roles/gitaly/tasks/main.yml
index 7f6fa7b3e..0f1a1f77f 100644
--- a/_support/benchmarking/roles/gitaly/tasks/main.yml
+++ b/_support/benchmarking/roles/gitaly/tasks/main.yml
@@ -8,3 +8,4 @@
 - include_tasks: setup_ruby.yml
 - include_tasks: setup_gitaly.yml
 - include_tasks: setup_gitlab_shell.yml
+- include_tasks: setup_profiling.yml
diff --git a/_support/benchmarking/roles/gitaly/tasks/setup_profiling.yml b/_support/benchmarking/roles/gitaly/tasks/setup_profiling.yml
new file mode 100644
index 000000000..e218ba60c
--- /dev/null
+++ b/_support/benchmarking/roles/gitaly/tasks/setup_profiling.yml
@@ -0,0 +1,81 @@
+---
+- name: Install perf
+  apt:
+    name:
+      - linux-tools-common
+      - "linux-tools-{{ ansible_kernel }}"
+      - "linux-headers-{{ ansible_kernel }}"
+    state: present
+
+- name: Install bcc dependencies
+  apt:
+    name:
+      - bison
+      - build-essential
+      - cmake
+      - clang
+      - flex
+      - git
+      - libedit-dev
+      - libllvm14
+      - llvm-14-dev
+      - libclang-14-dev
+      - python3
+      - zlib1g-dev
+      - libelf-dev
+      - libfl-dev
+      - python3-distutils
+    state: present
+
+- name: Clone bcc
+  git:
+    repo: https://github.com/iovisor/bcc.git
+    dest: /src/bcc
+    version: "{{ bcc_revision }}"
+  become_user: git
+
+# Required to build libbpf-tools
+- name: Symlink /usr/include/asm-generic to asm
+  file:
+    src: /usr/include/asm-generic
+    dest: /usr/include/asm
+    state: link
+
+# Required to build libbpf-tools
+- name: Symlink /usr/bin/llvm-strip-14 to llvm-strip
+  file:
+    src: /usr/bin/llvm-strip-14
+    dest: /usr/bin/llvm-strip
+    state: link
+
+- name: Make libbpf-tools
+  make:
+    jobs: "{{ ansible_processor_nproc }}"
+    chdir: /src/bcc/libbpf-tools
+  become_user: git
+
+- name: Install libbpf-tools
+  make:
+    target: install
+    jobs: "{{ ansible_processor_nproc }}"
+    chdir: /src/bcc/libbpf-tools
+
+- name: Clone Flamegraph
+  git:
+    repo: https://github.com/brendangregg/flamegraph.git
+    dest: /src/flamegraph
+  become_user: git
+
+- name: Install stackcollapse-perf.pl as stackcollapse
+  copy:
+    src: /src/flamegraph/stackcollapse-perf.pl
+    dest: /usr/local/bin/stackcollapse
+    mode: 0755
+    remote_src: true
+
+- name: Install flamegraph.pl as flamegraph
+  copy:
+    src: /src/flamegraph/flamegraph.pl
+    dest: /usr/local/bin/flamegraph
+    mode: 0755
+    remote_src: true
diff --git a/_support/benchmarking/roles/gitaly/vars/main.yml b/_support/benchmarking/roles/gitaly/vars/main.yml
index fe385e060..5d2ee7318 100644
--- a/_support/benchmarking/roles/gitaly/vars/main.yml
+++ b/_support/benchmarking/roles/gitaly/vars/main.yml
@@ -1,2 +1,3 @@
 ---
+bcc_revision: 88b5edbdc98a50dedf9a911b8f1ab5a63c574767
 ruby_build_version: 20221206