Merge branch 'jarv/benchmark-fixes' into 'master'

Updates benchmarking scripts for testing different disk types

See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5423

Merged-by: Will Chandler <wchandler@gitlab.com>
Approved-by: Will Chandler <wchandler@gitlab.com>
Reviewed-by: John Jarvis <jarv@gitlab.com>
Reviewed-by: Will Chandler <wchandler@gitlab.com>
Co-authored-by: John Jarvis <jarv@gitlab.com>

14 files changed, 103 insertions, 40 deletions

diff --git a/_support/benchmarking/.gitignore b/_support/benchmarking/.gitignore
index 1380e5950..7b1864e65 100644
--- a/_support/benchmarking/.gitignore
+++ b/_support/benchmarking/.gitignore
@@ -1,5 +1,7 @@
 /hosts.ini
 /terraform/*
 !/terraform/main.tf
+!/terraform/locals.tf
 /group_vars/all.yml
 /results/*
+config.yml
diff --git a/_support/benchmarking/.tool-versions b/_support/benchmarking/.tool-versions
new file mode 100644
index 000000000..db57b0f15
--- /dev/null
+++ b/_support/benchmarking/.tool-versions
@@ -0,0 +1,2 @@
+terraform 1.3.9
+python 3.11.2
diff --git a/_support/benchmarking/README.md b/_support/benchmarking/README.md
index 76367e6c5..bc439f19f 100644
--- a/_support/benchmarking/README.md
+++ b/_support/benchmarking/README.md
@@ -4,19 +4,20 @@
 
 An Ansible script for running RPC-level benchmarks against Gitaly.
 
-## Required tools
+**Note**: You must be a member of the `gitaly-benchmark-0150d6cf` GCP group.
 
-The following programs must be installed locally to run the script:
-
-- Ansible 2.14 or above
-- Terraform 1.2 or above
-- gcloud
+## Steps for use
 
-You must be a member of the `gitaly-benchmark-0150d6cf` GCP group.
+### 1. Setup your environment
 
-## Steps for use
+1. Ensure that [`gcloud`](https://cloud.google.com/sdk/docs/install) is installed and available on your path.
+1. Ensure that `python` and `terraform` are installed, or use [`asdf`](https://asdf-vm.com/guide/getting-started.html) to install them (recommended).
+1. Create a new Python virtualenv: `python3 -m venv env`
+1. Activate the virtualenv: `source env/bin/activate`
+1. Install Ansible: `python3 -m pip install -r requirements.txt`
+1. **Optional**: Copy `config.yml.example` to `config.yml` to customize the machine type uses for benchmarking
 
-### 1. Create instance
+### 2. Create instance
 
 ```shell
 ./create-benchmark-instance
@@ -32,7 +33,7 @@ Use the `gitaly_bench` user to SSH into the instance if desired:
 ssh gitaly_bench@<INSTANCE_ADDRESS>
 ```
 
-### 2. Configure instance
+### 3. Configure instance
 
 ```shell
 ./configure-benchmark-instance
@@ -42,7 +43,7 @@ Build and install Gitaly from source with from desired reference and install
 profiling tools like `perf` and `libbpf-tools`. A disk image containing the
 test repositories will be mounted to `/mnt/git-repositories` on the Gitaly node.
 
-### 3. Run benchmarks
+### 4. Run benchmarks
 
 ```shell
 ./run-benchmarks
@@ -86,7 +87,7 @@ When profiling is enabled, the following are also present:
   This uses `--call-graph=fp` for accurate stack traces for Golang.
 - `page-cachestat.txt` - Kernel page cache hit rate.
 
-### 4. Destroy instance
+### 5. Destroy instance
 
 ```shell
 ./destroy-benchmark-instance
diff --git a/_support/benchmarking/ansible.cfg b/_support/benchmarking/ansible.cfg
index 9eec3508d..64c5f01f3 100644
--- a/_support/benchmarking/ansible.cfg
+++ b/_support/benchmarking/ansible.cfg
@@ -3,3 +3,4 @@ pipelining = True
 
 [defaults]
 callbacks_enabled = profile_tasks
+interpreter_python = auto_silent
diff --git a/_support/benchmarking/config.yml.example b/_support/benchmarking/config.yml.example
new file mode 100644
index 000000000..30ba48f2e
--- /dev/null
+++ b/_support/benchmarking/config.yml.example
@@ -0,0 +1,15 @@
+---
+project: "gitaly-benchmark-0150d6cf"
+gitaly_machine_type: "t2d-standard-4"
+client_machine_type: "n1-standard-1"
+os_image: "ubuntu-os-cloud/ubuntu-2204-lts"
+repository_disk_type: "pd-balanced"
+boot_disk_size: 20
+benchmark_region: "us-central1"
+benchmark_zone: "us-central1-a"
+
+# Enable to use regional persistent disk https://cloud.google.com/compute/docs/disks/regional-persistent-disk
+# Regional PD is supported on only E2, N1, N2, and N2D machine type VMs.
+use_regional_disk: false
+# One of these zones must be the zone for benchmark_zone
+regional_disk_replica_zones: ["us-central1-a", "us-central1-b"]
diff --git a/_support/benchmarking/create-benchmark-instance b/_support/benchmarking/create-benchmark-instance
index 0173ac095..a104556dc 100755
--- a/_support/benchmarking/create-benchmark-instance
+++ b/_support/benchmarking/create-benchmark-instance
@@ -1,2 +1,2 @@
 #!/bin/sh
-exec ansible-playbook -l localhost create.yml "$@"
+exec ansible-playbook -i localhost, create.yml "$@"
diff --git a/_support/benchmarking/create.yml b/_support/benchmarking/create.yml
index 1e4942ba2..a47092408 100644
--- a/_support/benchmarking/create.yml
+++ b/_support/benchmarking/create.yml
@@ -1,6 +1,7 @@
 ---
 - name: Set user-defined variables
   hosts: localhost
+  connection: local
   vars_prompt:
     - name: gitaly_revision
       prompt: "Enter Gitaly revision to build"
diff --git a/_support/benchmarking/env/.gitignore b/_support/benchmarking/env/.gitignore
new file mode 100644
index 000000000..d6b7ef32c
--- /dev/null
+++ b/_support/benchmarking/env/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/_support/benchmarking/requirements.txt b/_support/benchmarking/requirements.txt
new file mode 100644
index 000000000..07d2c59e2
--- /dev/null
+++ b/_support/benchmarking/requirements.txt
@@ -0,0 +1,10 @@
+ansible==7.2.0
+ansible-core==2.14.2
+cffi==1.15.1
+cryptography==39.0.1
+Jinja2==3.1.2
+MarkupSafe==2.1.2
+packaging==23.0
+pycparser==2.21
+PyYAML==6.0
+resolvelib==0.8.1
diff --git a/_support/benchmarking/roles/client/tasks/main.yml b/_support/benchmarking/roles/client/tasks/main.yml
index 712161a04..660f5aff0 100644
--- a/_support/benchmarking/roles/client/tasks/main.yml
+++ b/_support/benchmarking/roles/client/tasks/main.yml
@@ -1,4 +1,11 @@
 ---
+# needed for Ansible > 4.0 https://github.com/ansible/ansible/issues/74830
+- name: Install acl
+  apt:
+    name:
+      - acl
+    state: present
+
 - name: Create 'git' user
   user:
     name: git
diff --git a/_support/benchmarking/roles/gitaly/tasks/initialize.yml b/_support/benchmarking/roles/gitaly/tasks/initialize.yml
index e5e5e1795..f88ab089b 100644
--- a/_support/benchmarking/roles/gitaly/tasks/initialize.yml
+++ b/_support/benchmarking/roles/gitaly/tasks/initialize.yml
@@ -4,6 +4,13 @@
     user: root
     key: "{{ hostvars[groups['client'][0]]['client_ssh_key'] | b64decode }}"
 
+# needed for Ansible > 4.0 https://github.com/ansible/ansible/issues/74830
+- name: Install acl
+  apt:
+    name:
+      - acl
+    state: present
+
 # Set high uid to ensure it's not squashed by another service
 - name: Create 'git' user
   user:
diff --git a/_support/benchmarking/roles/gitaly/tasks/setup_go.yml b/_support/benchmarking/roles/gitaly/tasks/setup_go.yml
index afafb220a..d4fb0478a 100644
--- a/_support/benchmarking/roles/gitaly/tasks/setup_go.yml
+++ b/_support/benchmarking/roles/gitaly/tasks/setup_go.yml
@@ -12,3 +12,7 @@
     dest: /usr/local
     remote_src: true
     creates: /usr/local/go/bin/go
+  retries: 3
+  delay: 3
+  register: result
+  until: result is success
diff --git a/_support/benchmarking/terraform/locals.tf b/_support/benchmarking/terraform/locals.tf
new file mode 100644
index 000000000..56cd4bf41
--- /dev/null
+++ b/_support/benchmarking/terraform/locals.tf
@@ -0,0 +1,3 @@
+locals {
+  config = try(yamldecode(file("../${path.root}/config.yml")), yamldecode(file("../${path.root}/config.yml.example")))
+}
diff --git a/_support/benchmarking/terraform/main.tf b/_support/benchmarking/terraform/main.tf
index c5c3f4045..2dc528600 100644
--- a/_support/benchmarking/terraform/main.tf
+++ b/_support/benchmarking/terraform/main.tf
@@ -1,60 +1,68 @@
-variable "project" { default = "gitaly-benchmark-0150d6cf" }
-variable "benchmark_region" { default = "us-central1" }
-variable "benchmark_zone" { default = "us-central1-a" }
-variable "gitaly_benchmarking_instance_name" { }
-variable "ssh_pubkey" { }
-variable "os_image" { default = "ubuntu-os-cloud/ubuntu-2204-lts" }
+variable "gitaly_benchmarking_instance_name" {}
+variable "ssh_pubkey" {}
 variable "startup_script" {
   default = <<EOF
     set -e
     if [ -d /src/gitaly ] ; then exit; fi
   EOF
 }
-variable "gitaly_machine_type" { default = "t2d-standard-4" }
-variable "client_machine_type" { default = "n1-standard-1" }
-variable "boot_disk_size" { default = "20" }
 
 provider "google" {
-  project = var.project
-  region  = var.benchmark_region
-  zone    = var.benchmark_zone
+  project = local.config.project
+  region  = local.config.benchmark_region
+  zone    = local.config.benchmark_zone
 }
 
 data "google_compute_disk" "repository-disk" {
-  name = "git-repos"
-  project = "gitaly-benchmark-0150d6cf"
+  name    = "git-repos"
+  project = local.config.project
 }
 
 resource "google_compute_disk" "repository-disk" {
-  name = format("%s-repository-disk", var.gitaly_benchmarking_instance_name)
-  type = "pd-balanced"
-  image = format("projects/%s/global/images/git-repositories", var.project)
+  name  = format("%s-repository-disk", var.gitaly_benchmarking_instance_name)
+  type  = local.config.repository_disk_type
+  image = format("projects/%s/global/images/git-repositories", local.config.project)
+}
+
+resource "google_compute_region_disk" "repository-region-disk" {
+  count         = local.config.use_regional_disk ? 1 : 0
+  name          = format("%s-repository-region-disk", var.gitaly_benchmarking_instance_name)
+  type          = local.config.repository_disk_type
+  snapshot      = google_compute_snapshot.repository-disk[0].id
+  replica_zones = local.config.regional_disk_replica_zones
+}
+
+resource "google_compute_snapshot" "repository-disk" {
+  count       = local.config.use_regional_disk ? 1 : 0
+  name        = format("%s-repository-snapshot", var.gitaly_benchmarking_instance_name)
+  source_disk = google_compute_disk.repository-disk.name
+  zone        = local.config.benchmark_zone
 }
 
 resource "google_compute_instance" "gitaly" {
   name         = format("%s-gitaly", var.gitaly_benchmarking_instance_name)
-  machine_type = var.gitaly_machine_type
+  machine_type = local.config.gitaly_machine_type
 
   boot_disk {
     initialize_params {
-      image = var.os_image
-      size = var.boot_disk_size
+      image = local.config.os_image
+      size  = local.config.boot_disk_size
     }
   }
 
   attached_disk {
-    source = google_compute_disk.repository-disk.name
+    source      = local.config.use_regional_disk ? google_compute_region_disk.repository-region-disk[0].self_link : google_compute_disk.repository-disk.self_link
     device_name = "repository-disk"
   }
 
   network_interface {
-    network = "default"
+    network    = "default"
     subnetwork = "default"
     access_config {}
   }
 
   metadata = {
-    ssh-keys = format("gitaly_bench:%s", var.ssh_pubkey)
+    ssh-keys       = format("gitaly_bench:%s", var.ssh_pubkey)
     startup-script = <<EOF
       ${var.startup_script}
     EOF
@@ -69,12 +77,12 @@ resource "google_compute_instance" "gitaly" {
 
 resource "google_compute_instance" "client" {
   name         = format("%s-client", var.gitaly_benchmarking_instance_name)
-  machine_type = var.client_machine_type
+  machine_type = local.config.client_machine_type
 
   boot_disk {
     initialize_params {
-      image = var.os_image
-      size = var.boot_disk_size
+      image = local.config.os_image
+      size  = local.config.boot_disk_size
     }
   }
 
@@ -84,7 +92,7 @@ resource "google_compute_instance" "client" {
   }
 
   metadata = {
-    ssh-keys = format("gitaly_bench:%s", var.ssh_pubkey)
+    ssh-keys       = format("gitaly_bench:%s", var.ssh_pubkey)
     startup-script = <<EOF
       ${var.startup_script}
     EOF