Merge branch 'toon-no-moar-ruby-scripts' into 'master'

tools: Rewrite Ruby scripts in Go

See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5635

Merged-by: Will Chandler <wchandler@gitlab.com>
Approved-by: Will Chandler <wchandler@gitlab.com>
Reviewed-by: Will Chandler <wchandler@gitlab.com>
Reviewed-by: John Cai <jcai@gitlab.com>
Co-authored-by: Toon Claes <toon@gitlab.com>

5 files changed, 287 insertions, 132 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4540c2436..c872b6f90 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ stages:
   - qa
 
 default:
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:git-2.36
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:git-2.36
   tags:
     - gitlab-org
   interruptible: true
@@ -112,13 +112,7 @@ workflow:
     # directory.
     - setpriv --reuid=${TEST_UID} --regid=${TEST_UID} --clear-groups --no-new-privs make ${TEST_TARGET} $(test "${GIT_VERSION}" = default && echo WITH_BUNDLED_GIT=YesPlease)
   after_script: &test_after_script
-    - |
-      # Checking for panics in ${TEST_FULL_OUTPUT}
-      if [ "${CI_JOB_STATUS}" = "failed" ] && grep 'Output":"panic' "${TEST_FULL_OUTPUT}" > /dev/null; then
-        echo -e "\e[0Ksection_start:`date +%s`:panic_stack_traces[collapsed=true]\r\e[0K\e[0;31mPanic stack traces\e[0m"
-        ruby -e "require 'json'; f = File.read(ENV['TEST_FULL_OUTPUT']); f.lines.each do |l| out = JSON.parse(l); puts out['Output']; end" | awk '/^panic/ || /goroutine/,/^\s*$/'
-        echo -e "\e[0Ksection_end:`date +%s`:panic_stack_traces\r\e[0K"
-      fi
+    - go run "${CI_PROJECT_DIR}/tools/panic-parser"
   artifacts: &test_artifacts
     paths:
       - ${TEST_LOG_DIR}
@@ -166,7 +160,7 @@ build:
     # for our "test" targets, which also run unprivileged.
     - install --directory --owner=${TEST_UID} --group=${TEST_UID} _build
     - setpriv --reuid=${TEST_UID} --regid=${TEST_UID} --clear-groups --no-new-privs make build $(test "${GIT_VERSION}" = default && echo build-bundled-git || echo build-git)
-    - _support/test-boot . ${TEST_BOOT_ARGS}
+    - go run ./tools/test-boot ${TEST_BOOT_ARGS} .
   parallel:
     matrix:
       - GO_VERSION: [ "1.18", "1.19" ]
@@ -274,7 +268,7 @@ test:praefect_smoke:
   <<: *test_definition
   script:
     - make -j$(nproc)
-    - ruby -rerb -e 'ERB.new(ARGF.read).run' _support/config.praefect.toml.ci-sql-test.erb > config.praefect.toml
+    - envsubst < _support/config.praefect.toml.ci-sql-test > config.praefect.toml
     - ./_build/bin/praefect -config config.praefect.toml sql-ping
     - ./_build/bin/praefect -config config.praefect.toml sql-migrate
 
@@ -287,7 +281,7 @@ test:sha256:
 
 test:fips:
   <<: *test_definition
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images/ubi-${UBI_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:git-2.36
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images/ubi-${UBI_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:git-2.36
   tags:
     - fips
   cache:
diff --git a/_support/config.praefect.toml.ci-sql-test.erb b/_support/config.praefect.toml.ci-sql-test
index fdd1081c1..5ac5a7963 100755
--- a/_support/config.praefect.toml.ci-sql-test.erb
+++ b/_support/config.praefect.toml.ci-sql-test
@@ -10,7 +10,7 @@ listen_addr = "127.0.0.1:2305"
 
 [database]
   # The following variables are defined in .gitlab-ci.yml.
-  host = '<%= ENV['PGHOST'] %>'
-  dbname = '<%= ENV['POSTGRES_DB'] %>'
-  user = '<%= ENV['PGUSER'] %>'
+  host = '${PGHOST}'
+  dbname = '${POSTGRES_DB}'
+  user = '${PGUSER}'
   sslmode = 'disable'
diff --git a/_support/test-boot b/_support/test-boot
deleted file mode 100755
index 6678a89a8..000000000
--- a/_support/test-boot
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'tempfile'
-require 'socket'
-require 'optparse'
-
-ADDR = 'socket'
-
-def main(params) # rubocop:disable Metrics/MethodLength
-  gitaly_dir = File.realpath(params[:dir])
-  build_dir = File.join(gitaly_dir, '_build')
-  bin_dir = File.join(build_dir, 'bin')
-
-  git_path, use_bundled_git =
-    if params[:"bundled-git"]
-      ['', true]
-    else
-      [File.join(build_dir, 'deps', 'git-distribution', 'bin-wrappers', 'git'), false]
-    end
-
-  version = IO.popen("#{File.join(bin_dir, 'gitaly')} -version").read.delete_prefix('Gitaly, version ').strip
-  version_from_file = IO.read(File.join(gitaly_dir, 'VERSION')).strip
-
-  # Use start_with? instead of == because the version output could use git describe, if it is a source install
-  # eg: Gitaly, version 1.75.0-14-gd1ecb43f
-  unless version.start_with?(version_from_file)
-    abort "\nversion check failed: VERSION file contained '#{version_from_file}'"\
-      " but 'gitaly -version' reported '#{version}'."\
-      ' If you are working from a fork, please fetch the latest tags.'
-  end
-
-  Dir.mktmpdir do |dir|
-    Dir.chdir(dir)
-
-    gitlab_shell_dir = File.join(dir, 'gitlab-shell')
-    Dir.mkdir(gitlab_shell_dir)
-    File.write(File.join(gitlab_shell_dir, '.gitlab_shell_secret'), 'test_gitlab_shell_token')
-    write_gitaly_config('config.toml',
-                        bin_dir: bin_dir,
-                        dir: dir,
-                        use_bundled_git: use_bundled_git,
-                        git_path: git_path,
-                        gitaly_dir: gitaly_dir,
-                        gitlab_shell_dir: gitlab_shell_dir)
-
-    pid = nil
-
-    begin
-      start = Time.now
-      pid = spawn(File.join(bin_dir, 'gitaly'), 'config.toml')
-      wait_connect
-      puts
-      puts "\n\nconnection established after #{Time.now - start} seconds\n\n"
-    ensure
-      if pid
-        Process.kill('KILL', pid)
-        Process.wait(pid)
-      end
-    end
-  end
-end
-
-def write_gitaly_config(config_path, bin_dir:, dir:, use_bundled_git:, git_path:, gitaly_dir:, gitlab_shell_dir:) # rubocop:disable Metrics/ParameterLists
-  File.write(config_path, <<~CONFIG
-    socket_path = "#{ADDR}"
-    bin_dir = "#{bin_dir}"
-
-    [[storage]]
-    name = "default"
-    path = "#{dir}"
-
-    [git]
-    use_bundled_binaries = #{use_bundled_git}
-    bin_path = "#{git_path}"
-
-    [gitaly-ruby]
-    dir = "#{gitaly_dir}/ruby"
-
-    [gitlab-shell]
-    dir = "#{gitlab_shell_dir}"
-
-    [gitlab]
-    url = 'http://gitlab_url'
-
-  CONFIG
-  )
-end
-
-def wait_connect
-  repeats = 100
-  sleep_time = 0.1
-
-  repeats.times do
-    Socket.unix(ADDR)
-    return
-  rescue StandardError
-    print '.'
-    sleep(sleep_time)
-  end
-
-  puts "failed to connect to gitaly after #{repeats * sleep_time}s"
-
-  abort
-end
-
-params = {}
-OptionParser.new do |parser|
-  parser.banner = "Usage: #{$0} [options] <GITALY_DIR>"
-  parser.on('--[no-]bundled-git', 'Set up Gitaly with bundled Git binaries')
-end.parse!(into: params)
-
-params[:dir] = ARGV.pop
-abort 'Gitaly source directory not provided' if params[:dir].nil?
-
-abort 'Extra arguments' unless ARGV.count.zero?
-
-main(params)
diff --git a/tools/panic-parser/main.go b/tools/panic-parser/main.go
new file mode 100644
index 000000000..f5f444753
--- /dev/null
+++ b/tools/panic-parser/main.go
@@ -0,0 +1,60 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+)
+
+// LogLine defines the relevant fields we want to parse from the json output.
+type LogLine struct {
+	Action, Output string
+}
+
+func main() {
+	logFile := os.Getenv("TEST_FULL_OUTPUT")
+	fmt.Printf("# Checking for panics in %s\n", logFile)
+
+	if os.Getenv("CI_JOB_STATUS") != "failed" {
+		return
+	}
+
+	f, err := os.Open(logFile)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+
+	decoder := json.NewDecoder(f)
+
+	var printIt bool
+	var inBacktrace bool
+	for decoder.More() {
+		var line LogLine
+		if err := decoder.Decode(&line); err != nil {
+			return
+		}
+		if line.Action != "output" {
+			if inBacktrace {
+				inBacktrace = false
+				fmt.Println("") // Print a trailing new line to separate panics.
+			}
+			continue
+		}
+		if strings.HasPrefix(line.Output, "panic:") {
+			inBacktrace = true
+			if !printIt {
+				fmt.Printf("\x1b[0Ksection_start:%v:panic_stack_traces[collapsed=true]\r\x1b[0K\x1b[0;31mPanic stack traces\x1b[0m\n", time.Now().Unix())
+			}
+			printIt = true
+		}
+		if inBacktrace {
+			fmt.Printf(line.Output)
+		}
+	}
+	if printIt {
+		fmt.Printf("\x1b[0Ksection_end:%v:panic_stack_traces\r\x1b[0K\n", time.Now().Unix())
+	}
+}
diff --git a/tools/test-boot/main.go b/tools/test-boot/main.go
new file mode 100644
index 000000000..ae748292d
--- /dev/null
+++ b/tools/test-boot/main.go
@@ -0,0 +1,219 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+)
+
+type gitalyConfig struct {
+	SocketPath     string
+	BinDir         string
+	Dir            string
+	UseBundledGit  bool
+	GitPath        string
+	GitalyDir      string
+	GitlabShellDir string
+}
+
+const configTemplate = `
+socket_path = "{{.SocketPath}}"
+bin_dir = "{{.BinDir}}"
+
+[[storage]]
+name = "default"
+path = "{{.Dir}}"
+
+[git]
+use_bundled_binaries = {{.UseBundledGit}}
+bin_path = "{{.GitPath}}"
+
+[gitlab-shell]
+dir = "{{.GitlabShellDir}}"
+
+[gitlab]
+url = 'http://gitlab_url'
+`
+
+func parseArgs() (string, bool, error) {
+	useBundledGit := flag.Bool("bundled-git", false, "Set up Gitaly with bundled Git binaries")
+	notUseBundledGit := flag.Bool("no-bundled-git", false, "Set up Gitaly without bundled Git binaries")
+
+	flag.Parse()
+	if flag.NArg() == 0 {
+		usage()
+		return "", false, fmt.Errorf("gitaly source directory not provided")
+	}
+	if flag.NArg() > 1 {
+		usage()
+		return "", false, fmt.Errorf("extra arguments")
+	}
+
+	if *useBundledGit && *notUseBundledGit {
+		usage()
+		return "", false, fmt.Errorf("use either --bundled-git or --no-bundled-git, not both")
+	}
+
+	gitalyDir, err := filepath.Abs(flag.Args()[0])
+	if err != nil {
+		return "", false, fmt.Errorf("failed to parse dir %v: %w", flag.Args()[0], err)
+	}
+
+	return gitalyDir, *useBundledGit, nil
+}
+
+func checkVersion(gitalyDir, gitalyBin string) error {
+	versionCmd := exec.Command(gitalyBin, "-version")
+	versionOutput, err := versionCmd.Output()
+	if err != nil {
+		return fmt.Errorf("failed to get Gitaly version output: %w", err)
+	}
+
+	version := strings.TrimSpace(strings.TrimPrefix(string(versionOutput), "Gitaly, version "))
+
+	versionFromFile, err := os.ReadFile(filepath.Join(gitalyDir, "VERSION"))
+	if err != nil {
+		return fmt.Errorf("failed to read Gitaly version file: %w", err)
+	}
+	// Use strings.HasPrefix() because the version output could use git describe, if it is a source install
+	// e.g.: Gitaly, version 1.75.0-14-gd1ecb43f
+	if !strings.HasPrefix(version, string(versionFromFile)) {
+		return fmt.Errorf("version check failed: VERSION file contained %q\n"+ //nolint:stylecheck
+			"but 'gitaly -version' reported %q.\n"+
+			"If you are working from a fork, please fetch the latest tags.\n",
+			versionFromFile, version)
+	}
+
+	return nil
+}
+
+func writeGitalyConfig(path string, params gitalyConfig) error {
+	t, err := template.New("config").Parse(configTemplate)
+	if err != nil {
+		return fmt.Errorf("parse template: %w", err)
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("create Gitaly config file: %w", err)
+	}
+	defer f.Close()
+
+	if err := t.Execute(f, params); err != nil {
+		return fmt.Errorf("generate Gitaly config file: %w", err)
+	}
+
+	return nil
+}
+
+func spawnAndWait(gitalyBin, configPath, socketPath string) (returnedError error) {
+	cmd := exec.Command(gitalyBin, configPath)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("start gitaly: %w", err)
+	}
+
+	defer func() {
+		_ = cmd.Process.Kill()
+		_ = cmd.Wait()
+		if returnedError != nil {
+			fmt.Fprintf(os.Stdout, "%s\n", stdout.String())
+			fmt.Fprintf(os.Stderr, "%s\n", stderr.String())
+		}
+	}()
+
+	start := time.Now()
+	for i := 0; i < 100; i++ {
+		conn, err := net.Dial("unix", socketPath)
+		if err == nil {
+			fmt.Printf("\n\nconnection established after %v\n\n", time.Since(start))
+			conn.Close()
+			return nil
+		}
+
+		fmt.Printf(".")
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	fmt.Println("")
+
+	return fmt.Errorf("failed to connect to gitaly after %v", time.Since(start))
+}
+
+func testBoot() error {
+	gitalyDir, useBundledGit, err := parseArgs()
+	if err != nil {
+		return err
+	}
+
+	buildDir := filepath.Join(gitalyDir, "_build")
+	binDir := filepath.Join(buildDir, "bin")
+
+	gitPath := filepath.Join(buildDir, "deps", "git-distribution", "bin-wrappers", "git")
+	if useBundledGit {
+		gitPath = ""
+	}
+
+	gitalyBin := filepath.Join(binDir, "gitaly")
+	if err := checkVersion(gitalyDir, gitalyBin); err != nil {
+		return err
+	}
+
+	tempDir, err := os.MkdirTemp("", "gitaly-test-boot")
+	if err != nil {
+		return fmt.Errorf("create temp directory: %w", err)
+	}
+
+	gitlabShellDir := filepath.Join(tempDir, "gitlab-shell")
+	if err := os.Mkdir(gitlabShellDir, 0o755); err != nil {
+		return fmt.Errorf("create gitlab-shell directory: %w", err)
+	}
+
+	err = os.WriteFile(filepath.Join(gitlabShellDir, ".gitlab_shell_secret"), []byte("test_gitlab_shell_token"), 0o644)
+	if err != nil {
+		return fmt.Errorf("write gitlab-shell secret: %w", err)
+	}
+
+	socketPath := filepath.Join(tempDir, "socket")
+	configPath := filepath.Join(tempDir, "config.toml")
+	err = writeGitalyConfig(configPath,
+		gitalyConfig{
+			SocketPath:     socketPath,
+			BinDir:         binDir,
+			Dir:            tempDir,
+			UseBundledGit:  useBundledGit,
+			GitPath:        gitPath,
+			GitalyDir:      gitalyDir,
+			GitlabShellDir: gitlabShellDir,
+		})
+	if err != nil {
+		return nil
+	}
+
+	if err := spawnAndWait(gitalyBin, configPath, socketPath); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	if err := testBoot(); err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "Usage: %s [options] <GITALY_DIR>\n", filepath.Base(os.Args[0]))
+	fmt.Fprintf(os.Stderr, "  --[no-]bundled-git    Set up Gitaly with bundled Git binaries\n\n")
+}