Run benchstats on CI

7 files changed, 119 insertions, 4 deletions

diff --git a/.gitignore b/.gitignore
index db328895..cb6db57f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ shared/pages/.update
 # Used by the makefile
 /.GOPATH
 /bin
+/bench.*.txt
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fe193eb6..03317fec 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,8 +17,16 @@ stages:
     paths:
       - .GOPATH/pkg/mod/
 
+.base-job:
+  only:
+    refs:
+      - merge_requests
+      - master
+
 .tests:
-  extends: .go-mod-cache
+  extends:
+    - .base-job
+    - .go-mod-cache
   stage: test
   needs: ['download deps']
   script:
@@ -36,12 +44,24 @@ license_management:
   stage: prepare
   variables:
     LICENSE_MANAGEMENT_SETUP_CMD: go mod vendor
+  only:
+    refs:
+      - merge_requests
+      - master
+    changes:
+      - go.{mod,sum}
 
 sast:
   stage: prepare
+  only:
+    refs:
+      - merge_requests
+      - master
 
 download deps:
-  extends: .go-mod-cache
+  extends:
+    - .base-job
+    - .go-mod-cache
   stage: prepare
   script:
     - make deps-download
@@ -51,7 +71,9 @@ download deps:
       - go.sum
 
 verify:
-  extends: .go-mod-cache
+  extends:
+    - .base-job
+    - .go-mod-cache
   stage: test
   needs: ['download deps']
   script:
@@ -84,8 +106,25 @@ race:
   - make race
 
 check deps:
-  extends: .go-mod-cache
+  extends:
+    - .base-job
+    - .go-mod-cache
   stage: test
   needs: ['download deps']
   script:
     - make deps-check
+
+benchmark:
+  extends:
+    - .base-job
+    - .go-mod-cache
+  stage: test
+  needs: ['download deps']
+  script:
+    - make setup
+    - make benchstat
+  artifacts:
+    when: always
+    expire_in: 1 week
+    paths:
+      - bench.*.txt
diff --git a/Makefile.build.mk b/Makefile.build.mk
index 24c2ec39..a3d45be8 100644
--- a/Makefile.build.mk
+++ b/Makefile.build.mk
@@ -4,6 +4,7 @@ all: gitlab-pages
 
 setup: clean .GOPATH/.ok
 	go get golang.org/x/tools/cmd/goimports@v0.0.0-20191010201905-e5ffc44a6fee
+	go get golang.org/x/perf/cmd/benchstat@v0.0.0-20191209155426-36b577b0eb03
 	go get golang.org/x/lint/golint@v0.0.0-20190930215403-16217165b5de
 	go get github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
 	go get github.com/fzipp/gocyclo@v0.0.0-20150627053110-6acd4345c835
diff --git a/Makefile.internal.mk b/Makefile.internal.mk
index e7d0e8fe..f77d401b 100644
--- a/Makefile.internal.mk
+++ b/Makefile.internal.mk
@@ -47,3 +47,6 @@ bin/goimports: .GOPATH/.ok
 bin/golint: .GOPATH/.ok
 	@test -x $@ || \
 	    { echo "Vendored golint not found, try running 'make setup'..."; exit 1; }
+bin/benchstat: .GOPATH/.ok
+	@test -x $@ || \
+	    { echo "Vendored golint not found, try running 'make setup'..."; exit 1; }
diff --git a/Makefile.util.mk b/Makefile.util.mk
index 412b7655..31ebe64c 100644
--- a/Makefile.util.mk
+++ b/Makefile.util.mk
@@ -26,6 +26,9 @@ acceptance: .GOPATH/.ok gitlab-pages
 bench: .GOPATH/.ok gitlab-pages
 	go test -bench=. -run=^$$ $(allpackages)
 
+benchstat: .GOPATH/.ok bin/benchstat
+	@_support/benchmark
+
 # The acceptance tests cannot count for coverage
 cover: bin/gocovmerge .GOPATH/.ok gitlab-pages
 	@echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!"
diff --git a/_support/benchmark b/_support/benchmark
new file mode 100755
index 00000000..04d23db5
--- /dev/null
+++ b/_support/benchmark
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -e
+
+bench_runs=3
+new_code_stats="bench.new.txt"
+pkgs="gitlab.com/gitlab-org/gitlab-pages/internal/domain"
+
+bench_loop() {
+    times="$1"
+    file="$2"
+
+    go test -bench=. "$pkgs" > "$file"
+
+    for _ in $(seq 2 "$times"); do
+        go test -bench=. "$pkgs" >> "$file"
+    done
+}
+
+echo "Running benchmarks $bench_runs times"
+bench_loop "$bench_runs" "$new_code_stats"
+./bin/benchstat "$new_code_stats"
+
+ if [ -n "$CI" ]; then
+    old_code_stats="bench.old.txt"
+    target_branch=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master}
+
+    echo
+    echo "CI detected - running in comparison mode"
+
+    git checkout -q -f "$target_branch"
+
+    echo "Running benchmarks on $target_branch $bench_runs times"
+    bench_loop "$bench_runs" "$old_code_stats"
+
+    git checkout -q -f "$CI_COMMIT_SHA"
+
+    _support/benchstat "$old_code_stats" "$new_code_stats"
+fi
+
+# -*- mode: sh; -*-
diff --git a/_support/benchstat b/_support/benchstat
new file mode 100755
index 00000000..b5516236
--- /dev/null
+++ b/_support/benchstat
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -e
+
+old_run=$1
+new_run=$2
+
+echo "Benchmarks comparison"
+./bin/benchstat -delta-test none "$old_run" "$new_run"
+
+first_line=$(./bin/benchstat -delta-test none -sort delta -csv "$old_run" "$new_run" | head -2 | tail -1)
+delta=$(echo "$first_line" | cut -d , -f 6 | cut -d % -f 1)
+
+if [ -z "$delta" ]; then
+    echo "Nothing to compare!"
+    exit 1
+fi
+
+[ "$delta" = "~" ] && exit 0
+
+limit=0.5
+if echo "$delta" "$limit" | awk '{exit !( $1 > $2)}'; then
+    echo "Performance degradation detected"
+    exit 1
+fi
+
+# -*- mode: sh; -*-