Merge branch 'pks-ci-winter-cleanup' into 'master'

ci: Winter cleanup

See merge request gitlab-org/gitaly!4136

2 files changed, 91 insertions, 173 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3eff79849..9ff92d543 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,11 +1,10 @@
 stages:
   - build
   - test
-  - publish
   - qa
 
 default:
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-golang-1.16-git-2.31
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
   tags:
     - gitlab-org
 
@@ -27,30 +26,6 @@ include:
   - template: Security/Coverage-Fuzzing.gitlab-ci.yml
   - template: Security/Secret-Detection.gitlab-ci.yml
 
-danger-review:
-  image: ruby:2.7
-  allow_failure: true
-  stage: build
-  only:
-    - merge_requests
-  except:
-    - tags
-    - master
-  variables:
-    BUNDLE_GEMFILE: danger/Gemfile
-  before_script:
-    - bundle install
-  script:
-    - >
-      if [ -z "$DANGER_GITLAB_API_TOKEN" ]; then
-        # Force danger to skip CI source GitLab and fallback to "local only git repo".
-        unset GITLAB_CI
-        # We need access to the base SHA to help danger determine the base commit for this shallow clone.
-        bundle exec danger dry_run --fail-on-errors=true --verbose --base='$CI_MERGE_REQUEST_DIFF_BASE_SHA'
-      else
-        bundle exec danger --fail-on-errors=true --verbose
-      fi
-
 .cache: &cache_definition
   cache: &cache_configuration
     key:
@@ -66,62 +41,51 @@ danger-review:
 .test_template: &test_definition
   <<: *cache_definition
   stage: test
-  artifacts:
-    paths:
-    - ruby/tmp/gitaly-rspec-test.log
-    when: on_failure
-    expire_in: 1 week
-
-.postgres_template: &postgres_definition
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
   services:
     - postgres:${POSTGRES_VERSION}
   variables: &postgres_variables
     PGHOST: postgres
-    PGPORT: "5432"
+    PGPORT: 5432
     PGUSER: postgres
+    POSTGRES_DB: praefect_test
     POSTGRES_HOST_AUTH_METHOD: trust
   before_script:
     - go version
     - git version
     - while ! psql -h $PGHOST -U $PGUSER -c 'SELECT 1' > /dev/null; do echo "awaiting Postgres service to be ready..." && sleep 1 ; done && echo "Postgres service is ready!"
+  artifacts:
+    paths:
+      - ruby/tmp/gitaly-rspec-test.log
+    reports:
+      junit: _build/reports/go-tests-report-go-${GO_VERSION}-git-${GIT_VERSION}.xml
+    when: on_failure
+    expire_in: 1 week
 
-.pgbouncer_template: &pgbouncer_definition
-  <<: *postgres_definition
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31-pgbouncer-1.14
+danger-review:
+  stage: build
+  allow_failure: true
+  only:
+    - merge_requests
+  except:
+    - tags
+    - master
   variables:
-    <<: *postgres_variables
-    PGHOST_PGBOUNCER: 0.0.0.0
-    PGPORT_PGBOUNCER: "6432"
+    BUNDLE_GEMFILE: danger/Gemfile
   before_script:
-    - go version
-    - git version
-    - pgbouncer --version
-    - adduser --no-create-home --disabled-password --disabled-login --quiet --force-badname --gecos '' pgbouncer-runner
-    - su pgbouncer-runner -c 'pgbouncer internal/praefect/datastore/glsql/testdata/pgbouncer.ini' &
-    - for i in {1..10}; do psql -U $PGUSER -c 'select now()' && break; done || { echo 'pgbouncer awaiting failed' ; exit 1; }
-
-verify:
-  <<: *cache_definition
-  stage: test
-  script:
-    - make verify
-
-proto:
-  <<: *cache_definition
-  stage: test
+    - bundle install
   script:
-    - make proto no-changes
-  artifacts:
-    paths:
-    - _build/proto.diff
-    - ruby/proto/gitaly/*
-    - proto/go/gitalypb/*
-    when: on_failure
+    - >
+      if [ -z "$DANGER_GITLAB_API_TOKEN" ]; then
+        # Force danger to skip CI source GitLab and fallback to "local only git repo".
+        unset GITLAB_CI
+        # We need access to the base SHA to help danger determine the base commit for this shallow clone.
+        bundle exec danger dry_run --fail-on-errors=true --verbose --base='$CI_MERGE_REQUEST_DIFF_BASE_SHA'
+      else
+        bundle exec danger --fail-on-errors=true --verbose
+      fi
 
 build:
   stage: build
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
   cache:
     <<: *cache_configuration
     policy: pull-push
@@ -138,10 +102,9 @@ build:
       # apply our default set of Git patches.
       - GO_VERSION: [ "1.16", "1.17" ]
 
-binaries:
+build:binaries:
   <<: *cache_definition
   stage: build
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
   only:
     - tags
   script:
@@ -161,8 +124,6 @@ binaries:
 
 test:
   <<: *test_definition
-  <<: *postgres_definition
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
   script:
     - _build/deps/git/install/bin/git version
     # This command will make all directories except of our build directory and Ruby code unwritable.
@@ -170,11 +131,6 @@ test:
     # they should all instead use a temporary directory for runtime data.
     - find . -type d \( -path ./_build -o -path ./ruby \) -prune -o -type d -exec chmod a-w {} \;
     - make ${TARGET}
-  artifacts:
-    paths:
-      - _build/reports/go-tests-report-go-${GO_VERSION}-git-${GIT_VERSION}.xml
-    reports:
-      junit: _build/reports/go-tests-report-go-${GO_VERSION}-git-${GIT_VERSION}.xml
   parallel:
     matrix:
       # These definitions are for the non-default Git versions.
@@ -193,10 +149,34 @@ test:
       - POSTGRES_VERSION: "11.14-alpine"
         TARGET: [ test, test-with-praefect ]
 
-nightly:git:
+test:coverage:
+  <<: *test_definition
+  script:
+    - make cover
+  artifacts:
+    reports:
+      cobertura: _build/cover/cobertura.xml
+
+test:pgbouncer:
+  # This job verifies all operations run without issues if PgBouncer is set in between service and PostgreSQL database.
+  <<: *test_definition
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31-pgbouncer-1.14
+  variables:
+    <<: *postgres_variables
+    PGHOST_PGBOUNCER: 0.0.0.0
+    PGPORT_PGBOUNCER: "6432"
+  before_script:
+    - go version
+    - git version
+    - pgbouncer --version
+    - adduser --no-create-home --disabled-password --disabled-login --quiet --force-badname --gecos '' pgbouncer-runner
+    - su pgbouncer-runner -c 'pgbouncer internal/praefect/datastore/glsql/testdata/pgbouncer.ini' &
+    - for i in {1..10}; do psql -U $PGUSER -c 'select now()' && break; done || { echo 'pgbouncer awaiting failed' ; exit 1; }
+  script:
+    - make test-postgres
+
+test:nightly:
   <<: *test_definition
-  <<: *postgres_definition
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
   script:
     - go version
     - make all ${TARGET}
@@ -207,15 +187,35 @@ nightly:git:
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
 
-cover:
+test:praefect_smoke:
+  <<: *test_definition
+  script:
+    - make
+    - ruby -rerb -e 'ERB.new(ARGF.read).run' _support/config.praefect.toml.ci-sql-test.erb > config.praefect.toml
+    - ./_build/bin/praefect -config config.praefect.toml sql-ping
+    - ./_build/bin/praefect -config config.praefect.toml sql-migrate
+
+test:fuzz_objectinfo:
+  extends: .fuzz_base
+  stage: test
+  script:
+    - apt update && apt install -y clang-7
+    - go get github.com/dvyukov/go-fuzz/go-fuzz && go get github.com/dvyukov/go-fuzz/go-fuzz-build
+    - /root/go/bin/go-fuzz-build -libfuzzer -o objectinfo_fuzzer.a ./internal/git/catfile
+    - clang-7 -fsanitize=fuzzer objectinfo_fuzzer.a -o objectinfo_fuzzer
+    - ./gitlab-cov-fuzz run -- ./objectinfo_fuzzer -max_total_time=300
+
+verify:
   <<: *cache_definition
-  <<: *postgres_definition
   stage: test
   script:
-    - make cover
+    - make verify
   artifacts:
-    reports:
-      cobertura: _build/cover/cobertura.xml
+    paths:
+      - _build/proto.diff
+      - ruby/proto/gitaly/*
+      - proto/go/gitalypb/*
+    when: on_failure
 
 gosec-sast:
   before_script:
@@ -258,46 +258,6 @@ secret_detection:
     - if: $CI_MERGE_REQUEST_IID
     - if: $CI_COMMIT_TAG
 
-praefect_sql_connect:
-  <<: *test_definition
-  services:
-    - postgres:${POSTGRES_VERSION}
-  variables:
-    POSTGRES_DB: praefect_test
-    POSTGRES_USER: praefect
-    POSTGRES_PASSWORD: sql-password
-  script:
-    - make
-    # Sanity check: direct ping with psql
-    - PGPASSWORD=$POSTGRES_PASSWORD psql -h postgres -U $POSTGRES_USER -d $POSTGRES_DB -c 'select now()'
-    - ruby -rerb -e 'ERB.new(ARGF.read).run' _support/config.praefect.toml.ci-sql-test.erb > config.praefect.toml
-    - ./_build/bin/praefect -config config.praefect.toml sql-ping
-    - ./_build/bin/praefect -config config.praefect.toml sql-migrate
-
-pgbouncer_test:
-  # This job verifies all operations run without issues if PgBouncer is set in between service and PostgreSQL database.
-  <<: *test_definition
-  <<: *pgbouncer_definition
-  script:
-    - make test-postgres
-
-lint:
-  stage: test
-  retry: 2
-  script:
-    - go version
-    - make lint
-
-objectinfo_fuzz_test:
-  extends: .fuzz_base
-  stage: test
-  script:
-    - apt update && apt install -y clang-7
-    - go get github.com/dvyukov/go-fuzz/go-fuzz && go get github.com/dvyukov/go-fuzz/go-fuzz-build
-    - /root/go/bin/go-fuzz-build -libfuzzer -o objectinfo_fuzzer.a ./internal/git/catfile
-    - clang-7 -fsanitize=fuzzer objectinfo_fuzzer.a -o objectinfo_fuzzer
-    - ./gitlab-cov-fuzz run -- ./objectinfo_fuzzer -max_total_time=300
-
 trigger-qa:
   stage: qa
   trigger:
diff --git a/_support/config.praefect.toml.ci-sql-test.erb b/_support/config.praefect.toml.ci-sql-test.erb
index 91012ad7c..fdd1081c1 100755
--- a/_support/config.praefect.toml.ci-sql-test.erb
+++ b/_support/config.praefect.toml.ci-sql-test.erb
@@ -1,58 +1,16 @@
-# Example Praefect configuration file
-
-# # TCP address to listen on
 listen_addr = "127.0.0.1:2305"
 
-# # Praefect can listen on a socket when placed on the same machine as all clients
-# socket_path = "/home/git/gitlab/tmp/sockets/private/praefect.socket"
-# # Optional: export metrics via Prometheus
-# prometheus_listen_addr = "127.0.01:10101"
-# # You can optionally configure Praefect to output JSON-formatted log messages to stdout
-# [logging]
-#   format = "json"
-# # Optional: Set log level to only log entries with that severity or above
-# # One of, in order: debug, info, warn, errror, fatal, panic
-# # Defaults to "info"
-#   level = "warn"
-# [sentry]
-#   sentry_environment = ""
-#   sentry_dsn = ""
-#
-#  Optional: authenticate Gitaly requests using a shared secret. This token works the same way as a gitaly token
-# [auth]
-#   token = 'abc123secret'
-#
-# # One or more Gitaly servers need to be configured to be managed. The names
-# of each server are used to link multiple nodes, or `gitaly_server`s together
-# as shard. listen_addr should be unique for all nodes.
-# Requires the protocol to be defined, e.g. tcp://host.tld:1234
-
 [[virtual_storage]]
-name = "praefect"
-
-[[virtual_storage.node]]
-  storage = "praefect-git-0"
-  address = "tcp://praefect-git-0.internal"
-  token = 'token1'
+  name = "dummy"
 
 [[virtual_storage.node]]
-  storage = "praefect-git-1"
-  address = "tcp://praefect-git-1.internal"
-  token = 'token2'
-
-[[virtual_storage.node]]
-  storage = "praefect-git-2"
-  address = "tcp://praefect-git-2.internal"
-  token = 'token3'
+  storage = "dummy"
+  address = "tcp://dummy"
+  token = "dummy"
 
 [database]
-# In CI this magical hostname 'postgres' points to our dedicated CI postgres instance.
-host = 'postgres'
-
-# POSTGRES_* variables are defined in gitlab-ci.yml
-dbname = '<%= ENV['POSTGRES_DB'] %>'
-user = '<%= ENV['POSTGRES_USER'] %>'
-password = '<%= ENV['POSTGRES_PASSWORD'] %>'
-
-# No SSL in CI
-sslmode = 'disable'
+  # The following variables are defined in .gitlab-ci.yml.
+  host = '<%= ENV['PGHOST'] %>'
+  dbname = '<%= ENV['POSTGRES_DB'] %>'
+  user = '<%= ENV['PGUSER'] %>'
+  sslmode = 'disable'