Merge branch 'smh-pipeline-dependencies' into 'master'

Parallelize CI jobs

Closes #3123

See merge request gitlab-org/gitaly!4571

2 files changed, 45 insertions, 33 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 10525131e..6809a65df 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -67,6 +67,7 @@ include:
     policy: pull
 
 .test_template: &test_definition
+  needs: []
   stage: test
   cache:
     - *cache_deps_configuration
@@ -81,6 +82,7 @@ include:
     POSTGRES_DB: praefect_test
     POSTGRES_HOST_AUTH_METHOD: trust
     TEST_REPORT: /tmp/go-tests-report.xml
+    TEST_COVERAGE_DIR: /tmp/coverage
   before_script: &test_before_script
     - go version
     - while ! psql -h $PGHOST -U $PGUSER -c 'SELECT 1' > /dev/null; do echo "awaiting Postgres service to be ready..." && sleep 1 ; done && echo "Postgres service is ready!"
@@ -93,6 +95,7 @@ include:
     expire_in: 1 week
 
 danger-review:
+  needs: []
   stage: build
   allow_failure: true
   variables:
@@ -107,6 +110,7 @@ danger-review:
     policy: pull-push
 
 build:
+  needs: []
   stage: build
   cache:
     - <<: *cache_deps_configuration
@@ -126,6 +130,7 @@ build:
       - GIT_VERSION: "v2.33.0"
 
 build:binaries:
+  needs: []
   stage: build
   cache:
     - *cache_deps_configuration
@@ -176,6 +181,7 @@ test:
 
 test:coverage:
   <<: *test_definition
+  coverage: /^total:\t+\(statements\)\t+\d+\.\d+%$/
   script:
     # We need to explicitly build all prerequisites so that we can run tests unprivileged.
     - make -j$(nproc) build prepare-tests $(pwd)/_build/tools/gocover-cobertura
@@ -184,7 +190,7 @@ test:coverage:
     reports:
       coverage_report:
         coverage_format: cobertura
-        path: _build/cover/cobertura.xml
+        path: ${TEST_COVERAGE_DIR}/cobertura.xml
 
 test:pgbouncer:
   <<: *test_definition
@@ -245,12 +251,16 @@ test:praefect_smoke:
     - ./_build/bin/praefect -config config.praefect.toml sql-migrate
 
 verify:
+  needs: []
   stage: test
   cache:
     - *cache_deps_configuration
     - *cache_gems_configuration
     - *cache_go_configuration
   script:
+    # Download the dependencies in case there was no cache hit, otherwise
+    # golang-ci lint will timeout downloading them.
+    - go mod download
     - make -j$(nproc) verify
   artifacts:
     paths:
@@ -260,6 +270,7 @@ verify:
     when: on_failure
 
 dbschema:
+  needs: []
   stage: test
   cache:
     - *cache_deps_configuration
@@ -281,14 +292,21 @@ dbschema:
     when: on_failure
 
 gosec-sast:
-  dependencies: []
+  needs: []
   cache:
     - *cache_go_configuration
   variables:
     GOPATH: "/go"
   before_script:
-    - apk add pkgconfig libgit2-dev gcc libc-dev
-    - mv .go /go
+    # Our pipeline places GOPATH to $CI_PROJECT_DIR/.go so it can be cached.
+    # This causes gosec-sast to find the module cache and scan all the sources of
+    # the dependencies as well. This makes the scan time grow massively. This is
+    # avoided by this job moving the GOPATH outside of the project directory along
+    # with the cached modules if they were successfully extracted.
+    #
+    # SAST_EXCLUDED_PATHS is not sufficient as it only filters out the results but
+    # still performs the expensive scan.
+    - if [ -d .go ]; then mv .go $GOPATH; fi
   rules:
     - if: $SAST_DISABLED
       when: never
@@ -297,11 +315,7 @@ gosec-sast:
     - if: $CI_COMMIT_TAG
 
 license_scanning:
-  dependencies: []
-  cache: []
-  before_script:
-    - sudo apt-get update
-    - sudo apt-get install -y libicu-dev libgit2-dev cmake
+  needs: []
   rules:
     - if: $LICENSE_SCANNING_DISABLED
       when: never
@@ -312,8 +326,7 @@ license_scanning:
     LICENSE_FINDER_CLI_OPTS: '--aggregate-paths=. ruby'
 
 gemnasium-dependency_scanning:
-  dependencies: []
-  cache: []
+  needs: []
   rules:
     - if: $DEPENDENCY_SCANNING_DISABLED
       when: never
@@ -322,10 +335,7 @@ gemnasium-dependency_scanning:
     - if: $CI_COMMIT_TAG
 
 secret_detection:
-  dependencies: []
-  cache: []
-  inherit:
-    default: false
+  needs: []
   rules:
     - if: $SECRET_DETECTION_DISABLED
       when: never
@@ -334,6 +344,7 @@ secret_detection:
     - if: $CI_COMMIT_TAG
 
 trigger-qa:
+  needs: []
   stage: qa
   trigger:
     project: gitlab-org/build/omnibus-gitlab-mirror
@@ -352,9 +363,9 @@ trigger-qa:
         GITALY_SERVER_VERSION: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
     - when: manual
       allow_failure: true
-  needs: []
 
 qa:nightly-praefect-migration-test:
+  needs: []
   stage: qa
   trigger:
     project: gitlab-org/quality/praefect-migration-testing
diff --git a/Makefile b/Makefile
index fb3924c4a..985db3b7c 100644
--- a/Makefile
+++ b/Makefile
@@ -26,7 +26,6 @@ ARCH := $(shell uname -m)
 # Directories
 SOURCE_DIR       := $(abspath $(dir $(lastword ${MAKEFILE_LIST})))
 BUILD_DIR        := ${SOURCE_DIR}/_build
-COVERAGE_DIR     := ${BUILD_DIR}/cover
 DEPENDENCY_DIR   := ${BUILD_DIR}/deps
 TOOLS_DIR        := ${BUILD_DIR}/tools
 GITALY_RUBY_DIR  := ${SOURCE_DIR}/ruby
@@ -217,19 +216,21 @@ endif
 # These variables control test options and artifacts
 ## List of Go packages which shall be tested.
 ## Go packages to test when using the test-go target.
-TEST_PACKAGES    ?= ${SOURCE_DIR}/...
+TEST_PACKAGES     ?= ${SOURCE_DIR}/...
 ## Test options passed to `go test`.
-TEST_OPTIONS     ?= -count=1
+TEST_OPTIONS      ?= -count=1
 ## Specify the output format used to print tests ["standard-verbose", "standard-quiet", "short"]
-TEST_FORMAT      ?= short
-TEST_REPORT      ?= ${BUILD_DIR}/reports/go-tests-report.xml
+TEST_FORMAT       ?= short
+TEST_REPORT       ?= ${BUILD_DIR}/reports/go-tests-report.xml
+## Specify the output directory for test coverage reports.
+TEST_COVERAGE_DIR ?= ${BUILD_DIR}/cover
 ## Directory where all runtime test data is being created.
-TEST_TMP_DIR     ?=
-TEST_REPO_DIR    := ${BUILD_DIR}/testrepos
-TEST_REPO        := ${TEST_REPO_DIR}/gitlab-test.git
-TEST_REPO_MIRROR := ${TEST_REPO_DIR}/gitlab-test-mirror.git
-TEST_REPO_GIT    := ${TEST_REPO_DIR}/gitlab-git-test.git
-BENCHMARK_REPO   := ${TEST_REPO_DIR}/benchmark.git
+TEST_TMP_DIR      ?=
+TEST_REPO_DIR     := ${BUILD_DIR}/testrepos
+TEST_REPO         := ${TEST_REPO_DIR}/gitlab-test.git
+TEST_REPO_MIRROR  := ${TEST_REPO_DIR}/gitlab-test-mirror.git
+TEST_REPO_GIT     := ${TEST_REPO_DIR}/gitlab-git-test.git
+BENCHMARK_REPO    := ${TEST_REPO_DIR}/benchmark.git
 
 # All executables provided by Gitaly
 GITALY_EXECUTABLES    = $(addprefix ${BUILD_DIR}/bin/,$(notdir $(shell find ${SOURCE_DIR}/cmd -mindepth 1 -maxdepth 1 -type d -print)) gitaly-git2go-v14)
@@ -423,18 +424,18 @@ rubocop: ${SOURCE_DIR}/.ruby-bundle
 
 .PHONY: cover
 ## Generate coverage report via Go tests.
-cover: TEST_OPTIONS  := ${TEST_OPTIONS} -coverprofile "${COVERAGE_DIR}/all.merged"
+cover: TEST_OPTIONS  := ${TEST_OPTIONS} -coverprofile "${TEST_COVERAGE_DIR}/all.merged"
 cover: prepare-tests libgit2 ${GOCOVER_COBERTURA}
-	${Q}rm -rf "${COVERAGE_DIR}"
-	${Q}mkdir -p "${COVERAGE_DIR}"
+	${Q}rm -rf "${TEST_COVERAGE_DIR}"
+	${Q}mkdir -p "${TEST_COVERAGE_DIR}"
 	${Q}$(call run_go_tests)
-	${Q}go tool cover -html  "${COVERAGE_DIR}/all.merged" -o "${COVERAGE_DIR}/all.html"
+	${Q}go tool cover -html  "${TEST_COVERAGE_DIR}/all.merged" -o "${TEST_COVERAGE_DIR}/all.html"
 	@ # sed is used below to convert file paths to repository root relative paths. See https://gitlab.com/gitlab-org/gitlab/-/issues/217664
-	${Q}${GOCOVER_COBERTURA} <"${COVERAGE_DIR}/all.merged" | sed 's;filename=\"$(shell go list -m)/;filename=\";g' >"${COVERAGE_DIR}/cobertura.xml"
+	${Q}${GOCOVER_COBERTURA} <"${TEST_COVERAGE_DIR}/all.merged" | sed 's;filename=\"$(shell go list -m)/;filename=\";g' >"${TEST_COVERAGE_DIR}/cobertura.xml"
 	${Q}echo ""
 	${Q}echo "=====> Total test coverage: <====="
 	${Q}echo ""
-	${Q}go tool cover -func "${COVERAGE_DIR}/all.merged"
+	${Q}go tool cover -func "${TEST_COVERAGE_DIR}/all.merged"
 
 .PHONY: proto
 ## Regenerate protobuf definitions.