From a4761b605c527f12a9647d94a30ca04ccdea36ec Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 9 Nov 2023 09:05:25 +0100 Subject: ci: reorder definitions for grouping functions We define a set of grouping functions that are used to group together output in our CI, where these groups then end up as collapsible sections in the respective pipeline platform. The way these functions are defined is not easily extensible though as we have an up front check for the CI _not_ being GitHub Actions, where we define the non-stub logic in the else branch. Reorder the conditional branches such that we explicitly handle GitHub Actions. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/lib.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index 369d462f13..26895ddbcc 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -1,16 +1,7 @@ # Library of functions shared by all CI scripts -if test true != "$GITHUB_ACTIONS" +if test true = "$GITHUB_ACTIONS" then - begin_group () { :; } - end_group () { :; } - - group () { - shift - "$@" - } - set -x -else begin_group () { need_to_end_group=t echo "::group::$1" >&2 @@ -42,6 +33,15 @@ else } begin_group "CI setup" +else + begin_group () { :; } + end_group () { :; } + + group () { + shift + "$@" + } + set -x fi # Set 'exit on error' for all CI scripts to let the caller know that -- cgit v1.2.3 From a7d499cb9361e4c4d6c8b5f841e95fc8cd97584d Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 9 Nov 2023 09:05:29 +0100 Subject: ci: make grouping setup more generic Make the grouping setup more generic by always calling `begin_group ()` and `end_group ()` regardless of whether we have stubbed those functions or not. This ensures we can more readily add support for additional CI platforms. Furthermore, the `group ()` function is made generic so that it is the same for both GitHub Actions and for other platforms. There is a semantic conflict here though: GitHub Actions used to call `set +x` in `group ()` whereas the non-GitHub case unconditionally uses `set -x`. The latter would get overriden if we kept the `set +x` in the generic version of `group ()`. To resolve this conflict, we simply drop the `set +x` in the generic variant of this function. As `begin_group ()` calls `set -x` anyway this is not much of a change though, as the only commands that aren't printed anymore now are the ones between the beginning of `group ()` and the end of `begin_group ()`. Last, this commit changes `end_group ()` to also accept a parameter that indicates _which_ group should end. This will be required by a later commit that introduces support for GitLab CI. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/lib.sh | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index 26895ddbcc..3938cad32c 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -14,36 +14,34 @@ then need_to_end_group= echo '::endgroup::' >&2 } - trap end_group EXIT - - group () { - set +x - begin_group "$1" - shift - # work around `dash` not supporting `set -o pipefail` - ( - "$@" 2>&1 - echo $? >exit.status - ) | - sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' - res=$(cat exit.status) - rm exit.status - end_group - return $res - } - - begin_group "CI setup" else begin_group () { :; } end_group () { :; } - group () { - shift - "$@" - } set -x fi +group () { + group="$1" + shift + begin_group "$group" + + # work around `dash` not supporting `set -o pipefail` + ( + "$@" 2>&1 + echo $? >exit.status + ) | + sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' + res=$(cat exit.status) + rm exit.status + + end_group "$group" + return $res +} + +begin_group "CI setup" +trap "end_group 'CI setup'" EXIT + # Set 'exit on error' for all CI scripts to let the caller know that # something went wrong. # @@ -285,5 +283,5 @@ esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" -end_group +end_group "CI setup" set -x -- cgit v1.2.3 From 412847ced432c558bc4c3ab40dc2ea22ffe7fb8f Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 9 Nov 2023 09:05:34 +0100 Subject: ci: group installation of Docker dependencies The output of CI jobs tends to be quite long-winded and hard to digest. To help with this, many CI systems provide the ability to group output into collapsible sections, and we're also doing this in some of our scripts. One notable omission is the script to install Docker dependencies. Address it to bring more structure to the output for Docker-based jobs. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-docker-dependencies.sh | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'ci') diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 78b7e326da..d0bc19d3bb 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -3,6 +3,10 @@ # Install dependencies required to build and test Git inside container # +. ${0%/*}/lib.sh + +begin_group "Install dependencies" + case "$jobname" in linux32) linux32 --32bit i386 sh -c ' @@ -20,3 +24,5 @@ pedantic) dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null ;; esac + +end_group "Install dependencies" -- cgit v1.2.3 From e624f206bc07201562619e2a9788e2b3762409fc Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 9 Nov 2023 09:05:38 +0100 Subject: ci: split out logic to set up failed test artifacts We have some logic in place to create a directory with the output from failed tests, which will then subsequently be uploaded as CI artifacts. We're about to add support for GitLab CI, which will want to reuse the logic. Split the logic into a separate function so that it is reusable. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/lib.sh | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index 3938cad32c..9abecf18b0 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -131,6 +131,26 @@ handle_failed_tests () { return 1 } +create_failed_test_artifacts () { + mkdir -p t/failed-test-artifacts + + for test_exit in t/test-results/*.exit + do + test 0 != "$(cat "$test_exit")" || continue + + test_name="${test_exit%.exit}" + test_name="${test_name##*/}" + printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" + echo "The full logs are in the 'print test failures' step below." + echo "See also the 'failed-tests-*' artifacts attached to this run." + cat "t/test-results/$test_name.markup" + + trash_dir="t/trash directory.$test_name" + cp "t/test-results/$test_name.out" t/failed-test-artifacts/ + tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + done +} + # GitHub Action doesn't set TERM, which is required by tput export TERM=${TERM:-dumb} @@ -171,24 +191,8 @@ then CC="${CC_PACKAGE:-${CC:-gcc}}" DONT_SKIP_TAGS=t handle_failed_tests () { - mkdir -p t/failed-test-artifacts echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV - - for test_exit in t/test-results/*.exit - do - test 0 != "$(cat "$test_exit")" || continue - - test_name="${test_exit%.exit}" - test_name="${test_name##*/}" - printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" - echo "The full logs are in the 'print test failures' step below." - echo "See also the 'failed-tests-*' artifacts attached to this run." - cat "t/test-results/$test_name.markup" - - trash_dir="t/trash directory.$test_name" - cp "t/test-results/$test_name.out" t/failed-test-artifacts/ - tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" - done + create_failed_test_artifacts return 1 } -- cgit v1.2.3 From 9f17bef9a6ead213ea62d399baa4c67e1e89398b Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 9 Nov 2023 09:05:42 +0100 Subject: ci: unify setup of some environment variables Both GitHub Actions and Azure Pipelines set up the environment variables GIT_TEST_OPTS, GIT_PROVE_OPTS and MAKEFLAGS. And while most values are actually the same, the setup is completely duplicate. With the upcoming support for GitLab CI this duplication would only extend even further. Unify the setup of those environment variables so that only the uncommon parts are separated. While at it, we also perform some additional small improvements: - We now always pass `--state=failed,slow,save` via GIT_PROVE_OPTS. It doesn't hurt on platforms where we don't persist the state, so this further reduces boilerplate. - When running on Windows systems we set `--no-chain-lint` and `--no-bin-wrappers`. Interestingly though, we did so _after_ already having exported the respective environment variables. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/lib.sh | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index 9abecf18b0..33be93852f 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -174,11 +174,8 @@ then # among *all* phases) cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" - export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" - export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" - MAKEFLAGS="$MAKEFLAGS --jobs=10" - test windows_nt != "$CI_OS_NAME" || - GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + GIT_TEST_OPTS="--write-junit-xml" + JOBS=10 elif test true = "$GITHUB_ACTIONS" then CI_TYPE=github-actions @@ -198,17 +195,27 @@ then cache_dir="$HOME/none" - export GIT_PROVE_OPTS="--timer --jobs 10" - export GIT_TEST_OPTS="--verbose-log -x --github-workflow-markup" - MAKEFLAGS="$MAKEFLAGS --jobs=10" - test windows != "$CI_OS_NAME" || - GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + GIT_TEST_OPTS="--github-workflow-markup" + JOBS=10 else echo "Could not identify CI type" >&2 env >&2 exit 1 fi +MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS" +GIT_PROVE_OPTS="--timer --jobs $JOBS --state=failed,slow,save" + +GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x" +case "$CI_OS_NAME" in +windows|windows_nt) + GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers" + ;; +esac + +export GIT_TEST_OPTS +export GIT_PROVE_OPTS + good_trees_file="$cache_dir/good-trees" mkdir -p "$cache_dir" -- cgit v1.2.3 From dd02c3b68c67f8e9a5daca3b52a562318738fa47 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 9 Nov 2023 09:05:46 +0100 Subject: ci: squelch warnings when testing with unusable Git repo Our CI jobs that run on Docker also use mostly the same architecture to build and test Git via the "ci/run-build-and-tests.sh" script. These scripts also provide some functionality to massage the Git repository we're supposedly operating in. In our Docker-based infrastructure we may not even have a Git repository available though, which leads to warnings when those functions execute. Make the helpers exit gracefully in case either there is no Git in our PATH, or when not running in a Git repository. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/lib.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index 33be93852f..eab0e24080 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -69,10 +69,32 @@ skip_branch_tip_with_tag () { fi } +# Check whether we can use the path passed via the first argument as Git +# repository. +is_usable_git_repository () { + # We require Git in our PATH, otherwise we cannot access repositories + # at all. + if ! command -v git >/dev/null + then + return 1 + fi + + # And the target directory needs to be a proper Git repository. + if ! git -C "$1" rev-parse 2>/dev/null + then + return 1 + fi +} + # Save some info about the current commit's tree, so we can skip the build # job if we encounter the same tree again and can provide a useful info # message. save_good_tree () { + if ! is_usable_git_repository . + then + return + fi + echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file" # limit the file size tail -1000 "$good_trees_file" >"$good_trees_file".tmp @@ -88,6 +110,11 @@ skip_good_tree () { return fi + if ! is_usable_git_repository . + then + return + fi + if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")" then # Haven't seen this tree yet, or no cached good trees file yet. @@ -119,6 +146,11 @@ skip_good_tree () { } check_unignored_build_artifacts () { + if ! is_usable_git_repository . + then + return + fi + ! git ls-files --other --exclude-standard --error-unmatch \ -- ':/*' 2>/dev/null || { -- cgit v1.2.3 From 0d3911ad73514850a79af478d62a94c754892d12 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 9 Nov 2023 09:05:50 +0100 Subject: ci: install test dependencies for linux-musl The linux-musl CI job executes tests on Alpine Linux, which is based on musl libc instead of glibc. We're missing some test dependencies though, which causes us to skip a subset of tests. Install these test dependencies to increase our test coverage on this platform. There are still some missing test dependecies, but these do not have a corresponding package in the Alpine repositories: - p4 and p4d, both parts of the Perforce version control system. - cvsps, which generates patch sets for CVS. - Subversion and the SVN::Core Perl library, the latter of which is not available in the Alpine repositories. While the tool itself is available, all Subversion-related tests are skipped without the SVN::Core Perl library anyway. The Apache2-based tests require a bit more care though. For one, the module path is different on Alpine Linux, which requires us to add it to the list of known module paths to detect it. But second, the WebDAV module on Alpine Linux is broken because it does not bundle the default database backend [1]. We thus need to skip the WebDAV-based tests on Alpine Linux for now. [1]: https://gitlab.alpinelinux.org/alpine/aports/-/issues/13112 Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-docker-dependencies.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'ci') diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index d0bc19d3bb..6e84528368 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -17,7 +17,9 @@ linux32) ;; linux-musl) apk add --update build-base curl-dev openssl-dev expat-dev gettext \ - pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null + pcre2-dev python3 musl-libintl perl-utils ncurses \ + apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ + bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null ;; pedantic) dnf -yq update >/dev/null && -- cgit v1.2.3 From 0e3b67e2aa25edb7e1a5c999c87b52a7b3a7649a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 9 Nov 2023 09:05:54 +0100 Subject: ci: add support for GitLab CI We already support Azure Pipelines and GitHub Workflows in the Git project, but until now we do not have support for GitLab CI. While it is arguably not in the interest of the Git project to maintain a ton of different CI platforms, GitLab has recently ramped up its efforts and tries to contribute to the Git project more regularly. Part of a problem we hit at GitLab rather frequently is that our own, custom CI setup we have is so different to the setup that the Git project has. More esoteric jobs like "linux-TEST-vars" that also set a couple of environment variables do not exist in GitLab's custom CI setup, and maintaining them to keep up with what Git does feels like wasted time. The result is that we regularly send patch series upstream that fail to compile or pass tests in GitHub Workflows. We would thus like to integrate the GitLab CI configuration into the Git project to help us send better patch series upstream and thus reduce overhead for the maintainer. Results of these pipeline runs will be made available (at least) in GitLab's mirror of the Git project at [1]. This commit introduces the integration into our regular CI scripts so that most of the setup continues to be shared across all of the CI solutions. Note that as the builds on GitLab CI run as unprivileged user, we need to pull in both sudo and shadow packages to our Alpine based job to set this up. [1]: https://gitlab.com/gitlab-org/git Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-docker-dependencies.sh | 13 ++++++++++- ci/lib.sh | 45 +++++++++++++++++++++++++++++++++++++++ ci/print-test-failures.sh | 6 ++++++ 3 files changed, 63 insertions(+), 1 deletion(-) (limited to 'ci') diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 6e84528368..48c43f0f90 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -16,11 +16,22 @@ linux32) ' ;; linux-musl) - apk add --update build-base curl-dev openssl-dev expat-dev gettext \ + apk add --update shadow sudo build-base curl-dev openssl-dev expat-dev gettext \ pcre2-dev python3 musl-libintl perl-utils ncurses \ apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null ;; +linux-*) + # Required so that apt doesn't wait for user input on certain packages. + export DEBIAN_FRONTEND=noninteractive + + apt update -q && + apt install -q -y sudo git make language-pack-is libsvn-perl apache2 libssl-dev \ + libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev \ + perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl \ + libdbd-sqlite3-perl libio-socket-ssl-perl libnet-smtp-ssl-perl ${CC_PACKAGE:-${CC:-gcc}} \ + apache2 cvs cvsps gnupg libcgi-pm-perl subversion + ;; pedantic) dnf -yq update >/dev/null && dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null diff --git a/ci/lib.sh b/ci/lib.sh index eab0e24080..6dfc90d7f5 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -14,6 +14,22 @@ then need_to_end_group= echo '::endgroup::' >&2 } +elif test true = "$GITLAB_CI" +then + begin_group () { + need_to_end_group=t + printf "\e[0Ksection_start:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K$1\n" + trap "end_group '$1'" EXIT + set -x + } + + end_group () { + test -n "$need_to_end_group" || return 0 + set +x + need_to_end_group= + printf "\e[0Ksection_end:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K\n" + trap - EXIT + } else begin_group () { :; } end_group () { :; } @@ -229,6 +245,35 @@ then GIT_TEST_OPTS="--github-workflow-markup" JOBS=10 +elif test true = "$GITLAB_CI" +then + CI_TYPE=gitlab-ci + CI_BRANCH="$CI_COMMIT_REF_NAME" + CI_COMMIT="$CI_COMMIT_SHA" + case "$CI_JOB_IMAGE" in + macos-*) + CI_OS_NAME=osx;; + alpine:*|fedora:*|ubuntu:*) + CI_OS_NAME=linux;; + *) + echo "Could not identify OS image" >&2 + env >&2 + exit 1 + ;; + esac + CI_REPO_SLUG="$CI_PROJECT_PATH" + CI_JOB_ID="$CI_JOB_ID" + CC="${CC_PACKAGE:-${CC:-gcc}}" + DONT_SKIP_TAGS=t + handle_failed_tests () { + create_failed_test_artifacts + return 1 + } + + cache_dir="$HOME/none" + + runs_on_pool=$(echo "$CI_JOB_IMAGE" | tr : -) + JOBS=$(nproc) else echo "Could not identify CI type" >&2 env >&2 diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 57277eefcd..c33ad4e3a2 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -51,6 +51,12 @@ do tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" continue ;; + gitlab-ci) + mkdir -p failed-test-artifacts + cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ + tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + continue + ;; *) echo "Unhandled CI type: $CI_TYPE" >&2 exit 1 -- cgit v1.2.3