diff options
author | Junio C Hamano <gitster@pobox.com> | 2023-12-10 03:37:48 +0300 |
---|---|---|
committer | Junio C Hamano <gitster@pobox.com> | 2023-12-10 03:37:48 +0300 |
commit | 14a4445d18eccc146b3340c2bdeeb15c81538938 (patch) | |
tree | eef5fec80766a065a5e4e4790803937812925e99 /ci | |
parent | 712177ed04c3c33bfe6075b9cc2d4d65e2cc04c2 (diff) | |
parent | 0e3b67e2aa25edb7e1a5c999c87b52a7b3a7649a (diff) |
Merge branch 'ps/ci-gitlab'
Add support for GitLab CI.
* ps/ci-gitlab:
ci: add support for GitLab CI
ci: install test dependencies for linux-musl
ci: squelch warnings when testing with unusable Git repo
ci: unify setup of some environment variables
ci: split out logic to set up failed test artifacts
ci: group installation of Docker dependencies
ci: make grouping setup more generic
ci: reorder definitions for grouping functions
Diffstat (limited to 'ci')
-rwxr-xr-x | ci/install-docker-dependencies.sh | 23 | ||||
-rwxr-xr-x | ci/lib.sh | 190 | ||||
-rwxr-xr-x | ci/print-test-failures.sh | 6 |
3 files changed, 165 insertions, 54 deletions
diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 78b7e326da..48c43f0f90 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -3,6 +3,10 @@ # Install dependencies required to build and test Git inside container # +. ${0%/*}/lib.sh + +begin_group "Install dependencies" + case "$jobname" in linux32) linux32 --32bit i386 sh -c ' @@ -12,11 +16,26 @@ linux32) ' ;; linux-musl) - apk add --update build-base curl-dev openssl-dev expat-dev gettext \ - pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null + apk add --update shadow sudo build-base curl-dev openssl-dev expat-dev gettext \ + pcre2-dev python3 musl-libintl perl-utils ncurses \ + apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ + bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null + ;; +linux-*) + # Required so that apt doesn't wait for user input on certain packages. + export DEBIAN_FRONTEND=noninteractive + + apt update -q && + apt install -q -y sudo git make language-pack-is libsvn-perl apache2 libssl-dev \ + libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev \ + perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl \ + libdbd-sqlite3-perl libio-socket-ssl-perl libnet-smtp-ssl-perl ${CC_PACKAGE:-${CC:-gcc}} \ + apache2 cvs cvsps gnupg libcgi-pm-perl subversion ;; pedantic) dnf -yq update >/dev/null && dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null ;; esac + +end_group "Install dependencies" @@ -1,16 +1,7 @@ # Library of functions shared by all CI scripts -if test true != "$GITHUB_ACTIONS" +if test true = "$GITHUB_ACTIONS" then - begin_group () { :; } - end_group () { :; } - - group () { - shift - "$@" - } - set -x -else begin_group () { need_to_end_group=t echo "::group::$1" >&2 @@ -23,27 +14,50 @@ else need_to_end_group= echo '::endgroup::' >&2 } - trap end_group EXIT +elif test true = "$GITLAB_CI" +then + begin_group () { + need_to_end_group=t + printf "\e[0Ksection_start:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K$1\n" + trap "end_group '$1'" EXIT + set -x + } - group () { + end_group () { + test -n "$need_to_end_group" || return 0 set +x - begin_group "$1" - shift - # work around `dash` not supporting `set -o pipefail` - ( - "$@" 2>&1 - echo $? >exit.status - ) | - sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' - res=$(cat exit.status) - rm exit.status - end_group - return $res + need_to_end_group= + printf "\e[0Ksection_end:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K\n" + trap - EXIT } +else + begin_group () { :; } + end_group () { :; } - begin_group "CI setup" + set -x fi +group () { + group="$1" + shift + begin_group "$group" + + # work around `dash` not supporting `set -o pipefail` + ( + "$@" 2>&1 + echo $? >exit.status + ) | + sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' + res=$(cat exit.status) + rm exit.status + + end_group "$group" + return $res +} + +begin_group "CI setup" +trap "end_group 'CI setup'" EXIT + # Set 'exit on error' for all CI scripts to let the caller know that # something went wrong. # @@ -71,10 +85,32 @@ skip_branch_tip_with_tag () { fi } +# Check whether we can use the path passed via the first argument as Git +# repository. +is_usable_git_repository () { + # We require Git in our PATH, otherwise we cannot access repositories + # at all. + if ! command -v git >/dev/null + then + return 1 + fi + + # And the target directory needs to be a proper Git repository. + if ! git -C "$1" rev-parse 2>/dev/null + then + return 1 + fi +} + # Save some info about the current commit's tree, so we can skip the build # job if we encounter the same tree again and can provide a useful info # message. save_good_tree () { + if ! is_usable_git_repository . + then + return + fi + echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file" # limit the file size tail -1000 "$good_trees_file" >"$good_trees_file".tmp @@ -90,6 +126,11 @@ skip_good_tree () { return fi + if ! is_usable_git_repository . + then + return + fi + if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")" then # Haven't seen this tree yet, or no cached good trees file yet. @@ -121,6 +162,11 @@ skip_good_tree () { } check_unignored_build_artifacts () { + if ! is_usable_git_repository . + then + return + fi + ! git ls-files --other --exclude-standard --error-unmatch \ -- ':/*' 2>/dev/null || { @@ -133,6 +179,26 @@ handle_failed_tests () { return 1 } +create_failed_test_artifacts () { + mkdir -p t/failed-test-artifacts + + for test_exit in t/test-results/*.exit + do + test 0 != "$(cat "$test_exit")" || continue + + test_name="${test_exit%.exit}" + test_name="${test_name##*/}" + printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" + echo "The full logs are in the 'print test failures' step below." + echo "See also the 'failed-tests-*' artifacts attached to this run." + cat "t/test-results/$test_name.markup" + + trash_dir="t/trash directory.$test_name" + cp "t/test-results/$test_name.out" t/failed-test-artifacts/ + tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + done +} + # GitHub Action doesn't set TERM, which is required by tput export TERM=${TERM:-dumb} @@ -156,11 +222,8 @@ then # among *all* phases) cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" - export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" - export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" - MAKEFLAGS="$MAKEFLAGS --jobs=10" - test windows_nt != "$CI_OS_NAME" || - GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + GIT_TEST_OPTS="--write-junit-xml" + JOBS=10 elif test true = "$GITHUB_ACTIONS" then CI_TYPE=github-actions @@ -173,40 +236,63 @@ then CC="${CC_PACKAGE:-${CC:-gcc}}" DONT_SKIP_TAGS=t handle_failed_tests () { - mkdir -p t/failed-test-artifacts echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV + create_failed_test_artifacts + return 1 + } - for test_exit in t/test-results/*.exit - do - test 0 != "$(cat "$test_exit")" || continue - - test_name="${test_exit%.exit}" - test_name="${test_name##*/}" - printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" - echo "The full logs are in the 'print test failures' step below." - echo "See also the 'failed-tests-*' artifacts attached to this run." - cat "t/test-results/$test_name.markup" - - trash_dir="t/trash directory.$test_name" - cp "t/test-results/$test_name.out" t/failed-test-artifacts/ - tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" - done + cache_dir="$HOME/none" + + GIT_TEST_OPTS="--github-workflow-markup" + JOBS=10 +elif test true = "$GITLAB_CI" +then + CI_TYPE=gitlab-ci + CI_BRANCH="$CI_COMMIT_REF_NAME" + CI_COMMIT="$CI_COMMIT_SHA" + case "$CI_JOB_IMAGE" in + macos-*) + CI_OS_NAME=osx;; + alpine:*|fedora:*|ubuntu:*) + CI_OS_NAME=linux;; + *) + echo "Could not identify OS image" >&2 + env >&2 + exit 1 + ;; + esac + CI_REPO_SLUG="$CI_PROJECT_PATH" + CI_JOB_ID="$CI_JOB_ID" + CC="${CC_PACKAGE:-${CC:-gcc}}" + DONT_SKIP_TAGS=t + handle_failed_tests () { + create_failed_test_artifacts return 1 } cache_dir="$HOME/none" - export GIT_PROVE_OPTS="--timer --jobs 10" - export GIT_TEST_OPTS="--verbose-log -x --github-workflow-markup" - MAKEFLAGS="$MAKEFLAGS --jobs=10" - test windows != "$CI_OS_NAME" || - GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + runs_on_pool=$(echo "$CI_JOB_IMAGE" | tr : -) + JOBS=$(nproc) else echo "Could not identify CI type" >&2 env >&2 exit 1 fi +MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS" +GIT_PROVE_OPTS="--timer --jobs $JOBS --state=failed,slow,save" + +GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x" +case "$CI_OS_NAME" in +windows|windows_nt) + GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers" + ;; +esac + +export GIT_TEST_OPTS +export GIT_PROVE_OPTS + good_trees_file="$cache_dir/good-trees" mkdir -p "$cache_dir" @@ -285,5 +371,5 @@ esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" -end_group +end_group "CI setup" set -x diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 57277eefcd..c33ad4e3a2 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -51,6 +51,12 @@ do tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" continue ;; + gitlab-ci) + mkdir -p failed-test-artifacts + cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ + tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + continue + ;; *) echo "Unhandled CI type: $CI_TYPE" >&2 exit 1 |