stages: - build - test - qa default: image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31 tags: - gitlab-org variables: DOCKER_DRIVER: overlay2 SAST_DISABLE_DIND: "true" SAST_DEFAULT_ANALYZERS: "gosec" # We use Gitaly's Git version by default. GIT_VERSION: "default" GO_VERSION: "1.17" RUBY_VERSION: "2.7" POSTGRES_VERSION: "12.6-alpine" PGBOUNCER_VERSION: "1.16.1" BUNDLE_PATH: "${CI_PROJECT_DIR}/.ruby" GOPATH: "${CI_PROJECT_DIR}/.go" include: - template: Workflows/MergeRequest-Pipelines.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml - template: Security/Coverage-Fuzzing.gitlab-ci.yml - template: Security/Secret-Detection.gitlab-ci.yml .cache_deps: cache: &cache_deps_configuration key: files: - Makefile prefix: git-${GIT_VERSION} paths: - _build/deps policy: pull .cache_gems: cache: &cache_gems_configuration key: files: - ruby/Gemfile.lock prefix: ruby-${RUBY_VERSION} paths: - .ruby policy: pull .cache_go: cache: &cache_go_configuration key: files: - go.sum prefix: go-${GO_VERSION} paths: - .go/pkg/mod - _build/cache policy: pull .test_template: &test_definition stage: test cache: - *cache_deps_configuration - *cache_gems_configuration - *cache_go_configuration services: - postgres:${POSTGRES_VERSION} variables: &postgres_variables PGHOST: postgres PGPORT: 5432 PGUSER: postgres POSTGRES_DB: praefect_test POSTGRES_HOST_AUTH_METHOD: trust before_script: &test_before_script - go version - while ! psql -h $PGHOST -U $PGUSER -c 'SELECT 1' > /dev/null; do echo "awaiting Postgres service to be ready..." && sleep 1 ; done && echo "Postgres service is ready!" artifacts: paths: - ruby/tmp/gitaly-rspec-test.log reports: junit: _build/reports/go-tests-report.xml when: on_failure expire_in: 1 week danger-review: stage: build allow_failure: true only: - merge_requests except: - tags - master variables: BUNDLE_GEMFILE: danger/Gemfile before_script: - bundle install script: - > if [ -z "$DANGER_GITLAB_API_TOKEN" ]; then # Force danger to skip CI source GitLab and fallback to "local only git repo". unset GITLAB_CI # We need access to the base SHA to help danger determine the base commit for this shallow clone. bundle exec danger dry_run --fail-on-errors=true --verbose --base='$CI_MERGE_REQUEST_DIFF_BASE_SHA' else bundle exec danger --fail-on-errors=true --verbose fi build: stage: build cache: - <<: *cache_deps_configuration policy: pull-push - <<: *cache_gems_configuration policy: pull-push - <<: *cache_go_configuration policy: pull-push script: - go version - make all git - _support/test-boot . parallel: matrix: # These definitions are for the non-default Git versions. #- GO_VERSION: [ "1.16", "1.17" ] # GIT_VERSION: [ ] # These definitions are for the default Git version where we want to # apply our default set of Git patches. - GO_VERSION: [ "1.16", "1.17" ] build:binaries: stage: build cache: - *cache_deps_configuration - *cache_go_configuration only: - tags script: # Just in case we start running CI builds on other architectures in future - go version - make build - cd _build && sha256sum bin/* | tee checksums.sha256.txt artifacts: paths: - _build/checksums.sha256.txt - _build/bin/ name: "${CI_JOB_NAME}:go-${GO_VERSION}-git-${GIT_VERSION}" expire_in: 6 months parallel: matrix: - GO_VERSION: [ "1.16", "1.17" ] test: <<: *test_definition script: # This command will make all directories except of our build directory and Ruby code unwritable. # The purpose is to verify that there is no test which writes into those directories anymore, as # they should all instead use a temporary directory for runtime data. - find . -type d \( -path ./_build -o -path ./ruby \) -prune -o -type d -exec chmod a-w {} \; # If we're testing with bundled Git, then we don't want to have the Git # installation around. Otherwise, Git would be able to resolve its binaries # by just looking at its own GIT_PREFIX and then pick binaries from that # installation directory. - if test -n "${WITH_BUNDLED_GIT}"; then rm -rf _build/deps/git/install; fi - make ${TARGET} parallel: matrix: # These definitions are for the non-default Git versions. #- GO_VERSION: [ "1.16", "1.17" ] # GIT_VERSION: [ ] # TARGET: test # These definitions are for the default Git version where we want to # apply our default set of Git patches. - GO_VERSION: [ "1.16", "1.17" ] TARGET: test - TARGET: [ test-with-proxies, test-with-praefect, race-go ] - WITH_BUNDLED_GIT: "with-bundled-git" TARGET: test # Execute tests with our minimum required Postgres version, as well. If # the minimum version changes, please change this to the new minimum # version. Furthermore, please make sure to update the minimum required # version in `datastore.CheckPostgresVersion()`. - POSTGRES_VERSION: "11.14-alpine" TARGET: [ test, test-with-praefect ] test:coverage: <<: *test_definition script: - make cover artifacts: reports: cobertura: _build/cover/cobertura.xml test:pgbouncer: <<: *test_definition services: - postgres:${POSTGRES_VERSION} - name: bitnami/pgbouncer:${PGBOUNCER_VERSION} alias: pgbouncer variables: <<: *postgres_variables # The following variables are used by PgBouncer to connect to Postgres. POSTGRESQL_HOST: "${PGHOST}" # The image doesn't support setting `auth_user`, so we're cheating and use # "command line injection" here. In any case, `auth_user` is required so # that we can connect as a different user, but authenticate as the actual # PGUSER. We can fix this when # https://github.com/bitnami/bitnami-docker-pgbouncer/pull/22 lands. POSTGRESQL_PORT: "${PGPORT} auth_user=${PGUSER}" POSTGRESQL_USERNAME: "${PGUSER}" # These variables define how PgBouncer itself is configured PGBOUNCER_AUTH_TYPE: trust PGBOUNCER_DATABASE: "*" PGBOUNCER_IGNORE_STARTUP_PARAMETERS: extra_float_digits PGBOUNCER_POOL_MODE: transaction PGBOUNCER_MAX_DB_CONNECTIONS: 100 # And these are finally used by Gitaly's tests. PGHOST_PGBOUNCER: pgbouncer PGPORT_PGBOUNCER: 6432 # We need to enable per-build networking such that the PgBouncer service # can reach Postgres. FF_NETWORK_PER_BUILD: "true" before_script: - *test_before_script - while ! psql -h "${PGHOST_PGBOUNCER}" -p "${PGPORT_PGBOUNCER}" -U "${PGUSER}" -c 'SELECT 1' > /dev/null; do echo "awaiting PgBouncer service to be ready..." && sleep 1 ; done && echo "PgBouncer service is ready!" script: - make test-postgres test:nightly: <<: *test_definition script: - go version - make all ${TARGET} parallel: matrix: - GIT_VERSION: [ "master", "next" ] TARGET: [ test, test-with-proxies, test-with-praefect ] rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' test:praefect_smoke: <<: *test_definition script: - make - ruby -rerb -e 'ERB.new(ARGF.read).run' _support/config.praefect.toml.ci-sql-test.erb > config.praefect.toml - ./_build/bin/praefect -config config.praefect.toml sql-ping - ./_build/bin/praefect -config config.praefect.toml sql-migrate test:fuzz_objectinfo: extends: .fuzz_base stage: test script: - apt update && apt install -y clang-7 - go get github.com/dvyukov/go-fuzz/go-fuzz && go get github.com/dvyukov/go-fuzz/go-fuzz-build - /root/go/bin/go-fuzz-build -libfuzzer -o objectinfo_fuzzer.a ./internal/git/catfile - clang-7 -fsanitize=fuzzer objectinfo_fuzzer.a -o objectinfo_fuzzer - ./gitlab-cov-fuzz run -- ./objectinfo_fuzzer -max_total_time=300 verify: stage: test cache: - *cache_gems_configuration - *cache_go_configuration script: - make verify artifacts: paths: - _build/proto.diff - ruby/proto/gitaly/* - proto/go/gitalypb/* when: on_failure dbschema: stage: test cache: - *cache_deps_configuration - *cache_go_configuration services: # The database version we use must match the version of `pg_dump` we have # available in the build image. - postgres:11.13-alpine variables: <<: *postgres_variables before_script: - while ! psql -h $PGHOST -U $PGUSER -c 'SELECT 1' > /dev/null; do echo "awaiting Postgres service to be ready..." && sleep 1 ; done && echo "Postgres service is ready!" script: - make dump-database-schema no-changes artifacts: paths: - _support/praefect-schema.sql when: on_failure gosec-sast: before_script: - apk add pkgconfig libgit2-dev gcc libc-dev rules: - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_MERGE_REQUEST_IID - if: $CI_COMMIT_TAG license_scanning: before_script: - sudo apt-get update - sudo apt-get install -y libicu-dev libgit2-dev cmake rules: - if: $LICENSE_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_MERGE_REQUEST_IID - if: $CI_COMMIT_TAG variables: LICENSE_FINDER_CLI_OPTS: '--aggregate-paths=. ruby' gemnasium-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_MERGE_REQUEST_IID - if: $CI_COMMIT_TAG secret_detection: inherit: default: false rules: - if: $SECRET_DETECTION_DISABLED when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_MERGE_REQUEST_IID - if: $CI_COMMIT_TAG trigger-qa: stage: qa trigger: project: gitlab-org/build/omnibus-gitlab-mirror variables: ALTERNATIVE_SOURCES: "true" GITALY_SERVER_VERSION: $CI_COMMIT_SHA GITALY_SERVER_ALTERNATIVE_REPO: $CI_PROJECT_URL ee: "true" rules: - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA != "" when: manual allow_failure: true variables: # Downstream pipeline does not fetch the merged result SHA. # Fix: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6482 GITALY_SERVER_VERSION: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - when: manual allow_failure: true needs: []