diff options
Diffstat (limited to '.gitlab/ci/review-apps')
| -rw-r--r-- | .gitlab/ci/review-apps/main.gitlab-ci.yml | 21 | ||||
| -rw-r--r-- | .gitlab/ci/review-apps/qa.gitlab-ci.yml | 129 | ||||
| -rw-r--r-- | .gitlab/ci/review-apps/rules.gitlab-ci.yml | 81 | ||||
| -rw-r--r-- | .gitlab/ci/review-apps/skip-qa.gitlab-ci.yml | 13 |
4 files changed, 155 insertions, 89 deletions
diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index 37ccecc0562..e28ffc82811 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -8,6 +8,7 @@ stages: include: - local: .gitlab/ci/global.gitlab-ci.yml - local: .gitlab/ci/rules.gitlab-ci.yml + - local: .gitlab/ci/review-apps/rules.gitlab-ci.yml - local: .gitlab/ci/review-apps/qa.gitlab-ci.yml - local: .gitlab/ci/review-apps/dast.gitlab-ci.yml @@ -94,6 +95,7 @@ review-deploy: - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt + - echo "QA_GITLAB_URL=${CI_ENVIRONMENT_URL}" > environment.env - *base-before_script script: - check_kube_domain @@ -102,7 +104,6 @@ review-deploy: - deploy || (display_deployment_debug && exit 1) - verify_deploy || exit 1 - disable_sign_ups || (delete_release && exit 1) - - create_sample_projects after_script: # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. # Set DAST_RUN to true when jobs are manually scheduled. @@ -110,9 +111,27 @@ review-deploy: artifacts: paths: - environment_url.txt + reports: + dotenv: environment.env expire_in: 7 days when: always +review-deploy-sample-projects: + extends: + - .review-workflow-base + - .review:rules:review-deploy + stage: deploy + needs: ["review-deploy"] + before_script: + - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) + - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) + - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) + - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt + - *base-before_script + script: + - date + - create_sample_projects + .review-stop-base: extends: .review-workflow-base environment: diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml index 631fe7fef30..21e6a8e42fb 100644 --- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml @@ -1,6 +1,6 @@ include: - project: gitlab-org/quality/pipeline-common - ref: 0.13.0 + ref: 1.2.2 file: - /ci/allure-report.yml - /ci/knapsack-report.yml @@ -8,85 +8,57 @@ include: .test-variables: variables: QA_GENERATE_ALLURE_REPORT: "true" - COLORIZED_LOGS: "true" + QA_CAN_TEST_PRAEFECT: "false" GITLAB_USERNAME: "root" GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITLAB_ADMIN_USERNAME: "root" GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITLAB_QA_ADMIN_ACCESS_TOKEN: "${REVIEW_APPS_ROOT_TOKEN}" - GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" + GITHUB_ACCESS_TOKEN: "${QA_GITHUB_ACCESS_TOKEN}" .bundle-base: extends: - .qa-cache image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.3 before_script: - - export QA_GITLAB_URL="$(cat environment_url.txt)" - cd qa && bundle install .review-qa-base: + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.3-git-2.33-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23 extends: - .use-docker-in-docker - .bundle-base - .test-variables - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.3-git-2.33-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23 stage: qa needs: - review-deploy - download-knapsack-report variables: - DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: /certs - DOCKER_CERT_PATH: /certs/client - DOCKER_TLS_VERIFY: 1 GIT_LFS_SKIP_SMUDGE: 1 WD_INSTALL_DIR: /usr/local/bin - before_script: - - scripts/checkout-mr-source-sha - - !reference [.bundle-base, before_script] + RSPEC_REPORT_OPTS: --force-color --order random --format documentation --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml script: - export EE_LICENSE="$(cat $REVIEW_APPS_EE_LICENSE_FILE)" - - qa_run_status=0 + - QA_COMMAND="bundle exec bin/qa ${QA_SCENARIO} ${QA_GITLAB_URL} -- ${QA_TESTS} ${RSPEC_REPORT_OPTS}" + - echo "Running - '${QA_COMMAND}'" + - eval "$QA_COMMAND" + after_script: - | - bundle exec rake "knapsack:rspec[\ - ${RSPEC_TAGS} \ - --tag ~orchestrated \ - --tag ~transient \ - --tag ~skip_signup_disabled \ - --tag ~requires_git_protocol_v2 \ - --tag ~requires_praefect \ - --force-color \ - --order random \ - --format documentation \ - --format RspecJunitFormatter --out tmp/rspec.xml \ - ]" || qa_run_status=$? - - if [ ${qa_run_status} -ne 0 ]; then - release_sha=$(echo "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-${CI_COMMIT_SHA}}" | cut -c1-11); - echo "Errors can be found at https://sentry.gitlab.net/gitlab/gitlab-review-apps/releases/${release_sha}/all-events/."; - fi - - exit ${qa_run_status} + echo "Sentry errors for the current review-app test run can be found via following url:" + echo "https://sentry.gitlab.net/gitlab/gitlab-review-apps/releases/$(echo "${CI_COMMIT_SHA}" | cut -c1-11)/all-events/." artifacts: paths: - qa/tmp reports: - junit: qa/tmp/rspec.xml + junit: qa/tmp/rspec-*.xml expire_in: 7 days when: always -.allure-report-base: - extends: .generate-allure-report-base - stage: post-qa - variables: - GITLAB_AUTH_TOKEN: $GITLAB_QA_MR_ALLURE_REPORT_TOKEN - ALLURE_PROJECT_PATH: $CI_PROJECT_PATH - ALLURE_MERGE_REQUEST_IID: $CI_MERGE_REQUEST_IID - ALLURE_RESULTS_GLOB: qa/tmp/allure-results/* - # Store knapsack report as artifact so the same report is reused across all jobs download-knapsack-report: extends: - .bundle-base - - .review:rules:review-qa-reliable + - .rules:app-or-qa-framework-changes-or-review-scenarios stage: prepare script: - bundle exec rake "knapsack:download[qa]" @@ -99,30 +71,39 @@ download-knapsack-report: review-qa-smoke: extends: - .review-qa-base - - .review:rules:review-qa-smoke - retry: 1 + - .rules:qa-smoke variables: + QA_SCENARIO: Test::Instance::Smoke QA_RUN_TYPE: review-qa-smoke - RSPEC_TAGS: --tag smoke + retry: 1 -review-qa-reliable: +review-qa-blocking: extends: - .review-qa-base - - .review:rules:review-qa-reliable + - .rules:qa-blocking + variables: + QA_SCENARIO: Test::Instance::ReviewBlocking + QA_RUN_TYPE: review-qa-blocking retry: 1 +review-qa-blocking-parallel: + extends: + - review-qa-blocking + - .rules:qa-blocking-parallel parallel: 10 - variables: - QA_RUN_TYPE: review-qa-reliable - RSPEC_TAGS: --tag reliable --tag sanity_feature_flags -review-qa-all: +review-qa-non-blocking: extends: - .review-qa-base - - .review:rules:review-qa-all - parallel: 5 + - .rules:qa-non-blocking variables: - QA_RUN_TYPE: review-qa-all - RSPEC_TAGS: --tag ~reliable --tag ~smoke --tag ~sanity_feature_flags + QA_SCENARIO: Test::Instance::ReviewNonBlocking + QA_RUN_TYPE: review-qa-non-blocking + allow_failure: true +review-qa-non-blocking-parallel: + extends: + - review-qa-non-blocking + - .rules:qa-non-blocking-parallel + parallel: 5 review-performance: extends: @@ -150,27 +131,25 @@ review-performance: performance: performance.json expire_in: 31d -# Generate single report for both smoke and reliable test jobs -# Both job types are essentially the same: -# * always executed -# * always blocking -allure-report-qa-blocking: - extends: - - .allure-report-base - - .review:rules:review-qa-blocking-report - needs: - - review-qa-smoke - - review-qa-reliable - variables: - ALLURE_JOB_NAME: review-qa-blocking - -allure-report-qa-all: +e2e-test-report: extends: - - .allure-report-base - - .review:rules:review-qa-all-report - needs: ["review-qa-all"] + - .generate-allure-report-base + - .rules:app-or-qa-framework-changes-or-review-scenarios + stage: post-qa variables: - ALLURE_JOB_NAME: review-qa-all + ALLURE_JOB_NAME: e2e-review-qa + ALLURE_PROJECT_PATH: $CI_PROJECT_PATH + ALLURE_RESULTS_GLOB: qa/tmp/allure-results/* + ALLURE_MERGE_REQUEST_IID: $CI_MERGE_REQUEST_IID + GITLAB_AUTH_TOKEN: $GITLAB_QA_MR_ALLURE_REPORT_TOKEN + GIT_STRATEGY: none + allow_failure: true + when: always + artifacts: # re-save rspec results for displaying in parent pipeline + expire_in: 1 day + when: always + paths: + - qa/tmp/rspec-*.xml upload-knapsack-report: extends: @@ -183,13 +162,13 @@ upload-knapsack-report: delete-test-resources: extends: - .bundle-base - - .review:rules:review-qa-cleanup + - .rules:app-or-qa-framework-changes-or-review-scenarios stage: post-qa variables: QA_TEST_RESOURCES_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-resources-*.json GITLAB_QA_ACCESS_TOKEN: $REVIEW_APPS_ROOT_TOKEN - COLORIZED_LOGS: "true" script: - export GITLAB_ADDRESS="$QA_GITLAB_URL" - bundle exec rake "test_resources:delete[$QA_TEST_RESOURCES_FILE_PATTERN]" allow_failure: true + when: always diff --git a/.gitlab/ci/review-apps/rules.gitlab-ci.yml b/.gitlab/ci/review-apps/rules.gitlab-ci.yml new file mode 100644 index 00000000000..56d3731bb56 --- /dev/null +++ b/.gitlab/ci/review-apps/rules.gitlab-ci.yml @@ -0,0 +1,81 @@ +# Specific specs passed +.specific-specs: &specific-specs + if: $QA_TESTS != "" + +# No specific specs passed +.all-specs: &all-specs + if: $QA_TESTS == "" + +# No specific specs in mr pipeline +.all-specs-mr: &all-specs-mr + if: $CI_MERGE_REQUEST_IID && $QA_TESTS == "" + when: manual + +# Triggered by change pattern +.app-changes: &app-changes + if: $APP_CHANGE_TRIGGER == "true" + +# QA framework changes present +.qa-framework-changes: &qa-framework-changes + if: $QA_FRAMEWORK_CHANGES == "true" + +.never-when-qa-framework-changes-or-no-specific-specs: + - <<: *qa-framework-changes + when: never + - <<: *all-specs + when: never + +.never-when-specific-specs-always-when-qa-framework-changes: + - <<: *specific-specs + when: never + - *qa-framework-changes + +# ------------------------------------------ +# Test +# ------------------------------------------ +.rules:qa-smoke: + rules: + # always trigger smoke suite if review pipeline got triggered by specific changes in application code + - <<: *app-changes + variables: + QA_TESTS: "" # unset QA_TESTS even if specific tests were inferred from stage label + - *qa-framework-changes + - if: $QA_SUITES =~ /Test::Instance::Smoke/ + +.rules:qa-blocking: + rules: + - <<: *app-changes + when: never + - !reference [.never-when-qa-framework-changes-or-no-specific-specs] + - if: $QA_SUITES =~ /Test::Instance::ReviewBlocking/ +.rules:qa-blocking-parallel: + rules: + # always trigger blocking suite if review pipeline got triggered by specific changes in application code + - <<: *app-changes + variables: + QA_TESTS: "" # unset QA_TESTS even if specific tests were inferred from stage label + - !reference [.never-when-specific-specs-always-when-qa-framework-changes] + - if: $QA_SUITES =~ /Test::Instance::ReviewBlocking/ + +.rules:qa-non-blocking: + rules: + - !reference [.never-when-qa-framework-changes-or-no-specific-specs] + - if: $QA_SUITES =~ /Test::Instance::ReviewNonBlocking/ +.rules:qa-non-blocking-parallel: + rules: + - !reference [.never-when-specific-specs-always-when-qa-framework-changes] + - *all-specs-mr # set full suite to manual when no specific specs passed in mr + - if: $QA_SUITES =~ /Test::Instance::ReviewNonBlocking/ + +# ------------------------------------------ +# Prepare/Report +# ------------------------------------------ +# if no rules for test execution are matched, pipeline will not have e2e test jobs +# so we need to skip knapsack, allure and test resource deletion jobs as well +.rules:app-or-qa-framework-changes-or-review-scenarios: + rules: + - *app-changes + - *qa-framework-changes + - if: $QA_SUITES =~ /Test::Instance::Smoke/ + - if: $QA_SUITES =~ /Test::Instance::ReviewBlocking/ + - if: $QA_SUITES =~ /Test::Instance::ReviewNonBlocking/ diff --git a/.gitlab/ci/review-apps/skip-qa.gitlab-ci.yml b/.gitlab/ci/review-apps/skip-qa.gitlab-ci.yml deleted file mode 100644 index 1305673a4d8..00000000000 --- a/.gitlab/ci/review-apps/skip-qa.gitlab-ci.yml +++ /dev/null @@ -1,13 +0,0 @@ -stages: - - review - -include: - - local: .gitlab/ci/global.gitlab-ci.yml - - local: .gitlab/ci/rules.gitlab-ci.yml - -no-op: - extends: - - .review:rules:start-review-app-pipeline - stage: review - script: - - echo "Skip Review App because the MR includes only quarantine changes" |