diff options
Diffstat (limited to '.gitlab/ci/package-and-test/main.gitlab-ci.yml')
| -rw-r--r-- | .gitlab/ci/package-and-test/main.gitlab-ci.yml | 600 |
1 files changed, 600 insertions, 0 deletions
diff --git a/.gitlab/ci/package-and-test/main.gitlab-ci.yml b/.gitlab/ci/package-and-test/main.gitlab-ci.yml new file mode 100644 index 00000000000..d96da7744ab --- /dev/null +++ b/.gitlab/ci/package-and-test/main.gitlab-ci.yml @@ -0,0 +1,600 @@ +# E2E tests pipeline loaded dynamically by script: scripts/generate-e2e-pipeline + +include: + - local: .gitlab/ci/global.gitlab-ci.yml + - local: .gitlab/ci/package-and-test/rules.gitlab-ci.yml + - local: .gitlab/ci/package-and-test/variables.gitlab-ci.yml + - project: gitlab-org/quality/pipeline-common + ref: 1.2.1 + file: + - /ci/base.gitlab-ci.yml + - /ci/allure-report.yml + - /ci/knapsack-report.yml + +stages: + - test + - report + - notify + +# ========================================== +# Templates +# ========================================== +.parallel: + parallel: 5 + variables: + QA_KNAPSACK_REPORT_PATH: $CI_PROJECT_DIR/qa/knapsack + +.ruby-image: + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.3 + +.bundle-install: + extends: + - .gitlab-qa-install + - .qa-cache + variables: + RUN_WITH_BUNDLE: "true" # installs and runs gitlab-qa via bundler + QA_PATH: qa + +.omnibus-env: + variables: + BUILD_ENV: build.env + script: + - | + SECURITY_SOURCES=$([[ ! "$CI_PROJECT_NAMESPACE" =~ ^gitlab-org\/security ]] || echo "true") + echo "SECURITY_SOURCES=${SECURITY_SOURCES:-false}" > $BUILD_ENV + echo "OMNIBUS_GITLAB_CACHE_UPDATE=${OMNIBUS_GITLAB_CACHE_UPDATE:-false}" >> $BUILD_ENV + for version_file in *_VERSION; do echo "$version_file=$(cat $version_file)" >> $BUILD_ENV; done + echo "Built environment file for omnibus build:" + cat $BUILD_ENV + artifacts: + reports: + dotenv: $BUILD_ENV + +.update-script: + script: + - export QA_COMMAND="bundle exec gitlab-qa Test::Omnibus::UpdateFromPrevious $RELEASE $GITLAB_VERSION $UPDATE_TYPE -- $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS" + - echo "Running - '$QA_COMMAND'" + - eval "$QA_COMMAND" + +.qa: + extends: + - .qa-base + - .bundle-install + - .gitlab-qa-report + stage: test + tags: + - e2e + needs: + - trigger-omnibus + - download-knapsack-report + variables: + QA_GENERATE_ALLURE_REPORT: "true" + QA_CAN_TEST_PRAEFECT: "false" + QA_INTERCEPT_REQUESTS: "true" + QA_RUN_TYPE: e2e-package-and-test + TEST_LICENSE_MODE: $QA_TEST_LICENSE_MODE + EE_LICENSE: $QA_EE_LICENSE + GITHUB_ACCESS_TOKEN: $QA_GITHUB_ACCESS_TOKEN + GITLAB_QA_ADMIN_ACCESS_TOKEN: $QA_ADMIN_ACCESS_TOKEN + +# ========================================== +# Prepare stage +# ========================================== +trigger-omnibus-env: + extends: + - .omnibus-env + - .rules:prepare + stage: .pre + +trigger-omnibus: + extends: .rules:prepare + stage: .pre + needs: + - trigger-omnibus-env + inherit: + variables: false + variables: + GITALY_SERVER_VERSION: $GITALY_SERVER_VERSION + GITLAB_ELASTICSEARCH_INDEXER_VERSION: $GITLAB_ELASTICSEARCH_INDEXER_VERSION + GITLAB_KAS_VERSION: $GITLAB_KAS_VERSION + GITLAB_METRICS_EXPORTER_VERSION: $GITLAB_METRICS_EXPORTER_VERSION + GITLAB_PAGES_VERSION: $GITLAB_PAGES_VERSION + GITLAB_SHELL_VERSION: $GITLAB_SHELL_VERSION + GITLAB_WORKHORSE_VERSION: $GITLAB_WORKHORSE_VERSION + GITLAB_VERSION: $CI_COMMIT_SHA + IMAGE_TAG: $CI_COMMIT_SHA + TOP_UPSTREAM_SOURCE_PROJECT: $CI_PROJECT_PATH + SECURITY_SOURCES: $SECURITY_SOURCES + CACHE_UPDATE: $OMNIBUS_GITLAB_CACHE_UPDATE + SKIP_QA_DOCKER: "true" + SKIP_QA_TEST: "true" + ee: "true" + trigger: + project: gitlab-org/build/omnibus-gitlab-mirror + strategy: depend + +download-knapsack-report: + extends: + - .bundle-install + - .ruby-image + - .rules:prepare + stage: .pre + script: + - bundle exec rake "knapsack:download[test]" + allow_failure: true + artifacts: + paths: + - qa/knapsack/ee-*.json + expire_in: 1 day + +# e2e test jobs run on separate runner which has separate cache setup +cache-gems: + extends: + - .bundle-install + - .ruby-image + - .qa-cache-push + - .rules:prepare + stage: .pre + tags: + - e2e + script: + - echo "Populated qa cache" + +# ========================================== +# Test stage +# ========================================== + +# ------------------------------------------ +# Manual jobs +# ------------------------------------------ + +# Run manual quarantine job +# this job requires passing QA_SCENARIO variable +# and optionally QA_TESTS to run specific quarantined tests +_ee:quarantine: + extends: + - .qa + - .rules:test:quarantine + needs: + - trigger-omnibus + stage: test + allow_failure: true + variables: + QA_RSPEC_TAGS: --tag quarantine + +# ------------------------------------------ +# FF changes +# ------------------------------------------ + +# Run specs with feature flags set to the opposite of the default state +ee:instance-parallel-ff-inverse: + extends: + - .qa + - .parallel + variables: + QA_SCENARIO: Test::Instance::Image + QA_KNAPSACK_REPORT_NAME: ee-instance-parallel + GITLAB_QA_OPTS: --set-feature-flags $QA_FEATURE_FLAGS + rules: + - !reference [.rules:test:feature-flags-deleted, rules] # skip job when only change is ff deletion + - !reference [.rules:test:feature-flags-set, rules] + +# ------------------------------------------ +# Jobs with parallel variant +# ------------------------------------------ +ee:instance: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::Image + rules: + - !reference [.rules:test:qa-non-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ +ee:instance-parallel: + extends: + - .parallel + - ee:instance + rules: + - !reference [.rules:test:feature-flags-set, rules] # always run instance-parallel to validate ff change + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + +ee:praefect: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::Praefect + QA_CAN_TEST_PRAEFECT: "true" + rules: + - !reference [.rules:test:qa-non-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ +ee:praefect-parallel: + extends: + - .parallel + - ee:praefect + rules: + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + +ee:relative-url: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::RelativeUrl + rules: + - !reference [.rules:test:qa-non-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ +ee:relative-url-parallel: + extends: + - .parallel + - ee:relative-url + rules: + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + +ee:decomposition-single-db: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::Image + GITLAB_QA_OPTS: --omnibus-config decomposition_single_db + rules: + - !reference [.rules:test:qa-non-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ +ee:decomposition-single-db-parallel: + extends: + - .parallel + - ee:decomposition-single-db + rules: + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + +ee:decomposition-multiple-db: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::Image + GITLAB_QA_OPTS: --omnibus-config decomposition_multiple_db + rules: + - !reference [.rules:test:qa-non-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ +ee:decomposition-multiple-db-parallel: + extends: + - .parallel + - ee:decomposition-multiple-db + rules: + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + +ee:object-storage: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::Image + QA_RSPEC_TAGS: --tag object_storage + GITLAB_QA_OPTS: --omnibus-config object_storage + rules: + - !reference [.rules:test:qa-non-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/ +ee:object-storage-parallel: + extends: ee:object-storage + parallel: 2 + rules: + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/ + +ee:object-storage-aws: + extends: ee:object-storage + variables: + AWS_S3_ACCESS_KEY: $QA_AWS_S3_ACCESS_KEY + AWS_S3_BUCKET_NAME: $QA_AWS_S3_BUCKET_NAME + AWS_S3_KEY_ID: $QA_AWS_S3_KEY_ID + AWS_S3_REGION: $QA_AWS_S3_REGION + GITLAB_QA_OPTS: --omnibus-config object_storage_aws +ee:object-storage-aws-parallel: + extends: ee:object-storage-aws + parallel: 2 + rules: + - !reference [ee:object-storage-parallel, rules] + +ee:object-storage-gcs: + extends: ee:object-storage + variables: + GCS_BUCKET_NAME: $QA_GCS_BUCKET_NAME + GOOGLE_PROJECT: $QA_GOOGLE_PROJECT + GOOGLE_JSON_KEY: $QA_GOOGLE_JSON_KEY + GOOGLE_CLIENT_EMAIL: $QA_GOOGLE_CLIENT_EMAIL + GITLAB_QA_OPTS: --omnibus-config object_storage_gcs +ee:object-storage-gcs-parallel: + extends: ee:object-storage-gcs + parallel: 2 + rules: + - !reference [ee:object-storage-parallel, rules] + +ee:packages: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::Image + QA_RSPEC_TAGS: --tag packages + GITLAB_QA_OPTS: --omnibus-config packages + rules: + - !reference [.rules:test:qa-non-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::Packages/ +ee:packages-parallel: + extends: ee:packages + parallel: 2 + rules: + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::Packages/ + +# ------------------------------------------ +# Non parallel jobs +# ------------------------------------------ +ee:update-minor: + extends: + - .qa + - .update-script + variables: + UPDATE_TYPE: minor + QA_RSPEC_TAGS: --tag smoke + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Instance::Smoke/ + +ee:update-major: + extends: + - .qa + - .update-script + variables: + UPDATE_TYPE: major + QA_RSPEC_TAGS: --tag smoke + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Instance::Smoke/ + +ee:gitaly-cluster: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::GitalyCluster + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::GitalyCluster/ + +ee:group-saml: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::GroupSAML + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::GroupSAML/ + +ee:instance-saml: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::InstanceSAML + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::InstanceSAML/ + +ee:jira: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::Jira + JIRA_ADMIN_USERNAME: $QA_JIRA_ADMIN_USERNAME + JIRA_ADMIN_PASSWORD: $QA_JIRA_ADMIN_PASSWORD + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::Jira/ + +ee:ldap-no-server: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::LDAPNoServer + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::LDAPNoServer/ + +ee:ldap-tls: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::LDAPTLS + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::LDAPTLS/ + +ee:ldap-no-tls: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::LDAPNoTLS + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::LDAPNoTLS/ + +ee:mtls: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::MTLS + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::MTLS/ + +ee:mattermost: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::Mattermost + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::Mattermost/ + +ee:registry: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::Registry + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::Registry/ + +ee:registry-with-cdn: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::RegistryWithCDN + GCS_CDN_BUCKET_NAME: $QA_GCS_CDN_BUCKET_NAME + GOOGLE_CDN_LB: $QA_GOOGLE_CDN_LB + GOOGLE_CDN_JSON_KEY: $QA_GOOGLE_CDN_JSON_KEY + GOOGLE_CDN_SIGNURL_KEY: $QA_GOOGLE_CDN_SIGNURL_KEY + GOOGLE_CDN_SIGNURL_KEY_NAME: $QA_GOOGLE_CDN_SIGNURL_KEY_NAME + before_script: + - unset GITLAB_QA_ADMIN_ACCESS_TOKEN + - !reference [.qa, before_script] + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::RegistryWithCDN/ + +ee:repository-storage: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::RepositoryStorage + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Instance::RepositoryStorage/ + +ee:service-ping-disabled: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::ServicePingDisabled + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::ServicePingDisabled/ + +ee:smtp: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::SMTP + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::SMTP/ + +ee:cloud-activation: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::Image + QA_RSPEC_TAGS: --tag cloud_activation + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::CloudActivation/ + +ee:large-setup: + extends: .qa + variables: + QA_SCENARIO: Test::Instance::Image + QA_RSPEC_TAGS: --tag can_use_large_setup + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Instance::LargeSetup/ + +ee:metrics: + extends: .qa + variables: + QA_SCENARIO: Test::Integration::Metrics + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Instance::Metrics/ + +ee:elasticsearch: + extends: .qa + variables: + QA_SCENARIO: "Test::Integration::Elasticsearch" + before_script: + - unset ELASTIC_URL # unset url which is globally defined in .gitlab-ci.yml + - !reference [.qa, before_script] + rules: + - !reference [.rules:test:qa, rules] + - if: $QA_SUITES =~ /Test::Integration::Elasticsearch/ + +ee:registry-object-storage-tls: + extends: ee:object-storage-aws + variables: + QA_SCENARIO: Test::Integration::RegistryTLS + QA_RSPEC_TAGS: "" + GITLAB_TLS_CERTIFICATE: $QA_GITLAB_TLS_CERTIFICATE + GITLAB_QA_OPTS: --omnibus-config registry_object_storage + +# ========================================== +# Post test stage +# ========================================== +e2e-test-report: + extends: + - .generate-allure-report-base + - .rules:report:allure-report + stage: report + variables: + GITLAB_AUTH_TOKEN: $GITLAB_QA_MR_ALLURE_REPORT_TOKEN + ALLURE_PROJECT_PATH: $CI_PROJECT_PATH + ALLURE_MERGE_REQUEST_IID: $CI_MERGE_REQUEST_IID + ALLURE_JOB_NAME: e2e-package-and-test + GIT_STRATEGY: none + artifacts: # save rspec results for displaying in parent pipeline + expire_in: 1 day + when: always + paths: + - gitlab-qa-run-*/**/rspec-*.xml + +upload-knapsack-report: + extends: + - .generate-knapsack-report-base + - .bundle-install + - .ruby-image + - .rules:report:process-results + stage: report + when: always + +relate-test-failures: + extends: + - .bundle-install + - .ruby-image + - .rules:report:process-results + stage: report + variables: + QA_FAILURES_REPORTING_PROJECT: gitlab-org/gitlab + QA_FAILURES_MAX_DIFF_RATIO: "0.15" + GITLAB_QA_ACCESS_TOKEN: $GITLAB_QA_PRODUCTION_ACCESS_TOKEN + when: on_failure + script: + - | + bundle exec gitlab-qa-report \ + --relate-failure-issue "$CI_PROJECT_DIR/gitlab-qa-run-*/**/rspec-*.json" \ + --project "$QA_FAILURES_REPORTING_PROJECT" \ + --max-diff-ratio "$QA_FAILURES_MAX_DIFF_RATIO" + +generate-test-session: + extends: + - .bundle-install + - .ruby-image + - .rules:report:process-results + stage: report + variables: + QA_TESTCASE_SESSIONS_PROJECT: gitlab-org/quality/testcase-sessions + GITLAB_QA_ACCESS_TOKEN: $QA_TEST_SESSION_TOKEN + GITLAB_CI_API_TOKEN: $QA_GITLAB_CI_TOKEN + when: always + script: + - | + bundle exec gitlab-qa-report \ + --generate-test-session "$CI_PROJECT_DIR/gitlab-qa-run-*/**/rspec-*.json" \ + --project "$QA_TESTCASE_SESSIONS_PROJECT" + artifacts: + when: always + expire_in: 1d + paths: + - qa/REPORT_ISSUE_URL + +notify-slack: + extends: + - .notify-slack-qa + - .bundle-install + - .ruby-image + - .rules:report:process-results + stage: notify + variables: + ALLURE_JOB_NAME: e2e-package-and-test + SLACK_ICON_EMOJI: ci_failing + STATUS_SYM: ☠️ + STATUS: failed + when: on_failure + script: + - bundle exec gitlab-qa-report --prepare-stage-reports "$CI_PROJECT_DIR/gitlab-qa-run-*/**/rspec-*.xml" # generate summary + - !reference [.notify-slack-qa, script] |