diff options
-rw-r--r-- | .gitlab-ci.yml | 156 | ||||
-rw-r--r-- | doc/development/testing_guide/review_apps.md | 84 | ||||
-rwxr-xr-x | scripts/review_apps/review-apps.sh | 145 |
3 files changed, 289 insertions, 96 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6cf7475afa..767df35bb42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -439,6 +439,50 @@ setup-test-env: - config/secrets.yml - vendor/gitaly-ruby +# GitLab Review apps +.review-base: &review-base + <<: *dedicated-no-docs-no-db-pull-cache-job + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base + stage: test + cache: {} + dependencies: [] + environment: &review-environment + name: review/${CI_COMMIT_REF_NAME} + url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} + only: + refs: + - branches@gitlab-org/gitlab-ce + - branches@gitlab-org/gitlab-ee + kubernetes: active + except: + refs: + - master + - /(^docs[\/-].*|.*-docs$)/ + before_script: [] + +.review-docker: &review-docker + <<: *review-base + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine + services: + - docker:stable-dind + tags: + - gitlab-org + - docker + variables: &review-docker-variables + GIT_DEPTH: "1" + DOCKER_DRIVER: overlay2 + DOCKER_HOST: tcp://docker:2375 + LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly" + QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}" + +build-qa-image: + <<: *review-docker + stage: prepare + script: + - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} ./qa/ + - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY} + - time docker push ${QA_IMAGE} + danger-review: <<: *pull-cache image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger @@ -875,85 +919,97 @@ no_ee_check: - //@gitlab-org/gitlab-ce # GitLab Review apps -review: - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - stage: test - allow_failure: true - before_script: - - gem install gitlab --no-document +review-deploy: + <<: *review-base variables: GIT_DEPTH: "1" - HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG" - DOMAIN: "-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN" + HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" + DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" GITLAB_HELM_CHART_REF: "master" + API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}" + environment: + <<: *review-environment + on_stop: review-stop + before_script: + - apk update && apk add jq + - gem install gitlab --no-document script: - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) - source ./scripts/review_apps/review-apps.sh + - wait_for_job_to_be_done "gitlab:assets:compile" - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng - check_kube_domain - download_gitlab_chart - ensure_namespace - install_tiller - install_external_dns - - deploy - environment: - name: review/$CI_COMMIT_REF_NAME - url: https://gitlab-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN - on_stop: stop_review - only: - refs: - - branches@gitlab-org/gitlab-ce - - branches@gitlab-org/gitlab-ee - kubernetes: active - except: - refs: - - master - - /(^docs[\/-].*|.*-docs$)/ + - time deploy + - add_license + +.review-qa-base: &review-qa-base + <<: *review-docker + variables: + <<: *review-docker-variables + API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}" + QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" + QA_CAN_TEST_GIT_PROTOCOL_V2: "false" + GITLAB_USERNAME: "root" + GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITLAB_ADMIN_USERNAME: "root" + GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" + EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" + artifacts: + paths: + - ./qa/gitlab-qa-run-* + expire_in: 7 days + when: always + before_script: + - echo "${QA_IMAGE}" + - echo "${CI_ENVIRONMENT_URL}" + - apk update && apk add curl jq + - source ./scripts/review_apps/review-apps.sh + - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} + - wait_for_job_to_be_done "review-deploy" + +review-qa-smoke: + <<: *review-qa-base + script: + - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" + +review-qa-all: + <<: *review-qa-base + script: + - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" + when: manual -stop_review: +review-stop: + <<: *review-base <<: *single-script-job image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - stage: test allow_failure: true - cache: {} - dependencies: [] variables: + <<: *single-script-job-variables SCRIPT_NAME: "review_apps/review-apps.sh" + when: manual + environment: + <<: *review-environment + action: stop script: - source $(basename "${SCRIPT_NAME}") - delete - cleanup - when: manual - environment: - name: review/$CI_COMMIT_REF_NAME - action: stop - only: - refs: - - branches@gitlab-org/gitlab-ce - - branches@gitlab-org/gitlab-ee - kubernetes: active - except: - - master - - /(^docs[\/-].*|.*-docs$)/ -schedule:review_apps_cleanup: - <<: *dedicated-no-docs-pull-cache-job - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base +schedule:review-cleanup: + <<: *review-base stage: build allow_failure: true - cache: {} - dependencies: [] - before_script: - - gem install gitlab --no-document variables: GIT_DEPTH: "1" - script: - - ruby -rrubygems scripts/review_apps/automated_cleanup.rb environment: name: review/auto-cleanup - action: stop only: refs: - schedules@gitlab-org/gitlab-ce @@ -962,3 +1018,7 @@ schedule:review_apps_cleanup: except: - tags - /(^docs[\/-].*|.*-docs$)/ + before_script: + - gem install gitlab --no-document + script: + - ruby -rrubygems scripts/review_apps/automated_cleanup.rb diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index 1830641431e..a6ed9e85a41 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -7,31 +7,36 @@ Review Apps are automatically deployed by each pipeline, both in ## How does it work? 1. On every [pipeline][gitlab-pipeline] during the `test` stage, the - [`review` job][review-job] is automatically started. -1. The `review` job [triggers a pipeline][cng-pipeline] in the - [`CNG-mirror`][cng-mirror] project. - - We use the `CNG-mirror` project so that the `CNG`, (**C**loud **N**ative - **G**itLab), project's registry is not overloaded with a lot of transient - Docker images. -1. The `CNG-mirror` pipeline creates the Docker images of each component (e.g. - `gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.) based on the commit from the - [GitLab pipeline][gitlab-pipeline] and store them in its - [registry][cng-mirror-registry]. -1. Once all images are built, the Review App is deployed using - [the official GitLab Helm chart][helm-chart] to the - [`review-apps-ee` Kubernetes cluster on GCP][review-apps-ee] - - The actual scripts used to deploy the Review App can be found at - [`scripts/review_apps/review-apps.sh`][review-apps.sh] - - These scripts are basically - [our official Auto DevOps scripts][Auto-DevOps.gitlab-ci.yml] where the - default CNG images are overridden with the images built and stored in the - [`CNG-mirror` project's registry][cng-mirror-registry]. - - Since we're using [the official GitLab Helm chart][helm-chart], this means - you get a dedicated environment for your branch that's very close to what it - would look in production. -1. Once the `review` job succeeds, you should be able to use your Review App - thanks to the direct link to it from the MR widget. The default username is - `root` and its password can be found in the 1Password secure note named + [`review-deploy`][review-deploy-job] job is automatically started. +1. The `review-deploy` job: + 1. Waits for the `gitlab:assets:compile` job to finish since the + [`CNG-mirror`][cng-mirror] pipeline triggerred in the following step + depends on it. + 1. [Triggers a pipeline][cng-pipeline] in the [`CNG-mirror`][cng-mirror] + project. + - We use the `CNG-mirror` project so that the `CNG`, (**C**loud + **N**ative **G**itLab), project's registry is not overloaded with a + lot of transient Docker images. + - The `CNG-mirror` pipeline creates the Docker images of each component + (e.g. `gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.) based on the + commit from the [GitLab pipeline][gitlab-pipeline] and store them in + its [registry][cng-mirror-registry]. + 1. Once all images are built by [`CNG-mirror`][cng-mirror], the Review App + is deployed using [the official GitLab Helm chart][helm-chart] to the + [`review-apps-ce`][review-apps-ce] / [`review-apps-ee`][review-apps-ee] + Kubernetes cluster on GCP. + - The actual scripts used to deploy the Review App can be found at + [`scripts/review_apps/review-apps.sh`][review-apps.sh]. + - These scripts are basically + [our official Auto DevOps scripts][Auto-DevOps.gitlab-ci.yml] where the + default CNG images are overridden with the images built and stored in the + [`CNG-mirror` project's registry][cng-mirror-registry]. + - Since we're using [the official GitLab Helm chart][helm-chart], this means + you get a dedicated environment for your branch that's very close to what + it would look in production. +1. Once the `review-deploy` job succeeds, you should be able to use your Review + App thanks to the direct link to it from the MR widget. The default username + is `root` and its password can be found in the 1Password secure note named **gitlab-{ce,ee} Review App's root password** (note that there's currently [a bug where the default password seems to be overridden][password-bug]). @@ -39,16 +44,23 @@ Review Apps are automatically deployed by each pipeline, both in - The Kubernetes cluster is connected to the `gitlab-{ce,ee}` projects using [GitLab's Kubernetes integration][gitlab-k8s-integration]. This basically - allows to have a link to the Review App directly from the merge request widget. -- The manual `stop_review` in the `test` stage can be used to stop a Review App - manually, and is also started by GitLab once a branch is deleted. -- Review Apps are cleaned up regularly using a pipeline schedule that runs - the [`scripts/review_apps/automated_cleanup.rb`][automated_cleanup.rb] script. + allows to have a link to the Review App directly from the merge request + widget. - If the Review App deployment fails, you can simply retry it (there's no need - to run the `stop_review` job first). -- If you're unable to log in using the `root` username and password, you may - encounter [this bug][password-bug]. Stop the Review App via the `stop_review` - manual job and then retry the `review` job to redeploy the Review App. + to run the [`review-stop`][gitlab-ci-yml] job first). +- The manual [`review-stop`][gitlab-ci-yml] in the `test` stage can be used to + stop a Review App manually, and is also started by GitLab once a branch is + deleted. +- Review Apps are cleaned up regularly using a pipeline schedule that runs + the [`schedule:review-cleanup`][gitlab-ci-yml] job. + +## QA runs + +On every [pipeline][gitlab-pipeline] during the `test` stage, the +`review-qa-smoke` job is automatically started: it runs the smoke QA suite. +You can also manually start the `review-qa-all`: it runs the full QA suite. + +Note that both jobs first wait for the `review-deploy` job to be finished. ## Frequently Asked Questions @@ -74,15 +86,17 @@ find a way to limit it to only us.** > This isn't enabled for forks. [gitlab-pipeline]: https://gitlab.com/gitlab-org/gitlab-ce/pipelines/35850709 -[review-job]: https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/118076368 +[review-deploy-job]: https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/118076368 [cng-mirror]: https://gitlab.com/gitlab-org/build/CNG-mirror [cng-pipeline]: https://gitlab.com/gitlab-org/build/CNG-mirror/pipelines/35883435 [cng-mirror-registry]: https://gitlab.com/gitlab-org/build/CNG-mirror/container_registry [helm-chart]: https://gitlab.com/charts/gitlab/ +[review-apps-ce]: https://console.cloud.google.com/kubernetes/clusters/details/us-central1-a/review-apps-ce?project=gitlab-review-apps [review-apps-ee]: https://console.cloud.google.com/kubernetes/clusters/details/us-central1-b/review-apps-ee?project=gitlab-review-apps [review-apps.sh]: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/scripts/review_apps/review-apps.sh [automated_cleanup.rb]: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/scripts/review_apps/automated_cleanup.rb [Auto-DevOps.gitlab-ci.yml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +[gitlab-ci-yml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.gitlab-ci.yml [gitlab-k8s-integration]: https://docs.gitlab.com/ee/user/project/clusters/index.html [password-bug]: https://gitlab.com/gitlab-org/gitlab-ce/issues/53621 diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 00e23f12bc0..f3f788e0217 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -1,6 +1,8 @@ [[ "$TRACE" ]] && set -x export TILLER_NAMESPACE="$KUBE_NAMESPACE" +function echoerr() { printf "\033[0;31m%s\n\033[0m" "$*" >&2; } + function check_kube_domain() { if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set" @@ -88,19 +90,16 @@ function deploy() { replicas="1" service_enabled="false" postgres_enabled="$POSTGRES_ENABLED" - gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce" - gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce" - gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce" - gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" - gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" - gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce" - - if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then - gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee" - gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee" - gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee" - gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee" - fi + + IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror" + IMAGE_VERSION="${CI_PROJECT_NAME#gitlab-}" + gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${IMAGE_VERSION}" + gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-${IMAGE_VERSION}" + gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-unicorn-${IMAGE_VERSION}" + gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-${IMAGE_VERSION}" + gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly" + gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell" + gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}" # canary uses stable db [[ "$track" == "canary" ]] && postgres_enabled="false" @@ -155,6 +154,8 @@ HELM_CMD=$(cat << EOF --set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_NAME" \ --set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \ --set gitlab.unicorn.image.tag="$CI_COMMIT_REF_NAME" \ + --set gitlab.task-runner.image.repository="$gitlab_task_runner_image_repository" \ + --set gitlab.task-runner.image.tag="$CI_COMMIT_REF_NAME" \ --set gitlab.gitaly.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" \ --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \ --set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \ @@ -229,3 +230,121 @@ function install_external_dns() { --set rbac.create="true" fi } + +function get_pod() { + local app_name="${1}" + local status="${2-Running}" + get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name" + echoerr "Running '${get_pod_cmd}'" + + while true; do + local pod_name="$(eval $get_pod_cmd)" + [[ "${pod_name}" == "" ]] || break + + echoerr "Waiting till '${app_name}' pod is ready"; + sleep 5; + done + + echoerr "The pod name is '${pod_name}'." + echo "${pod_name}" +} + +function add_license() { + if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi + + task_runner_pod=$(get_pod "task-runner"); + if [ -z "${task_runner_pod}" ]; then echo "Task runner pod not found" && return; fi + + echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab + kubectl -n "$KUBE_NAMESPACE" cp /tmp/license.gitlab ${task_runner_pod}:/tmp/license.gitlab + rm /tmp/license.gitlab + + kubectl -n "$KUBE_NAMESPACE" exec -it ${task_runner_pod} -- /srv/gitlab/bin/rails runner -e production \ + ' + content = File.read("/tmp/license.gitlab").strip; + FileUtils.rm_f("/tmp/license.gitlab"); + + unless License.where(data:content).empty? + puts "License already exists"; + Kernel.exit 0; + end + + unless License.new(data: content).save + puts "Could not add license"; + Kernel.exit 0; + end + + puts "License added"; + ' +} + +function get_job_id() { + local job_name="${1}" + local query_string="${2:+&${2}}" + + local max_page=3 + local page=1 + + while true; do + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}" + echoerr "GET ${url}" + + local job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".[] | select(.name == \"${job_name}\") | .id") + [[ "${job_id}" == "" && "${page}" -lt "$max_page" ]] || break + + ((page++)) + done + + if [[ "${job_id}" == "" ]]; then + echoerr "The '${job_name}' job ID couldn't be retrieved!" + else + echoerr "The '${job_name}' job ID is ${job_id}" + echo "${job_id}" + fi +} + +function play_job() { + local job_name="${1}" + local job_id=$(get_job_id "${job_name}" "scope=manual"); + if [ -z "${job_id}" ]; then return; fi + + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play" + echoerr "POST ${url}" + + local job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".web_url") + echo "Manual job '${job_name}' started at: ${job_url}" +} + +function wait_for_job_to_be_done() { + local job_name="${1}" + local query_string="${2}" + local job_id=$(get_job_id "${job_name}" "${query_string}"); + if [ -z "${job_id}" ]; then return; fi + + echoerr "Waiting for the '${job_name}' job to finish..." + + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}" + echo "GET ${url}" + + # In case the job hasn't finished yet. Keep trying until the job times out. + local interval=30 + local elapsed=0 + while true; do + local job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".status" | sed -e s/\"//g) + [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break + + printf "." + ((elapsed+=$interval)) + sleep ${interval} + done + + echoerr "Waited '${job_name}' for ${elapsed} seconds." + + if [[ "${job_status}" == "failed" ]]; then + echo "The '${job_name}' failed." + elif [[ "${job_status}" == "manual" ]]; then + echo "The '${job_name}' is manual." + else + echo "The '${job_name}' passed." + fi +} |