From 42102cd13aac63a81eb0208ebc74e2301799c621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 27 Mar 2019 10:36:01 +0100 Subject: Add more logging in the Review Apps deployment steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- scripts/review_apps/review-apps.sh | 286 +++++++++++++++++++++---------------- 1 file changed, 161 insertions(+), 125 deletions(-) (limited to 'scripts/review_apps') diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 32fce946c17..e324f972640 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -1,12 +1,115 @@ [[ "$TRACE" ]] && set -x export TILLER_NAMESPACE="$KUBE_NAMESPACE" -function echoerr() { printf "\033[0;31m%s\n\033[0m" "$*" >&2; } -function echoinfo() { printf "\033[0;33m%s\n\033[0m" "$*" >&2; } +function echoerr() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2; + else + printf "\033[0;31m%s\n\033[0m" "${1}" >&2; + fi +} + +function echoinfo() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2; + else + printf "\033[0;33m%s\n\033[0m" "${1}" >&2; + fi +} + +function deployExists() { + local namespace="${1}" + local deploy="${2}" + echoinfo "Checking if ${deploy} exists in the ${namespace} namespace..." true + + helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1 + local deploy_exists=$? + + echoinfo "Deployment status for ${deploy} is ${deploy_exists}" + return $deploy_exists +} + +function previousDeployFailed() { + set +e + local deploy="${1}" + echoinfo "Checking for previous deployment of ${deploy}" true + + helm status ${deploy} >/dev/null 2>&1 + local status=$? + + # if `status` is `0`, deployment exists, has a status + if [ $status -eq 0 ]; then + echoinfo "Previous deployment found, checking status..." + deployment_status=$(helm status ${deploy} | grep ^STATUS | cut -d' ' -f2) + echoinfo "Previous deployment state: ${deployment_status}" + if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then + status=0; + else + status=1; + fi + else + echoerr "Previous deployment NOT found." + fi + set -e + return $status +} + +function delete() { + if [ -z "$CI_ENVIRONMENT_SLUG" ]; then + echoerr "No release given, aborting the delete!" + return + fi + + local track="${1-stable}" + local name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + echoinfo "Deleting release '$name'..." true + + helm delete --purge "$name" || true +} + +function cleanup() { + if [ -z "$CI_ENVIRONMENT_SLUG" ]; then + echoerr "No release given, aborting the delete!" + return + fi + + echoinfo "Cleaning up '$CI_ENVIRONMENT_SLUG'..." true + + kubectl -n "$KUBE_NAMESPACE" delete \ + ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa \ + -l release="$CI_ENVIRONMENT_SLUG" \ + || true +} + +function get_pod() { + local app_name="${1}" + local status="${2-Running}" + get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name" + echoinfo "Running '${get_pod_cmd}'" true + + while true; do + local pod_name="$(eval $get_pod_cmd)" + [[ "${pod_name}" == "" ]] || break + + echoinfo "Waiting till '${app_name}' pod is ready"; + sleep 5; + done + + echoinfo "The pod name is '${pod_name}'." + echo "${pod_name}" +} function perform_review_app_deployment() { check_kube_domain - download_gitlab_chart ensure_namespace install_tiller install_external_dns @@ -15,6 +118,8 @@ function perform_review_app_deployment() { } function check_kube_domain() { + echoinfo "Checking that Kube domain exists..." true + if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set" echo "You can do it in Auto DevOps project settings or defining a variable at group or project level" @@ -25,36 +130,56 @@ function check_kube_domain() { fi } -function download_gitlab_chart() { - curl -o gitlab.tar.bz2 https://gitlab.com/charts/gitlab/-/archive/$GITLAB_HELM_CHART_REF/gitlab-$GITLAB_HELM_CHART_REF.tar.bz2 - tar -xjf gitlab.tar.bz2 - cd gitlab-$GITLAB_HELM_CHART_REF - - helm init --client-only - helm repo add gitlab https://charts.gitlab.io - helm dependency update - helm dependency build -} - function ensure_namespace() { + echoinfo "Ensuring the ${KUBE_NAMESPACE} namespace exists..." true + kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" } function install_tiller() { - echo "Checking Tiller..." + echoinfo "Checking deployment/tiller-deploy status in the ${TILLER_NAMESPACE} namespace..." true + + echoinfo "Initiating the Helm client..." + helm init --client-only + helm init \ --upgrade \ --replicas 2 + kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy" + if ! helm version --debug; then echo "Failed to init Tiller." return 1 fi - echo "" +} + +function install_external_dns() { + local release_name="dns-gitlab-review-app" + local domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}') + echoinfo "Installing external DNS for domain ${domain}..." true + + if ! deployExists "${KUBE_NAMESPACE}" "${release_name}" || previousDeployFailed "${release_name}" ; then + echoinfo "Installing external-dns Helm chart" + helm repo update + helm install stable/external-dns \ + -n "${release_name}" \ + --namespace "${KUBE_NAMESPACE}" \ + --set provider="aws" \ + --set aws.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \ + --set aws.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \ + --set aws.zoneType="public" \ + --set domainFilters[0]="${domain}" \ + --set txtOwnerId="${KUBE_NAMESPACE}" \ + --set rbac.create="true" \ + --set policy="sync" + else + echoinfo "The external-dns Helm chart is already successfully deployed." + fi } function create_secret() { - echo "Create secret..." + echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password secret in the ${KUBE_NAMESPACE} namespace..." true kubectl create secret generic -n "$KUBE_NAMESPACE" \ $CI_ENVIRONMENT_SLUG-gitlab-initial-root-password \ @@ -62,43 +187,28 @@ function create_secret() { --dry-run -o json | kubectl apply -f - } -function deployExists() { - local namespace="${1}" - local deploy="${2}" - helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1 - return $? -} +function download_gitlab_chart() { + echoinfo "Downloading the GitLab chart..." true -function previousDeployFailed() { - set +e - deploy="${1}" - echo "Checking for previous deployment of ${deploy}" - deployment_status=$(helm status ${deploy} >/dev/null 2>&1) - status=$? - # if `status` is `0`, deployment exists, has a status - if [ $status -eq 0 ]; then - echo "Previous deployment found, checking status" - deployment_status=$(helm status ${deploy} | grep ^STATUS | cut -d' ' -f2) - echo "Previous deployment state: $deployment_status" - if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then - status=0; - else - status=1; - fi - else - echo "Previous deployment NOT found." - fi - set -e - return $status + curl -o gitlab.tar.bz2 https://gitlab.com/charts/gitlab/-/archive/$GITLAB_HELM_CHART_REF/gitlab-$GITLAB_HELM_CHART_REF.tar.bz2 + tar -xjf gitlab.tar.bz2 + cd gitlab-$GITLAB_HELM_CHART_REF + + echoinfo "Adding the gitlab repo to Helm..." + helm repo add gitlab https://charts.gitlab.io + + echoinfo "Building the gitlab chart's dependencies..." + helm dependency build . } function deploy() { - track="${1-stable}" - name="$CI_ENVIRONMENT_SLUG" + local track="${1-stable}" + local name="$CI_ENVIRONMENT_SLUG" if [[ "$track" != "stable" ]]; then name="$name-$track" fi + echoinfo "Deploying ${name}..." true replicas="1" service_enabled="false" @@ -140,9 +250,7 @@ function deploy() { fi create_secret - - helm repo add gitlab https://charts.gitlab.io/ - helm dep update . + download_gitlab_chart HELM_CMD=$(cat << EOF helm upgrade --install \ @@ -188,92 +296,20 @@ HELM_CMD=$(cat << EOF EOF ) - echo "Deploying with:" - echo $HELM_CMD + echoinfo "Deploying with:" + echoinfo "${HELM_CMD}" eval $HELM_CMD } -function delete() { - track="${1-stable}" - name="$CI_ENVIRONMENT_SLUG" - - if [ -z "$CI_ENVIRONMENT_SLUG" ]; then - echo "No release given, aborting the delete!" - return - fi - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi - - if ! deployExists "${KUBE_NAMESPACE}" "${name}"; then - echo "The release $name doesn't exist, aborting the cleanup!" - return - fi - - echo "Deleting release '$name'..." - helm delete --purge "$name" || true -} - -function cleanup() { - if [ -z "$CI_ENVIRONMENT_SLUG" ]; then - echo "No release given, aborting the delete!" - return - fi - - echo "Cleaning up '$CI_ENVIRONMENT_SLUG'..." - kubectl -n "$KUBE_NAMESPACE" delete \ - ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa \ - -l release="$CI_ENVIRONMENT_SLUG" \ - || true -} - -function install_external_dns() { - local release_name="dns-gitlab-review-app" - local domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}') - - if ! deployExists "${KUBE_NAMESPACE}" "${release_name}" || previousDeployFailed "${release_name}" ; then - echo "Installing external-dns helm chart" - helm repo update - helm install stable/external-dns \ - -n "${release_name}" \ - --namespace "${KUBE_NAMESPACE}" \ - --set provider="aws" \ - --set aws.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \ - --set aws.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \ - --set aws.zoneType="public" \ - --set domainFilters[0]="${domain}" \ - --set txtOwnerId="${KUBE_NAMESPACE}" \ - --set rbac.create="true" \ - --set policy="sync" - fi -} - -function get_pod() { - local app_name="${1}" - local status="${2-Running}" - get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name" - echoinfo "Running '${get_pod_cmd}'" - - while true; do - local pod_name="$(eval $get_pod_cmd)" - [[ "${pod_name}" == "" ]] || break - - echoinfo "Waiting till '${app_name}' pod is ready"; - sleep 5; - done - - echoinfo "The pod name is '${pod_name}'." - echo "${pod_name}" -} - function add_license() { if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi task_runner_pod=$(get_pod "task-runner"); if [ -z "${task_runner_pod}" ]; then echo "Task runner pod not found" && return; fi + echoinfo "Installing license..." true + echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab kubectl -n "$KUBE_NAMESPACE" cp /tmp/license.gitlab ${task_runner_pod}:/tmp/license.gitlab rm /tmp/license.gitlab -- cgit v1.2.3