From 9649c2ef474701646e74bfd5134613528018a6c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 18 Apr 2019 21:18:20 +0200 Subject: Organize better Review Apps and QA jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also, play manual jobs once dependency jobs are done instead of polling for the dependent jobs to be finished. Signed-off-by: Rémy Coutable --- scripts/review_apps/review-apps.sh | 97 -------------------------- scripts/utils.sh | 136 +++++++++++++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 102 deletions(-) (limited to 'scripts') diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index b55ce1af55e..8be22dc0278 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -1,26 +1,6 @@ [[ "$TRACE" ]] && set -x export TILLER_NAMESPACE="$KUBE_NAMESPACE" -function echoerr() { - local header="${2}" - - if [ -n "${header}" ]; then - printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2; - else - printf "\033[0;31m%s\n\033[0m" "${1}" >&2; - fi -} - -function echoinfo() { - local header="${2}" - - if [ -n "${header}" ]; then - printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2; - else - printf "\033[0;33m%s\n\033[0m" "${1}" >&2; - fi -} - function deployExists() { local namespace="${1}" local deploy="${2}" @@ -328,80 +308,3 @@ function add_license() { puts "License added"; ' } - -function get_job_id() { - local job_name="${1}" - local query_string="${2:+&${2}}" - - local max_page=3 - local page=1 - - while true; do - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}" - echoinfo "GET ${url}" - - local job_id - job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last") - [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break - - let "page++" - done - - if [[ "${job_id}" == "" ]]; then - echoerr "The '${job_name}' job ID couldn't be retrieved!" - else - echoinfo "The '${job_name}' job ID is ${job_id}" - echo "${job_id}" - fi -} - -function play_job() { - local job_name="${1}" - local job_id - job_id=$(get_job_id "${job_name}" "scope=manual"); - if [ -z "${job_id}" ]; then return; fi - - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play" - echoinfo "POST ${url}" - - local job_url - job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".web_url") - echoinfo "Manual job '${job_name}' started at: ${job_url}" -} - -function wait_for_job_to_be_done() { - local job_name="${1}" - local query_string="${2}" - local job_id - job_id=$(get_job_id "${job_name}" "${query_string}") - if [ -z "${job_id}" ]; then return; fi - - echoinfo "Waiting for the '${job_name}' job to finish..." - - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}" - echoinfo "GET ${url}" - - # In case the job hasn't finished yet. Keep trying until the job times out. - local interval=30 - local elapsed_seconds=0 - while true; do - local job_status - job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".status" | sed -e s/\"//g) - [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break - - printf "." - let "elapsed_seconds+=interval" - sleep ${interval} - done - - local elapsed_minutes=$((elapsed_seconds / 60)) - echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes." - - if [[ "${job_status}" == "failed" ]]; then - echoerr "The '${job_name}' failed." - elif [[ "${job_status}" == "manual" ]]; then - echoinfo "The '${job_name}' is manual." - else - echoinfo "The '${job_name}' passed." - fi -} diff --git a/scripts/utils.sh b/scripts/utils.sh index 2d2ba115563..4a6567b8a62 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -1,4 +1,4 @@ -retry() { +function retry() { if eval "$@"; then return 0 fi @@ -13,15 +13,15 @@ retry() { return 1 } -setup_db_user_only() { +function setup_db_user_only() { if [ "$GITLAB_DATABASE" = "postgresql" ]; then - . scripts/create_postgres_user.sh + source scripts/create_postgres_user.sh else - . scripts/create_mysql_user.sh + source scripts/create_mysql_user.sh fi } -setup_db() { +function setup_db() { setup_db_user_only bundle exec rake db:drop db:create db:schema:load db:migrate @@ -30,3 +30,129 @@ setup_db() { bundle exec rake add_limits_mysql fi } + +function install_api_client_dependencies_with_apk() { + apk add --update openssl curl jq +} + +function install_api_client_dependencies_with_apt() { + apt update && apt install jq -y +} + +function install_gitlab_gem() { + gem install gitlab --no-document +} + +function echoerr() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2; + else + printf "\033[0;31m%s\n\033[0m" "${1}" >&2; + fi +} + +function echoinfo() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2; + else + printf "\033[0;33m%s\n\033[0m" "${1}" >&2; + fi +} + +function get_job_id() { + local job_name="${1}" + local query_string="${2:+&${2}}" + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + local max_page=3 + local page=1 + + while true; do + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}" + echoinfo "GET ${url}" + + local job_id + job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last") + [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break + + let "page++" + done + + if [[ "${job_id}" == "" ]]; then + echoerr "The '${job_name}' job ID couldn't be retrieved!" + else + echoinfo "The '${job_name}' job ID is ${job_id}" + echo "${job_id}" + fi +} + +function play_job() { + local job_name="${1}" + local job_id + job_id=$(get_job_id "${job_name}" "scope=manual"); + if [ -z "${job_id}" ]; then return; fi + + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play" + echoinfo "POST ${url}" + + local job_url + job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url") + echoinfo "Manual job '${job_name}' started at: ${job_url}" +} + +function wait_for_job_to_be_done() { + local job_name="${1}" + local query_string="${2}" + local job_id + job_id=$(get_job_id "${job_name}" "${query_string}") + if [ -z "${job_id}" ]; then return; fi + + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + echoinfo "Waiting for the '${job_name}' job to finish..." + + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}" + echoinfo "GET ${url}" + + # In case the job hasn't finished yet. Keep trying until the job times out. + local interval=30 + local elapsed_seconds=0 + while true; do + local job_status + job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".status" | sed -e s/\"//g) + [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break + + printf "." + let "elapsed_seconds+=interval" + sleep ${interval} + done + + local elapsed_minutes=$((elapsed_seconds / 60)) + echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes." + + if [[ "${job_status}" == "failed" ]]; then + echoerr "The '${job_name}' failed." + elif [[ "${job_status}" == "manual" ]]; then + echoinfo "The '${job_name}' is manual." + else + echoinfo "The '${job_name}' passed." + fi +} -- cgit v1.2.3