From 1d388ed855838a2d50588c131f9f23815f148344 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 30 Jan 2020 03:08:50 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- qa/qa.rb | 2 + .../fixtures/monitored_auto_devops/.gitlab-ci.yml | 337 +++++++++++++++++++++ qa/qa/page/project/operations/metrics.rb | 49 +++ qa/qa/page/project/pipeline/index.rb | 22 +- qa/qa/page/project/sub_menus/operations.rb | 9 + 5 files changed, 416 insertions(+), 3 deletions(-) create mode 100644 qa/qa/fixtures/monitored_auto_devops/.gitlab-ci.yml create mode 100644 qa/qa/page/project/operations/metrics.rb (limited to 'qa') diff --git a/qa/qa.rb b/qa/qa.rb index 08686aa8571..685cab36b05 100644 --- a/qa/qa.rb +++ b/qa/qa.rb @@ -289,6 +289,8 @@ module QA autoload :AddExisting, 'qa/page/project/operations/kubernetes/add_existing' autoload :Show, 'qa/page/project/operations/kubernetes/show' end + + autoload :Metrics, 'qa/page/project/operations/metrics' end module Wiki diff --git a/qa/qa/fixtures/monitored_auto_devops/.gitlab-ci.yml b/qa/qa/fixtures/monitored_auto_devops/.gitlab-ci.yml new file mode 100644 index 00000000000..a65ae5aa1d9 --- /dev/null +++ b/qa/qa/fixtures/monitored_auto_devops/.gitlab-ci.yml @@ -0,0 +1,337 @@ +# This is stripped down version of the .gitlab-ci.yml found +# here: https://gitlab.com/joshlambert/autodevops-deploy. +# +# It performs only the deploy stage. + +image: alpine:latest + +variables: + # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + AUTO_DEVOPS_DOMAIN: my-fake-domain.com + + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password + POSTGRES_ENABLED: 'false' + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + + KUBERNETES_VERSION: 1.11.6 + HELM_VERSION: 2.12.2 + + DOCKER_DRIVER: overlay2 + +stages: + - production + +# This job continuously deploys to production on every push to `master`. + +production: + stage: production + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + only: + refs: + - master + kubernetes: active + +# --------------------------------------------------------------------------- + +.auto_devops: &auto_devops | + # Auto DevOps variables and functions + [[ "$TRACE" ]] && set -x + auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} + export DATABASE_URL=${DATABASE_URL-$auto_database_url} + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG + export CI_APPLICATION_TAG=$CI_COMMIT_SHA + export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID} + export TILLER_NAMESPACE=$KUBE_NAMESPACE + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products + export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + + function get_replicas() { + track="${1:-stable}" + percentage="${2:-100}" + + env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) + env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) + + if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then + # for stable track get number of replicas from `PRODUCTION_REPLICAS` + eval new_replicas=\$${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + new_replicas=$REPLICAS + fi + else + # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` + eval new_replicas=\$${env_track}_${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + eval new_replicas=\${env_track}_REPLICAS + fi + fi + + replicas="${new_replicas:-1}" + replicas="$(($replicas * $percentage / 100))" + + # always return at least one replicas + if [[ $replicas -gt 0 ]]; then + echo "$replicas" + else + echo 1 + fi + } + + + # Extracts variables prefixed with K8S_SECRET_ + # and creates a Kubernetes secret. + # + # e.g. If we have the following environment variables: + # K8S_SECRET_A=value1 + # K8S_SECRET_B=multi\ word\ value + # + # Then we will create a secret with the following key-value pairs: + # data: + # A: dmFsdWUxCg== + # B: bXVsdGkgd29yZCB2YWx1ZQo= + function create_application_secret() { + track="${1-stable}" + export APPLICATION_SECRET_NAME=$(application_secret_name "$track") + + env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables + + kubectl create secret \ + -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ + --from-env-file k8s_prefixed_variables -o yaml --dry-run | + kubectl replace -n "$KUBE_NAMESPACE" --force -f - + + export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) + + rm k8s_prefixed_variables + } + + function deploy_name() { + name="$CI_ENVIRONMENT_SLUG" + track="${1-stable}" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + echo $name + } + + function application_secret_name() { + track="${1-stable}" + name=$(deploy_name "$track") + + echo "${name}-secret" + } + + + function deploy() { + track="${1-stable}" + percentage="${2:-100}" + name=$(deploy_name "$track") + + replicas="1" + service_enabled="true" + postgres_enabled="$POSTGRES_ENABLED" + + # if track is different than stable, + # re-use all attached resources + if [[ "$track" != "stable" ]]; then + service_enabled="false" + postgres_enabled="false" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then + secret_name='gitlab-registry' + else + secret_name='' + fi + + create_application_secret "$track" + + env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') + eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS + if [ -n "$env_ADDITIONAL_HOSTS" ]; then + additional_hosts="{$env_ADDITIONAL_HOSTS}" + elif [ -n "$ADDITIONAL_HOSTS" ]; then + additional_hosts="{$ADDITIONAL_HOSTS}" + fi + + if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then + echo "Deploying first release with database initialization..." + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="registry.gitlab.com/joshlambert/ruby-gke/master" \ + --set image.tag="63492726c2264a0277141d6a6573c3d22ecd7de3" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ + --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set service.additionalHosts="$additional_hosts" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.initializeCommand="$DB_INITIALIZE" \ + --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ + --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + + echo "Deploying second release..." + helm upgrade --reuse-values \ + --wait \ + --set application.initializeCommand="" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + else + echo "Deploying new release..." + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="registry.gitlab.com/joshlambert/ruby-gke/master" \ + --set image.tag="63492726c2264a0277141d6a6573c3d22ecd7de3" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ + --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set service.additionalHosts="$additional_hosts" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ + --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + + kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name" + } + + + function install_dependencies() { + apk add -U openssl curl tar gzip bash ca-certificates git + wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub + wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.23-r3/glibc-2.23-r3.apk + apk add glibc-2.23-r3.apk + rm glibc-2.23-r3.apk + + curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx + mv linux-amd64/helm /usr/bin/ + mv linux-amd64/tiller /usr/bin/ + helm version --client + tiller -version + + curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" + chmod +x /usr/bin/kubectl + kubectl version --client + } + + function download_chart() { + if [[ ! -d chart ]]; then + auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} + auto_chart_name=$(basename $auto_chart) + auto_chart_name=${auto_chart_name%.tgz} + else + auto_chart="chart" + auto_chart_name="chart" + fi + + helm init --client-only + helm repo add gitlab https://charts.gitlab.io + if [[ ! -d "$auto_chart" ]]; then + helm fetch ${auto_chart} --untar + fi + if [ "$auto_chart_name" != "chart" ]; then + mv ${auto_chart_name} chart + fi + + helm dependency update chart/ + helm dependency build chart/ + } + + function ensure_namespace() { + kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" + } + + function check_kube_domain() { + if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then + echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set" + echo "You can do it in Auto DevOps project settings or defining a secret variable at group or project level" + echo "You can also manually add it in .gitlab-ci.yml" + false + else + true + fi + } + + function initialize_tiller() { + echo "Checking Tiller..." + + export HELM_HOST="localhost:44134" + tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & + echo "Tiller is listening on ${HELM_HOST}" + + if ! helm version --debug; then + echo "Failed to init Tiller." + return 1 + fi + echo "" + } + + function create_secret() { + echo "Create secret..." + if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then + return + fi + + kubectl create secret -n "$KUBE_NAMESPACE" \ + docker-registry gitlab-registry \ + --docker-server="$CI_REGISTRY" \ + --docker-username="$CI_REGISTRY_USER" \ + --docker-password="$CI_REGISTRY_PASSWORD" \ + --docker-email="$GITLAB_USER_EMAIL" \ + -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - + } + + function persist_environment_url() { + echo $CI_ENVIRONMENT_URL > environment_url.txt + } + +before_script: + - *auto_devops diff --git a/qa/qa/page/project/operations/metrics.rb b/qa/qa/page/project/operations/metrics.rb new file mode 100644 index 00000000000..418cc925186 --- /dev/null +++ b/qa/qa/page/project/operations/metrics.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module QA + module Page + module Project + module Operations + class Metrics < Page::Base + EXPECTED_TITLE = 'Memory Usage (Total)' + LOADING_MESSAGE = 'Waiting for performance data' + + view 'app/assets/javascripts/monitoring/components/dashboard.vue' do + element :prometheus_graphs + end + + view 'app/assets/javascripts/monitoring/components/charts/time_series.vue' do + element :prometheus_graph_widgets + end + + view 'app/assets/javascripts/monitoring/components/panel_type.vue' do + element :prometheus_widgets_dropdown + element :alert_widget_menu_item + end + + def wait_for_metrics + wait_for_data + return if has_metrics? + + wait_until(max_duration: 180) do + wait_for_data + has_metrics? + end + end + + def wait_for_data + wait_until(reload: false) { !has_text?(LOADING_MESSAGE) } if has_text?(LOADING_MESSAGE) + end + + def has_metrics? + within_element :prometheus_graphs do + has_text?(EXPECTED_TITLE) + end + end + end + end + end + end +end + +QA::Page::Project::Operations::Metrics.prepend_if_ee('QA::EE::Page::Project::Operations::Metrics') diff --git a/qa/qa/page/project/pipeline/index.rb b/qa/qa/page/project/pipeline/index.rb index 684ad4a59d5..f018e4bf018 100644 --- a/qa/qa/page/project/pipeline/index.rb +++ b/qa/qa/page/project/pipeline/index.rb @@ -9,6 +9,7 @@ module QA::Page view 'app/assets/javascripts/pipelines/components/pipelines_table_row.vue' do element :pipeline_commit_status + element :pipeline_retry_button end def click_on_latest_pipeline @@ -18,10 +19,25 @@ module QA::Page end def wait_for_latest_pipeline_success + wait_for_latest_pipeline_status { has_text?('passed') } + end + + def wait_for_latest_pipeline_completion + wait_for_latest_pipeline_status { has_text?('passed') || has_text?('failed') } + end + + def wait_for_latest_pipeline_status wait_until(reload: false, max_duration: 300) do - within_element_by_index(:pipeline_commit_status, 0) do - has_text?('passed') - end + within_element_by_index(:pipeline_commit_status, 0) { yield } + end + end + + def wait_for_latest_pipeline_success_or_retry + wait_for_latest_pipeline_completion + + if has_text?('failed') + click_element :pipeline_retry_button + wait_for_latest_pipeline_success end end end diff --git a/qa/qa/page/project/sub_menus/operations.rb b/qa/qa/page/project/sub_menus/operations.rb index d266cb21417..bcbc1dc16d3 100644 --- a/qa/qa/page/project/sub_menus/operations.rb +++ b/qa/qa/page/project/sub_menus/operations.rb @@ -12,6 +12,7 @@ module QA view 'app/views/layouts/nav/sidebar/_project.html.haml' do element :link_operations element :operations_environments_link + element :operations_metrics_link end end end @@ -24,6 +25,14 @@ module QA end end + def go_to_operations_metrics + hover_operations do + within_submenu do + click_element(:operations_metrics_link) + end + end + end + def go_to_operations_kubernetes hover_operations do within_submenu do -- cgit v1.2.3