diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-20 01:11:55 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-20 01:11:55 +0300 |
commit | 5a8431feceba47fd8e1804d9aa1b1730606b71d5 (patch) | |
tree | e5df8e0ceee60f4af8093f5c4c2f934b8abced05 /scripts | |
parent | 4d477238500c347c6553d335d920bedfc5a46869 (diff) |
Add latest changes from gitlab-org/gitlab@12-5-stable-ee
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/lint-doc.sh | 3 | ||||
-rwxr-xr-x | scripts/notify-slack | 14 | ||||
-rwxr-xr-x | scripts/review_apps/automated_cleanup.rb | 36 | ||||
-rw-r--r-- | scripts/review_apps/base-config.yaml | 70 | ||||
-rwxr-xr-x | scripts/review_apps/review-apps.sh | 98 | ||||
-rwxr-xr-x | scripts/static-analysis | 38 | ||||
-rw-r--r-- | scripts/sync-stable-branch.sh | 32 | ||||
-rwxr-xr-x | scripts/trigger-build | 35 |
8 files changed, 206 insertions, 120 deletions
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh index d097c2aee91..7f5d6130fe4 100755 --- a/scripts/lint-doc.sh +++ b/scripts/lint-doc.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash cd "$(dirname "$0")/.." +echo "=> Linting documents at path $(pwd) as $(whoami)..." # Use long options (e.g. --header instead of -H) for curl examples in documentation. echo '=> Checking for cURL short options...' @@ -25,7 +26,7 @@ fi # Make sure no files in doc/ are executable EXEC_PERM_COUNT=$(find doc/ -type f -perm 755 | wc -l) -echo '=> Checking for executable permissions...' +echo "=> Checking $(pwd)/doc for executable permissions..." if [ "${EXEC_PERM_COUNT}" -ne 0 ] then echo '✖ ERROR: Executable permissions should not be used in documentation! Use `chmod 644` to the files in question:' >&2 diff --git a/scripts/notify-slack b/scripts/notify-slack deleted file mode 100755 index 5907fd8b986..00000000000 --- a/scripts/notify-slack +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Sends Slack notification MSG to CI_SLACK_WEBHOOK_URL (which needs to be set). -# ICON_EMOJI needs to be set to an icon emoji name (without the `:` around it). - -CHANNEL=$1 -MSG=$2 -ICON_EMOJI=$3 - -if [ -z "$CHANNEL" ] || [ -z "$CI_SLACK_WEBHOOK_URL" ] || [ -z "$MSG" ] || [ -z "$ICON_EMOJI" ]; then - echo "Missing argument(s) - Use: $0 channel message icon_emoji" - echo "and set CI_SLACK_WEBHOOK_URL environment variable." -else - curl -X POST --data-urlencode 'payload={"channel": "#'"$CHANNEL"'", "username": "GitLab QA Bot", "text": "'"$MSG"'", "icon_emoji": "'":$ICON_EMOJI:"'"}' "$CI_SLACK_WEBHOOK_URL" -fi diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index 9edc1a2b857..c7ab8829088 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -58,10 +58,16 @@ class AutomatedCleanup checked_environments = [] delete_threshold = threshold_time(days: days_for_delete) stop_threshold = threshold_time(days: days_for_stop) + deployments_look_back_threshold = threshold_time(days: days_for_delete * 5) + + releases_to_delete = [] + + gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment| + break if Time.parse(deployment.created_at) < deployments_look_back_threshold - gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE).auto_paginate do |deployment| environment = deployment.environment + next unless environment next unless environment.name.start_with?('review/') next if checked_environments.include?(environment.slug) @@ -71,7 +77,7 @@ class AutomatedCleanup if deployed_at < delete_threshold delete_environment(environment, deployment) release = Quality::HelmClient::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) - delete_helm_release(release) + releases_to_delete << release elsif deployed_at < stop_threshold stop_environment(environment, deployment) else @@ -80,6 +86,8 @@ class AutomatedCleanup checked_environments << environment.slug end + + delete_helm_releases(releases_to_delete) end def perform_helm_releases_cleanup!(days:) @@ -87,13 +95,20 @@ class AutomatedCleanup threshold_day = threshold_time(days: days) + releases_to_delete = [] + helm_releases.each do |release| + # Prevents deleting `dns-gitlab-review-app` releases or other unrelated releases + next unless release.name.start_with?('review-') + if release.status == 'FAILED' || release.last_update < threshold_day - delete_helm_release(release) + releases_to_delete << release else print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving') end end + + delete_helm_releases(releases_to_delete) end private @@ -114,10 +129,17 @@ class AutomatedCleanup helm.releases(args: args) end - def delete_helm_release(release) - print_release_state(subject: 'Release', release_name: release.name, release_status: release.status, release_date: release.last_update, action: 'cleaning') - helm.delete(release_name: release.name) - kubernetes.cleanup(release_name: release.name) + def delete_helm_releases(releases) + return if releases.empty? + + releases.each do |release| + print_release_state(subject: 'Release', release_name: release.name, release_status: release.status, release_date: release.last_update, action: 'cleaning') + end + + releases_names = releases.map(&:name) + helm.delete(release_name: releases_names) + kubernetes.cleanup(release_name: releases_names, wait: false) + rescue Quality::HelmClient::CommandFailedError => ex raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS) diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml index 573a5ccde11..7aaa7544c19 100644 --- a/scripts/review_apps/base-config.yaml +++ b/scripts/review_apps/base-config.yaml @@ -14,11 +14,11 @@ gitlab: gitaly: resources: requests: - cpu: 300m - memory: 200M + cpu: 1200m + memory: 240M limits: - cpu: 600m - memory: 420M + cpu: 1800m + memory: 360M persistence: size: 10G gitlab-exporter: @@ -35,22 +35,25 @@ gitlab: gitlab-shell: resources: requests: - cpu: 125m - memory: 20M + cpu: 500m + memory: 25M limits: - cpu: 250m - memory: 40M + cpu: 750m + memory: 37.5M maxReplicas: 3 hpa: targetAverageValue: 130m + deployment: + livenessProbe: + timeoutSeconds: 5 sidekiq: resources: requests: - cpu: 300m - memory: 800M + cpu: 650m + memory: 880M limits: - cpu: 400m - memory: 1.6G + cpu: 975m + memory: 1320M task-runner: resources: requests: @@ -62,11 +65,11 @@ gitlab: unicorn: resources: requests: - cpu: 400m - memory: 1.4G + cpu: 500m + memory: 1540M limits: - cpu: 800m - memory: 2.8G + cpu: 750m + memory: 2310M deployment: readinessProbe: initialDelaySeconds: 5 # Default is 0 @@ -75,11 +78,11 @@ gitlab: workhorse: resources: requests: - cpu: 175m - memory: 100M + cpu: 250m + memory: 50M limits: - cpu: 350m - memory: 200M + cpu: 375m + memory: 75M readinessProbe: initialDelaySeconds: 5 # Default is 0 periodSeconds: 15 # Default is 10 @@ -87,11 +90,11 @@ gitlab: gitlab-runner: resources: requests: - cpu: 355m - memory: 300M + cpu: 450m + memory: 100M limits: - cpu: 710m - memory: 600M + cpu: 675m + memory: 150M minio: resources: requests: @@ -108,10 +111,10 @@ nginx-ingress: resources: requests: cpu: 100m - memory: 250M + memory: 450M limits: cpu: 200m - memory: 500M + memory: 675M minAvailable: 1 service: enableHttp: false @@ -133,10 +136,11 @@ postgresql: enabled: false resources: requests: - cpu: 250m - memory: 256M + cpu: 300m + memory: 250M limits: - cpu: 500m + cpu: 450m + memory: 375M prometheus: install: false redis: @@ -157,8 +161,8 @@ registry: minReplicas: 1 resources: requests: - cpu: 50m - memory: 32M - limits: cpu: 100m - memory: 64M + memory: 30M + limits: + cpu: 200m + memory: 45M diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 51768d07860..ed872783856 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -1,5 +1,4 @@ [[ "$TRACE" ]] && set -x -export TILLER_NAMESPACE="$KUBE_NAMESPACE" function deploy_exists() { local namespace="${1}" @@ -14,16 +13,18 @@ function deploy_exists() { } function previous_deploy_failed() { - local deploy="${1}" + local namespace="${1}" + local deploy="${2}" + echoinfo "Checking for previous deployment of ${deploy}" true - helm status "${deploy}" >/dev/null 2>&1 + helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1 local status=$? # if `status` is `0`, deployment exists, has a status if [ $status -eq 0 ]; then echoinfo "Previous deployment found, checking status..." - deployment_status=$(helm status "${deploy}" | grep ^STATUS | cut -d' ' -f2) + deployment_status=$(helm status --tiller-namespace "${namespace}" "${deploy}" | grep ^STATUS | cut -d' ' -f2) echoinfo "Previous deployment state: ${deployment_status}" if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then status=0; @@ -37,16 +38,17 @@ function previous_deploy_failed() { } function delete_release() { - if [ -z "$CI_ENVIRONMENT_SLUG" ]; then + local namespace="${KUBE_NAMESPACE}" + local deploy="${CI_ENVIRONMENT_SLUG}" + + if [ -z "$deploy" ]; then echoerr "No release given, aborting the delete!" return fi - local name="$CI_ENVIRONMENT_SLUG" - - echoinfo "Deleting release '$name'..." true + echoinfo "Deleting release '$deploy'..." true - helm delete --purge "$name" + helm delete --purge --tiller-namespace "${namespace}" "${deploy}" } function delete_failed_release() { @@ -59,7 +61,7 @@ function delete_failed_release() { echoinfo "No Review App with ${CI_ENVIRONMENT_SLUG} is currently deployed." else # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade` - if previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then + if previous_deploy_failed "${KUBE_NAMESPACE}" "$CI_ENVIRONMENT_SLUG" ; then echoinfo "Review App deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG" delete_release else @@ -117,6 +119,7 @@ function ensure_namespace() { } function install_tiller() { + local TILLER_NAMESPACE="$KUBE_NAMESPACE" echoinfo "Checking deployment/tiller-deploy status in the ${TILLER_NAMESPACE} namespace..." true echoinfo "Initiating the Helm client..." @@ -131,11 +134,12 @@ function install_tiller() { --override "spec.template.spec.tolerations[0].key"="dedicated" \ --override "spec.template.spec.tolerations[0].operator"="Equal" \ --override "spec.template.spec.tolerations[0].value"="helm" \ - --override "spec.template.spec.tolerations[0].effect"="NoSchedule" + --override "spec.template.spec.tolerations[0].effect"="NoSchedule" \ + --tiller-namespace "${TILLER_NAMESPACE}" kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy" - if ! helm version --debug; then + if ! helm version --debug --tiller-namespace "${TILLER_NAMESPACE}"; then echo "Failed to init Tiller." return 1 fi @@ -147,7 +151,7 @@ function install_external_dns() { domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}') echoinfo "Installing external DNS for domain ${domain}..." true - if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${release_name}" ; then + if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${KUBE_NAMESPACE}" "${release_name}" ; then echoinfo "Installing external-dns Helm chart" helm repo update # Default requested: CPU => 0, memory => 0 @@ -179,6 +183,17 @@ function create_application_secret() { "${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password" \ --from-literal="password=${REVIEW_APPS_ROOT_PASSWORD}" \ --dry-run -o json | kubectl apply -f - + + if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi + + echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-license secret in the ${KUBE_NAMESPACE} namespace..." true + + echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab + + kubectl create secret generic -n "$KUBE_NAMESPACE" \ + "${CI_ENVIRONMENT_SLUG}-gitlab-license" \ + --from-file=license=/tmp/license.gitlab \ + --dry-run -o json | kubectl apply -f - } function download_chart() { @@ -195,9 +210,19 @@ function download_chart() { helm dependency build . } +function base_config_changed() { + if [ -z "${CI_MERGE_REQUEST_IID}" ]; then return; fi + + curl "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/changes" | jq '.changes | any(.old_path == "scripts/review_apps/base-config.yaml")' +} + function deploy() { local name="$CI_ENVIRONMENT_SLUG" local edition="${GITLAB_EDITION-ce}" + local base_config_file_ref="master" + if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="$CI_COMMIT_SHA"; fi + local base_config_file="https://gitlab.com/gitlab-org/gitlab/raw/${base_config_file_ref}/scripts/review_apps/base-config.yaml" + echoinfo "Deploying ${name}..." true IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror" @@ -239,12 +264,20 @@ HELM_CMD=$(cat << EOF EOF ) +if [ -n "${REVIEW_APPS_EE_LICENSE}" ]; then +HELM_CMD=$(cat << EOF + ${HELM_CMD} \ + --set global.gitlab.license.secret="${CI_ENVIRONMENT_SLUG}-gitlab-license" +EOF +) +fi + HELM_CMD=$(cat << EOF - $HELM_CMD \ + ${HELM_CMD} \ --namespace="$KUBE_NAMESPACE" \ - --version="$CI_PIPELINE_ID-$CI_JOB_ID" \ - -f "../scripts/review_apps/base-config.yaml" \ - "$name" . + --version="${CI_PIPELINE_ID}-${CI_JOB_ID}" \ + -f "${base_config_file}" \ + "${name}" . EOF ) @@ -263,34 +296,3 @@ function display_deployment_debug() { echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}" kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1 } - -function add_license() { - if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi - - task_runner_pod=$(get_pod "task-runner"); - if [ -z "${task_runner_pod}" ]; then echo "Task runner pod not found" && return; fi - - echoinfo "Installing license..." true - - echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab - kubectl -n "$KUBE_NAMESPACE" cp /tmp/license.gitlab "${task_runner_pod}":/tmp/license.gitlab - rm /tmp/license.gitlab - - kubectl -n "$KUBE_NAMESPACE" exec -it "${task_runner_pod}" -- /srv/gitlab/bin/rails runner -e production \ - ' - content = File.read("/tmp/license.gitlab").strip; - FileUtils.rm_f("/tmp/license.gitlab"); - - unless License.where(data:content).empty? - puts "License already exists"; - Kernel.exit 0; - end - - unless License.new(data: content).save - puts "Could not add license"; - Kernel.exit 0; - end - - puts "License added"; - ' -} diff --git a/scripts/static-analysis b/scripts/static-analysis index 602cd847a71..b7f7100c365 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -26,17 +26,35 @@ def emit_errors(static_analysis) end end -tasks = [ - %w[bin/rake lint:all], - %w[bundle exec license_finder], - %w[yarn run eslint], - %w[yarn run stylelint], - %w[yarn run prettier-all], - %w[bundle exec rubocop --parallel], - %w[scripts/lint-conflicts.sh], - %w[scripts/lint-rugged] -] +def jobs_to_run(node_index, node_total) + all_tasks = [ + %w[bin/rake lint:all], + %w[bundle exec license_finder], + %w[yarn run eslint], + %w[yarn run stylelint], + %w[yarn run prettier-all], + %w[bundle exec rubocop --parallel], + %w[scripts/lint-conflicts.sh], + %w[scripts/lint-rugged] + ] + case node_total + when 1 + all_tasks + when 2 + rake_lint_all, *rest_jobs = all_tasks + case node_index + when 1 + [rake_lint_all] + else + rest_jobs + end + else + raise "Parallelization > 2 (currently set to #{node_total}) isn't supported yet!" + end +end + +tasks = jobs_to_run((ENV['CI_NODE_INDEX'] || 1).to_i, (ENV['CI_NODE_TOTAL'] || 1).to_i) static_analysis = Gitlab::Popen::Runner.new static_analysis.run(tasks) do |cmd, &run| diff --git a/scripts/sync-stable-branch.sh b/scripts/sync-stable-branch.sh new file mode 100644 index 00000000000..fc62453d743 --- /dev/null +++ b/scripts/sync-stable-branch.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# This script triggers a merge train job to sync an EE stable branch to its +# corresponding CE stable branch. + +set -e + +if [[ "$MERGE_TRAIN_TRIGGER_TOKEN" == '' ]] +then + echo 'The variable MERGE_TRAIN_TRIGGER_TOKEN must be set to a non-empy value' + exit 1 +fi + +if [[ "$MERGE_TRAIN_TRIGGER_URL" == '' ]] +then + echo 'The variable MERGE_TRAIN_TRIGGER_URL must be set to a non-empy value' + exit 1 +fi + +if [[ "$CI_COMMIT_REF_NAME" == '' ]] +then + echo 'The variable CI_COMMIT_REF_NAME must be set to a non-empy value' + exit 1 +fi + +curl -X POST \ + -F token="$MERGE_TRAIN_TRIGGER_TOKEN" \ + -F ref=master \ + -F "variables[MERGE_FOSS]=1" \ + -F "variables[SOURCE_BRANCH]=$CI_COMMIT_REF_NAME" \ + -F "variables[TARGET_BRANCH]=${CI_COMMIT_REF_NAME/-ee/}" \ + "$MERGE_TRAIN_TRIGGER_URL" diff --git a/scripts/trigger-build b/scripts/trigger-build index badbb562021..74c1df258c0 100755 --- a/scripts/trigger-build +++ b/scripts/trigger-build @@ -17,7 +17,7 @@ module Trigger end class Base - def invoke!(post_comment: false) + def invoke!(post_comment: false, downstream_job_name: nil) pipeline = Gitlab.run_trigger( downstream_project_path, trigger_token, @@ -28,7 +28,18 @@ module Trigger puts "Waiting for downstream pipeline status" Trigger::CommitComment.post!(pipeline, access_token) if post_comment - Trigger::Pipeline.new(downstream_project_path, pipeline.id, access_token) + downstream_job = + if downstream_job_name + Gitlab.pipeline_jobs(downstream_project_path, pipeline.id).auto_paginate.find do |potential_job| + potential_job.name == downstream_job_name + end + end + + if downstream_job + Trigger::Job.new(downstream_project_path, downstream_job.id, access_token) + else + Trigger::Pipeline.new(downstream_project_path, pipeline.id, access_token) + end end private @@ -187,6 +198,14 @@ module Trigger attr_reader :project, :id, :api_token + def self.unscoped_class_name + name.split('::').last + end + + def self.gitlab_api_method_name + unscoped_class_name.downcase + end + def initialize(project, id, api_token) @project = project @id = id @@ -199,17 +218,17 @@ module Trigger def wait! loop do - raise "Pipeline timed out after waiting for #{duration} minutes!" if timeout? + raise "#{self.class.unscoped_class_name} timed out after waiting for #{duration} minutes!" if timeout? case status when :created, :pending, :running print "." sleep INTERVAL when :success - puts "Pipeline succeeded in #{duration} minutes!" + puts "#{self.class.unscoped_class_name} succeeded in #{duration} minutes!" break else - raise "Pipeline did not succeed!" + raise "#{self.class.unscoped_class_name} did not succeed!" end STDOUT.flush @@ -225,7 +244,7 @@ module Trigger end def status - Gitlab.pipeline(project, id).status.to_sym + Gitlab.public_send(self.class.gitlab_api_method_name, project, id).status.to_sym # rubocop:disable GitlabSecurity/PublicSend rescue Gitlab::Error::Error => error puts "Ignoring the following error: #{error}" # Ignore GitLab API hiccups. If GitLab is really down, we'll hit the job @@ -233,11 +252,13 @@ module Trigger :running end end + + Job = Class.new(Pipeline) end case ARGV[0] when 'omnibus' - Trigger::Omnibus.new.invoke!(post_comment: true).wait! + Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait! when 'cng' Trigger::CNG.new.invoke!.wait! else |