Add latest changes from gitlab-org/gitlab@12-5-stable-ee

8 files changed, 206 insertions, 120 deletions

diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index d097c2aee91..7f5d6130fe4 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 cd "$(dirname "$0")/.."
+echo "=> Linting documents at path $(pwd) as $(whoami)..."
 
 # Use long options (e.g. --header instead of -H) for curl examples in documentation.
 echo '=> Checking for cURL short options...'
@@ -25,7 +26,7 @@ fi
 
 # Make sure no files in doc/ are executable
 EXEC_PERM_COUNT=$(find doc/ -type f -perm 755 | wc -l)
-echo '=> Checking for executable permissions...'
+echo "=> Checking $(pwd)/doc for executable permissions..."
 if [ "${EXEC_PERM_COUNT}" -ne 0 ]
 then
   echo '✖ ERROR: Executable permissions should not be used in documentation! Use `chmod 644` to the files in question:' >&2
diff --git a/scripts/notify-slack b/scripts/notify-slack
deleted file mode 100755
index 5907fd8b986..00000000000
--- a/scripts/notify-slack
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-# Sends Slack notification MSG to CI_SLACK_WEBHOOK_URL (which needs to be set).
-# ICON_EMOJI needs to be set to an icon emoji name (without the `:` around it).
-
-CHANNEL=$1
-MSG=$2
-ICON_EMOJI=$3
-
-if [ -z "$CHANNEL" ] || [ -z "$CI_SLACK_WEBHOOK_URL" ] || [ -z "$MSG" ] || [ -z "$ICON_EMOJI" ]; then
-    echo "Missing argument(s) - Use: $0 channel message icon_emoji"
-    echo "and set CI_SLACK_WEBHOOK_URL environment variable."
-else
-    curl -X POST --data-urlencode 'payload={"channel": "#'"$CHANNEL"'", "username": "GitLab QA Bot", "text": "'"$MSG"'", "icon_emoji": "'":$ICON_EMOJI:"'"}' "$CI_SLACK_WEBHOOK_URL"
-fi
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index 9edc1a2b857..c7ab8829088 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -58,10 +58,16 @@ class AutomatedCleanup
     checked_environments = []
     delete_threshold = threshold_time(days: days_for_delete)
     stop_threshold = threshold_time(days: days_for_stop)
+    deployments_look_back_threshold = threshold_time(days: days_for_delete * 5)
+
+    releases_to_delete = []
+
+    gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
+      break if Time.parse(deployment.created_at) < deployments_look_back_threshold
 
-    gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE).auto_paginate do |deployment|
       environment = deployment.environment
 
+      next unless environment
       next unless environment.name.start_with?('review/')
       next if checked_environments.include?(environment.slug)
 
@@ -71,7 +77,7 @@ class AutomatedCleanup
       if deployed_at < delete_threshold
         delete_environment(environment, deployment)
         release = Quality::HelmClient::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace)
-        delete_helm_release(release)
+        releases_to_delete << release
       elsif deployed_at < stop_threshold
         stop_environment(environment, deployment)
       else
@@ -80,6 +86,8 @@ class AutomatedCleanup
 
       checked_environments << environment.slug
     end
+
+    delete_helm_releases(releases_to_delete)
   end
 
   def perform_helm_releases_cleanup!(days:)
@@ -87,13 +95,20 @@ class AutomatedCleanup
 
     threshold_day = threshold_time(days: days)
 
+    releases_to_delete = []
+
     helm_releases.each do |release|
+      # Prevents deleting `dns-gitlab-review-app` releases or other unrelated releases
+      next unless release.name.start_with?('review-')
+
       if release.status == 'FAILED' || release.last_update < threshold_day
-        delete_helm_release(release)
+        releases_to_delete << release
       else
         print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving')
       end
     end
+
+    delete_helm_releases(releases_to_delete)
   end
 
   private
@@ -114,10 +129,17 @@ class AutomatedCleanup
     helm.releases(args: args)
   end
 
-  def delete_helm_release(release)
-    print_release_state(subject: 'Release', release_name: release.name, release_status: release.status, release_date: release.last_update, action: 'cleaning')
-    helm.delete(release_name: release.name)
-    kubernetes.cleanup(release_name: release.name)
+  def delete_helm_releases(releases)
+    return if releases.empty?
+
+    releases.each do |release|
+      print_release_state(subject: 'Release', release_name: release.name, release_status: release.status, release_date: release.last_update, action: 'cleaning')
+    end
+
+    releases_names = releases.map(&:name)
+    helm.delete(release_name: releases_names)
+    kubernetes.cleanup(release_name: releases_names, wait: false)
+
   rescue Quality::HelmClient::CommandFailedError => ex
     raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS)
 
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index 573a5ccde11..7aaa7544c19 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -14,11 +14,11 @@ gitlab:
   gitaly:
     resources:
       requests:
-        cpu: 300m
-        memory: 200M
+        cpu: 1200m
+        memory: 240M
       limits:
-        cpu: 600m
-        memory: 420M
+        cpu: 1800m
+        memory: 360M
     persistence:
       size: 10G
   gitlab-exporter:
@@ -35,22 +35,25 @@ gitlab:
   gitlab-shell:
     resources:
       requests:
-        cpu: 125m
-        memory: 20M
+        cpu: 500m
+        memory: 25M
       limits:
-        cpu: 250m
-        memory: 40M
+        cpu: 750m
+        memory: 37.5M
     maxReplicas: 3
     hpa:
       targetAverageValue: 130m
+    deployment:
+      livenessProbe:
+        timeoutSeconds: 5
   sidekiq:
     resources:
       requests:
-        cpu: 300m
-        memory: 800M
+        cpu: 650m
+        memory: 880M
       limits:
-        cpu: 400m
-        memory: 1.6G
+        cpu: 975m
+        memory: 1320M
   task-runner:
     resources:
       requests:
@@ -62,11 +65,11 @@ gitlab:
   unicorn:
     resources:
       requests:
-        cpu: 400m
-        memory: 1.4G
+        cpu: 500m
+        memory: 1540M
       limits:
-        cpu: 800m
-        memory: 2.8G
+        cpu: 750m
+        memory: 2310M
     deployment:
       readinessProbe:
         initialDelaySeconds: 5  # Default is 0
@@ -75,11 +78,11 @@ gitlab:
     workhorse:
       resources:
         requests:
-          cpu: 175m
-          memory: 100M
+          cpu: 250m
+          memory: 50M
         limits:
-          cpu: 350m
-          memory: 200M
+          cpu: 375m
+          memory: 75M
       readinessProbe:
         initialDelaySeconds: 5  # Default is 0
         periodSeconds: 15  # Default is 10
@@ -87,11 +90,11 @@ gitlab:
 gitlab-runner:
   resources:
     requests:
-      cpu: 355m
-      memory: 300M
+      cpu: 450m
+      memory: 100M
     limits:
-      cpu: 710m
-      memory: 600M
+      cpu: 675m
+      memory: 150M
 minio:
   resources:
     requests:
@@ -108,10 +111,10 @@ nginx-ingress:
     resources:
       requests:
         cpu: 100m
-        memory: 250M
+        memory: 450M
       limits:
         cpu: 200m
-        memory: 500M
+        memory: 675M
     minAvailable: 1
     service:
       enableHttp: false
@@ -133,10 +136,11 @@ postgresql:
     enabled: false
   resources:
     requests:
-      cpu: 250m
-      memory: 256M
+      cpu: 300m
+      memory: 250M
     limits:
-      cpu: 500m
+      cpu: 450m
+      memory: 375M
 prometheus:
   install: false
 redis:
@@ -157,8 +161,8 @@ registry:
     minReplicas: 1
   resources:
     requests:
-      cpu: 50m
-      memory: 32M
-    limits:
       cpu: 100m
-      memory: 64M
+      memory: 30M
+    limits:
+      cpu: 200m
+      memory: 45M
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 51768d07860..ed872783856 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -1,5 +1,4 @@
 [[ "$TRACE" ]] && set -x
-export TILLER_NAMESPACE="$KUBE_NAMESPACE"
 
 function deploy_exists() {
   local namespace="${1}"
@@ -14,16 +13,18 @@ function deploy_exists() {
 }
 
 function previous_deploy_failed() {
-  local deploy="${1}"
+  local namespace="${1}"
+  local deploy="${2}"
+
   echoinfo "Checking for previous deployment of ${deploy}" true
 
-  helm status "${deploy}" >/dev/null 2>&1
+  helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1
   local status=$?
 
   # if `status` is `0`, deployment exists, has a status
   if [ $status -eq 0 ]; then
     echoinfo "Previous deployment found, checking status..."
-    deployment_status=$(helm status "${deploy}" | grep ^STATUS | cut -d' ' -f2)
+    deployment_status=$(helm status --tiller-namespace "${namespace}" "${deploy}" | grep ^STATUS | cut -d' ' -f2)
     echoinfo "Previous deployment state: ${deployment_status}"
     if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then
       status=0;
@@ -37,16 +38,17 @@ function previous_deploy_failed() {
 }
 
 function delete_release() {
-  if [ -z "$CI_ENVIRONMENT_SLUG" ]; then
+  local namespace="${KUBE_NAMESPACE}"
+  local deploy="${CI_ENVIRONMENT_SLUG}"
+
+  if [ -z "$deploy" ]; then
     echoerr "No release given, aborting the delete!"
     return
   fi
 
-  local name="$CI_ENVIRONMENT_SLUG"
-
-  echoinfo "Deleting release '$name'..." true
+  echoinfo "Deleting release '$deploy'..." true
 
-  helm delete --purge "$name"
+  helm delete --purge --tiller-namespace "${namespace}" "${deploy}"
 }
 
 function delete_failed_release() {
@@ -59,7 +61,7 @@ function delete_failed_release() {
     echoinfo "No Review App with ${CI_ENVIRONMENT_SLUG} is currently deployed."
   else
     # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
-    if previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then
+    if previous_deploy_failed "${KUBE_NAMESPACE}" "$CI_ENVIRONMENT_SLUG" ; then
       echoinfo "Review App deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
       delete_release
     else
@@ -117,6 +119,7 @@ function ensure_namespace() {
 }
 
 function install_tiller() {
+  local TILLER_NAMESPACE="$KUBE_NAMESPACE"
   echoinfo "Checking deployment/tiller-deploy status in the ${TILLER_NAMESPACE} namespace..." true
 
   echoinfo "Initiating the Helm client..."
@@ -131,11 +134,12 @@ function install_tiller() {
     --override "spec.template.spec.tolerations[0].key"="dedicated" \
     --override "spec.template.spec.tolerations[0].operator"="Equal" \
     --override "spec.template.spec.tolerations[0].value"="helm" \
-    --override "spec.template.spec.tolerations[0].effect"="NoSchedule"
+    --override "spec.template.spec.tolerations[0].effect"="NoSchedule" \
+    --tiller-namespace "${TILLER_NAMESPACE}"
 
   kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
 
-  if ! helm version --debug; then
+  if ! helm version --debug --tiller-namespace "${TILLER_NAMESPACE}"; then
     echo "Failed to init Tiller."
     return 1
   fi
@@ -147,7 +151,7 @@ function install_external_dns() {
   domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}')
   echoinfo "Installing external DNS for domain ${domain}..." true
 
-  if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${release_name}" ; then
+  if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${KUBE_NAMESPACE}" "${release_name}" ; then
     echoinfo "Installing external-dns Helm chart"
     helm repo update
     # Default requested: CPU => 0, memory => 0
@@ -179,6 +183,17 @@ function create_application_secret() {
     "${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password" \
     --from-literal="password=${REVIEW_APPS_ROOT_PASSWORD}" \
     --dry-run -o json | kubectl apply -f -
+
+  if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
+
+  echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-license secret in the ${KUBE_NAMESPACE} namespace..." true
+
+  echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab
+
+  kubectl create secret generic -n "$KUBE_NAMESPACE" \
+    "${CI_ENVIRONMENT_SLUG}-gitlab-license" \
+    --from-file=license=/tmp/license.gitlab \
+    --dry-run -o json | kubectl apply -f -
 }
 
 function download_chart() {
@@ -195,9 +210,19 @@ function download_chart() {
   helm dependency build .
 }
 
+function base_config_changed() {
+  if [ -z "${CI_MERGE_REQUEST_IID}" ]; then return; fi
+
+  curl "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/changes" | jq '.changes | any(.old_path == "scripts/review_apps/base-config.yaml")'
+}
+
 function deploy() {
   local name="$CI_ENVIRONMENT_SLUG"
   local edition="${GITLAB_EDITION-ce}"
+  local base_config_file_ref="master"
+  if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="$CI_COMMIT_SHA"; fi
+  local base_config_file="https://gitlab.com/gitlab-org/gitlab/raw/${base_config_file_ref}/scripts/review_apps/base-config.yaml"
+
   echoinfo "Deploying ${name}..." true
 
   IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
@@ -239,12 +264,20 @@ HELM_CMD=$(cat << EOF
 EOF
 )
 
+if [ -n "${REVIEW_APPS_EE_LICENSE}" ]; then
+HELM_CMD=$(cat << EOF
+  ${HELM_CMD} \
+  --set global.gitlab.license.secret="${CI_ENVIRONMENT_SLUG}-gitlab-license"
+EOF
+)
+fi
+
 HELM_CMD=$(cat << EOF
-  $HELM_CMD \
+  ${HELM_CMD} \
   --namespace="$KUBE_NAMESPACE" \
-  --version="$CI_PIPELINE_ID-$CI_JOB_ID" \
-  -f "../scripts/review_apps/base-config.yaml" \
-  "$name" .
+  --version="${CI_PIPELINE_ID}-${CI_JOB_ID}" \
+  -f "${base_config_file}" \
+  "${name}" .
 EOF
 )
 
@@ -263,34 +296,3 @@ function display_deployment_debug() {
   echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}"
   kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1
 }
-
-function add_license() {
-  if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
-
-  task_runner_pod=$(get_pod "task-runner");
-  if [ -z "${task_runner_pod}" ]; then echo "Task runner pod not found" && return; fi
-
-  echoinfo "Installing license..." true
-
-  echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab
-  kubectl -n "$KUBE_NAMESPACE" cp /tmp/license.gitlab "${task_runner_pod}":/tmp/license.gitlab
-  rm /tmp/license.gitlab
-
-  kubectl -n "$KUBE_NAMESPACE" exec -it "${task_runner_pod}" -- /srv/gitlab/bin/rails runner -e production \
-    '
-    content = File.read("/tmp/license.gitlab").strip;
-    FileUtils.rm_f("/tmp/license.gitlab");
-
-    unless License.where(data:content).empty?
-      puts "License already exists";
-      Kernel.exit 0;
-    end
-
-    unless License.new(data: content).save
-      puts "Could not add license";
-      Kernel.exit 0;
-    end
-
-    puts "License added";
-    '
-}
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 602cd847a71..b7f7100c365 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -26,17 +26,35 @@ def emit_errors(static_analysis)
   end
 end
 
-tasks = [
-  %w[bin/rake lint:all],
-  %w[bundle exec license_finder],
-  %w[yarn run eslint],
-  %w[yarn run stylelint],
-  %w[yarn run prettier-all],
-  %w[bundle exec rubocop --parallel],
-  %w[scripts/lint-conflicts.sh],
-  %w[scripts/lint-rugged]
-]
+def jobs_to_run(node_index, node_total)
+  all_tasks = [
+    %w[bin/rake lint:all],
+    %w[bundle exec license_finder],
+    %w[yarn run eslint],
+    %w[yarn run stylelint],
+    %w[yarn run prettier-all],
+    %w[bundle exec rubocop --parallel],
+    %w[scripts/lint-conflicts.sh],
+    %w[scripts/lint-rugged]
+  ]
 
+  case node_total
+  when 1
+    all_tasks
+  when 2
+    rake_lint_all, *rest_jobs = all_tasks
+    case node_index
+    when 1
+      [rake_lint_all]
+    else
+      rest_jobs
+    end
+  else
+    raise "Parallelization > 2 (currently set to #{node_total}) isn't supported yet!"
+  end
+end
+
+tasks = jobs_to_run((ENV['CI_NODE_INDEX'] || 1).to_i, (ENV['CI_NODE_TOTAL'] || 1).to_i)
 static_analysis = Gitlab::Popen::Runner.new
 
 static_analysis.run(tasks) do |cmd, &run|
diff --git a/scripts/sync-stable-branch.sh b/scripts/sync-stable-branch.sh
new file mode 100644
index 00000000000..fc62453d743
--- /dev/null
+++ b/scripts/sync-stable-branch.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# This script triggers a merge train job to sync an EE stable branch to its
+# corresponding CE stable branch.
+
+set -e
+
+if [[ "$MERGE_TRAIN_TRIGGER_TOKEN" == '' ]]
+then
+    echo 'The variable MERGE_TRAIN_TRIGGER_TOKEN must be set to a non-empy value'
+    exit 1
+fi
+
+if [[ "$MERGE_TRAIN_TRIGGER_URL" == '' ]]
+then
+    echo 'The variable MERGE_TRAIN_TRIGGER_URL must be set to a non-empy value'
+    exit 1
+fi
+
+if [[ "$CI_COMMIT_REF_NAME" == '' ]]
+then
+    echo 'The variable CI_COMMIT_REF_NAME must be set to a non-empy value'
+    exit 1
+fi
+
+curl -X POST \
+    -F token="$MERGE_TRAIN_TRIGGER_TOKEN" \
+    -F ref=master \
+    -F "variables[MERGE_FOSS]=1" \
+    -F "variables[SOURCE_BRANCH]=$CI_COMMIT_REF_NAME" \
+    -F "variables[TARGET_BRANCH]=${CI_COMMIT_REF_NAME/-ee/}" \
+    "$MERGE_TRAIN_TRIGGER_URL"
diff --git a/scripts/trigger-build b/scripts/trigger-build
index badbb562021..74c1df258c0 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -17,7 +17,7 @@ module Trigger
   end
 
   class Base
-    def invoke!(post_comment: false)
+    def invoke!(post_comment: false, downstream_job_name: nil)
       pipeline = Gitlab.run_trigger(
         downstream_project_path,
         trigger_token,
@@ -28,7 +28,18 @@ module Trigger
       puts "Waiting for downstream pipeline status"
 
       Trigger::CommitComment.post!(pipeline, access_token) if post_comment
-      Trigger::Pipeline.new(downstream_project_path, pipeline.id, access_token)
+      downstream_job =
+        if downstream_job_name
+          Gitlab.pipeline_jobs(downstream_project_path, pipeline.id).auto_paginate.find do |potential_job|
+            potential_job.name == downstream_job_name
+          end
+        end
+
+      if downstream_job
+        Trigger::Job.new(downstream_project_path, downstream_job.id, access_token)
+      else
+        Trigger::Pipeline.new(downstream_project_path, pipeline.id, access_token)
+      end
     end
 
     private
@@ -187,6 +198,14 @@ module Trigger
 
     attr_reader :project, :id, :api_token
 
+    def self.unscoped_class_name
+      name.split('::').last
+    end
+
+    def self.gitlab_api_method_name
+      unscoped_class_name.downcase
+    end
+
     def initialize(project, id, api_token)
       @project = project
       @id = id
@@ -199,17 +218,17 @@ module Trigger
 
     def wait!
       loop do
-        raise "Pipeline timed out after waiting for #{duration} minutes!" if timeout?
+        raise "#{self.class.unscoped_class_name} timed out after waiting for #{duration} minutes!" if timeout?
 
         case status
         when :created, :pending, :running
           print "."
           sleep INTERVAL
         when :success
-          puts "Pipeline succeeded in #{duration} minutes!"
+          puts "#{self.class.unscoped_class_name} succeeded in #{duration} minutes!"
           break
         else
-          raise "Pipeline did not succeed!"
+          raise "#{self.class.unscoped_class_name} did not succeed!"
         end
 
         STDOUT.flush
@@ -225,7 +244,7 @@ module Trigger
     end
 
     def status
-      Gitlab.pipeline(project, id).status.to_sym
+      Gitlab.public_send(self.class.gitlab_api_method_name, project, id).status.to_sym # rubocop:disable GitlabSecurity/PublicSend
     rescue Gitlab::Error::Error => error
       puts "Ignoring the following error: #{error}"
       # Ignore GitLab API hiccups. If GitLab is really down, we'll hit the job
@@ -233,11 +252,13 @@ module Trigger
       :running
     end
   end
+
+  Job = Class.new(Pipeline)
 end
 
 case ARGV[0]
 when 'omnibus'
-  Trigger::Omnibus.new.invoke!(post_comment: true).wait!
+  Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
 when 'cng'
   Trigger::CNG.new.invoke!.wait!
 else