Add latest changes from gitlab-org/gitlab@12-4-stable-ee

9 files changed, 193 insertions, 42 deletions

diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index e2c22785963..d097c2aee91 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -24,12 +24,12 @@ then
 fi
 
 # Make sure no files in doc/ are executable
-EXEC_PERM_COUNT=$(find doc/ app/ -type f -perm 755 | wc -l)
+EXEC_PERM_COUNT=$(find doc/ -type f -perm 755 | wc -l)
 echo '=> Checking for executable permissions...'
 if [ "${EXEC_PERM_COUNT}" -ne 0 ]
 then
   echo '✖ ERROR: Executable permissions should not be used in documentation! Use `chmod 644` to the files in question:' >&2
-  find doc/ app/ -type f -perm 755
+  find doc/ -type f -perm 755
   exit 1
 fi
 
diff --git a/scripts/notify-slack b/scripts/notify-slack
new file mode 100755
index 00000000000..5907fd8b986
--- /dev/null
+++ b/scripts/notify-slack
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Sends Slack notification MSG to CI_SLACK_WEBHOOK_URL (which needs to be set).
+# ICON_EMOJI needs to be set to an icon emoji name (without the `:` around it).
+
+CHANNEL=$1
+MSG=$2
+ICON_EMOJI=$3
+
+if [ -z "$CHANNEL" ] || [ -z "$CI_SLACK_WEBHOOK_URL" ] || [ -z "$MSG" ] || [ -z "$ICON_EMOJI" ]; then
+    echo "Missing argument(s) - Use: $0 channel message icon_emoji"
+    echo "and set CI_SLACK_WEBHOOK_URL environment variable."
+else
+    curl -X POST --data-urlencode 'payload={"channel": "#'"$CHANNEL"'", "username": "GitLab QA Bot", "text": "'"$MSG"'", "icon_emoji": "'":$ICON_EMOJI:"'"}' "$CI_SLACK_WEBHOOK_URL"
+fi
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index 4166070f7cd..9edc1a2b857 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -18,7 +18,8 @@ class AutomatedCleanup
   ].freeze
 
   def self.ee?
-    ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('CHANGELOG-EE.md')
+    # Support former project name for `dev`
+    %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME'])
   end
 
   def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN'])
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index 403c1df97aa..573a5ccde11 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -35,11 +35,14 @@ gitlab:
   gitlab-shell:
     resources:
       requests:
-        cpu: 70m
+        cpu: 125m
         memory: 20M
       limits:
-        cpu: 140m
+        cpu: 250m
         memory: 40M
+    maxReplicas: 3
+    hpa:
+      targetAverageValue: 130m
   sidekiq:
     resources:
       requests:
@@ -59,34 +62,43 @@ gitlab:
   unicorn:
     resources:
       requests:
-        cpu: 600m
+        cpu: 400m
         memory: 1.4G
       limits:
-        cpu: 1.2G
+        cpu: 800m
         memory: 2.8G
+    deployment:
+      readinessProbe:
+        initialDelaySeconds: 5  # Default is 0
+        periodSeconds: 15  # Default is 10
+        timeoutSeconds: 5  # Default is 2
     workhorse:
       resources:
         requests:
-          cpu: 100m
+          cpu: 175m
           memory: 100M
         limits:
-          cpu: 200m
+          cpu: 350m
           memory: 200M
+      readinessProbe:
+        initialDelaySeconds: 5  # Default is 0
+        periodSeconds: 15  # Default is 10
+        timeoutSeconds: 5  # Default is 2
 gitlab-runner:
   resources:
     requests:
-      cpu: 300m
+      cpu: 355m
       memory: 300M
     limits:
-      cpu: 600m
+      cpu: 710m
       memory: 600M
 minio:
   resources:
     requests:
-      cpu: 100m
+      cpu: 5m
       memory: 128M
     limits:
-      cpu: 200m
+      cpu: 10m
       memory: 280M
 nginx-ingress:
   controller:
@@ -95,10 +107,10 @@ nginx-ingress:
     replicaCount: 2
     resources:
       requests:
-        cpu: 150m
+        cpu: 100m
         memory: 250M
       limits:
-        cpu: 300m
+        cpu: 200m
         memory: 500M
     minAvailable: 1
     service:
@@ -141,7 +153,8 @@ redis:
 redis-ha:
   enabled: false
 registry:
-  minReplicas: 1
+  hpa:
+    minReplicas: 1
   resources:
     requests:
       cpu: 50m
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index a4fb4749720..51768d07860 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -255,25 +255,13 @@ EOF
 }
 
 function display_deployment_debug() {
+  # Get all pods for this release
+  echoinfo "Pods for release ${CI_ENVIRONMENT_SLUG}"
   kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG}
 
-  migrations_pod=$(get_pod "migrations");
-  if [ -z "${migrations_pod}" ]; then
-    echoerr "Migrations pod not found."
-  else
-    echoinfo "Logs tail of the ${migrations_pod} pod..."
-
-    kubectl logs -n "$KUBE_NAMESPACE" "${migrations_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
-  fi
-
-  unicorn_pod=$(get_pod "unicorn");
-  if [ -z "${unicorn_pod}" ]; then
-    echoerr "Unicorn pod not found."
-  else
-    echoinfo "Logs tail of the ${unicorn_pod} pod..."
-
-    kubectl logs -n "$KUBE_NAMESPACE" -c unicorn "${unicorn_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
-  fi
+  # Get all non-completed jobs
+  echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}"
+  kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1
 }
 
 function add_license() {
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
new file mode 100644
index 00000000000..1c75958aef1
--- /dev/null
+++ b/scripts/rspec_helpers.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+function retrieve_tests_metadata() {
+  mkdir -p knapsack/ rspec_flaky/ rspec_profiling/
+
+  if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
+    wget -O "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+  fi
+
+  if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
+    wget -O "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+  fi
+}
+
+function update_tests_metadata() {
+  echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+
+  scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*_pg9_*.json
+  if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
+    scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+  fi
+
+  rm -f knapsack/rspec*.json
+
+  scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json
+
+  export FLAKY_RSPEC_GENERATE_REPORT="1"
+  scripts/prune-old-flaky-specs "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+
+  if [[ -n ${TESTS_METADATA_S3_BUCKET} ]]; then
+    scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+  fi
+
+  rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
+
+  scripts/insert-rspec-profiling-data
+}
+
+function rspec_simple_job() {
+  local rspec_opts="${1}"
+
+  export NO_KNAPSACK="1"
+  export CACHE_CLASSES="true"
+
+  scripts/gitaly-test-spawn
+
+  bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}
+}
+
+function rspec_paralellized_job() {
+  read -ra job_name <<< "$CI_JOB_NAME"
+  local test_tool="${job_name[0]}"
+  local test_level="${job_name[1]}"
+  local database="${job_name[2]}"
+  local rspec_opts="${1}"
+  local spec_folder_prefix=""
+
+  if [[ "${test_tool}" =~ "-ee" ]]; then
+    spec_folder_prefix="ee/"
+  fi
+
+  export CACHE_CLASSES="true"
+  export KNAPSACK_LOG_LEVEL="debug"
+  export KNAPSACK_REPORT_PATH="knapsack/${test_tool}_${test_level}_${database}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
+
+  cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
+
+  if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
+    pattern=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})")
+    export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
+  fi
+
+  echo "KNAPSACK_TEST_FILE_PATTERN: ${KNAPSACK_TEST_FILE_PATTERN}"
+
+  if [[ -d "ee/" ]]; then
+    export KNAPSACK_GENERATE_REPORT="true"
+    export FLAKY_RSPEC_GENERATE_REPORT="true"
+    export SUITE_FLAKY_RSPEC_REPORT_PATH="${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    export FLAKY_RSPEC_REPORT_PATH="rspec_flaky/all_${test_tool}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
+    export NEW_FLAKY_RSPEC_REPORT_PATH="rspec_flaky/new_${test_tool}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
+
+    if [[ ! -f $FLAKY_RSPEC_REPORT_PATH ]]; then
+      echo "{}" > "${FLAKY_RSPEC_REPORT_PATH}"
+    fi
+
+    if [[ ! -f $NEW_FLAKY_RSPEC_REPORT_PATH ]]; then
+      echo "{}" > "${NEW_FLAKY_RSPEC_REPORT_PATH}"
+    fi
+  fi
+
+  scripts/gitaly-test-spawn
+
+  mkdir -p tmp/memory_test
+
+  export MEMORY_TEST_PATH="tmp/memory_test/${test_tool}_${test_level}_${database}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
+
+  knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
+
+  date
+}
diff --git a/scripts/trigger-build b/scripts/trigger-build
index 4d8110fce10..badbb562021 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -12,7 +12,8 @@ end
 
 module Trigger
   def self.ee?
-    ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('CHANGELOG-EE.md')
+    # Support former project name for `dev`
+    %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME'])
   end
 
   class Base
@@ -89,7 +90,7 @@ module Trigger
     private
 
     def downstream_project_path
-      'gitlab-org/omnibus-gitlab'
+      ENV['OMNIBUS_PROJECT_PATH'] || 'gitlab-org/build/omnibus-gitlab-mirror'
     end
 
     def ref
@@ -97,7 +98,7 @@ module Trigger
     end
 
     def trigger_token
-      ENV['BUILD_TRIGGER_TOKEN']
+      ENV['CI_JOB_TOKEN']
     end
 
     def access_token
diff --git a/scripts/trigger-build-docs b/scripts/trigger-build-docs
index 046ee5bceb8..ea717d9ccbf 100755
--- a/scripts/trigger-build-docs
+++ b/scripts/trigger-build-docs
@@ -16,14 +16,12 @@ end
 GITLAB_DOCS_REPO = 'gitlab-org/gitlab-docs'.freeze
 
 #
-# Truncate the remote docs branch name otherwise we hit the filesystem
-# limit and the directory name where NGINX serves the site won't match
-# the branch name.
+# This is the branch that will be created in the gitlab-docs project.
+# Name it after the product we're previewing and the ID of the MR that
+# kicked the review app.
 #
 def docs_branch
-  # The maximum string length a file can have on a filesystem (ext4)
-  # is 63 characters. CI_ENVIRONMENT_SLUG is limited to 24 characters.
-  ENV["CI_ENVIRONMENT_SLUG"]
+  "docs-preview-#{slug}-#{ENV["CI_MERGE_REQUEST_IID"]}"
 end
 
 #
diff --git a/scripts/update-feature-categories b/scripts/update-feature-categories
new file mode 100755
index 00000000000..ed5d8dccdd6
--- /dev/null
+++ b/scripts/update-feature-categories
@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'uri'
+require 'net/http'
+require 'yaml'
+
+url = URI("https://gitlab.com/gitlab-com/www-gitlab-com/raw/master/data/stages.yml")
+
+http = Net::HTTP.new(url.host, url.port)
+http.use_ssl = true
+
+request = Net::HTTP::Get.new(url)
+
+response = http.request(request)
+
+stages_doc = YAML.safe_load(response.read_body)
+feature_categories = stages_doc["stages"].values
+  .flat_map { |stage| stage["groups"].values }
+  .flat_map { |group| group["categories"] }
+  .select(&:itself)
+  .uniq
+  .sort
+
+File.open("#{__dir__}/../config/feature_categories.yml", 'w') do |file|
+  file.puts(<<~HEADER_COMMENT)
+    #
+    # This file contains a list of all feature categories in GitLab
+    # It is generated from the stages file at #{url}.
+    # If you would like to update it, please run
+    # `./scripts/update-feature-categories` to generate a new copy
+    #
+    # PLEASE DO NOT EDIT THIS FILE MANUALLY.
+    #
+  HEADER_COMMENT
+  file.write(feature_categories.to_yaml)
+end