Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42

11 files changed, 128 insertions, 20 deletions

diff --git a/scripts/build_assets_image b/scripts/build_assets_image
index 12beddfa184..e6a5f036fe5 100755
--- a/scripts/build_assets_image
+++ b/scripts/build_assets_image
@@ -20,13 +20,21 @@ cp Dockerfile.assets assets_container.build/
 
 COMMIT_REF_SLUG_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG}
 COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA}
-COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}
 
 DESTINATIONS="--destination=$COMMIT_REF_SLUG_DESTINATION --destination=$COMMIT_SHA_DESTINATION"
 
+# For EE branch builds, add a truncated SHA destination for later use by Omnibus
+# auto-deploy builds
+if [[ "${ASSETS_IMAGE_NAME}" == "gitlab-assets-ee" ]] && [ -n "$CI_COMMIT_BRANCH" ]
+then
+  COMMIT_SHORT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA:0:11}
+  DESTINATIONS="$DESTINATIONS --destination=$COMMIT_SHORT_SHA_DESTINATION"
+fi
+
 # Also tag the image with GitLab version, if running on a tag pipeline, so
 # other projects can simply use that instead of computing the slug.
 if [ -n "$CI_COMMIT_TAG" ]; then
+  COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}
   DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION"
 fi
 
diff --git a/scripts/generate-test-mapping b/scripts/generate-test-mapping
new file mode 100755
index 00000000000..eabe6a5b513
--- /dev/null
+++ b/scripts/generate-test-mapping
@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_generator'
+
+test_mapping_json = ARGV.shift
+crystalball_yamls = ARGV
+
+unless test_mapping_json && !crystalball_yamls.empty?
+  puts "usage: #{__FILE__} <test_mapping_json> [crystalball_yamls...]"
+  exit 1
+end
+
+map_generator = Tooling::TestMapGenerator.new
+map_generator.parse(crystalball_yamls)
+mapping = map_generator.mapping
+
+File.write(test_mapping_json, JSON.pretty_generate(mapping))
+puts "Saved #{test_mapping_json}."
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index 87256269de2..9ae6ce400da 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
+set -o pipefail
 
-cd "$(dirname "$0")/.."
+cd "$(dirname "$0")/.." || exit 1
 echo "=> Linting documents at path $(pwd) as $(whoami)..."
 echo
 ERRORCODE=0
@@ -65,8 +66,11 @@ then
 else
   MERGE_BASE=$(git merge-base ${CI_MERGE_REQUEST_TARGET_BRANCH_SHA} ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA})
   MD_DOC_PATH=$(git diff --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" 'doc/*.md')
-  echo -e "Merged results pipeline detected. Testing only the following files:\n${MD_DOC_PATH}"
- fi
+  if [ -n "${MD_DOC_PATH}" ]
+  then
+    echo -e "Merged results pipeline detected. Testing only the following files:\n${MD_DOC_PATH}"
+  fi
+fi
 
 function run_locally_or_in_docker() {
   local cmd=$1
@@ -77,7 +81,7 @@ function run_locally_or_in_docker() {
     $cmd $args
   elif hash docker 2>/dev/null
   then
-    docker run -t -v ${PWD}:/gitlab -w /gitlab --rm registry.gitlab.com/gitlab-org/gitlab-docs/lint:latest ${cmd} ${args}
+    docker run -t -v ${PWD}:/gitlab -w /gitlab --rm registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.12-vale-2.6.1-markdownlint-0.24.0 ${cmd} ${args}
   else
     echo
     echo "  ✖ ERROR: '${cmd}' not found. Install '${cmd}' or Docker to proceed." >&2
@@ -96,10 +100,15 @@ function run_locally_or_in_docker() {
 
 echo '=> Linting markdown style...'
 echo
-run_locally_or_in_docker 'markdownlint' "--config .markdownlint.json ${MD_DOC_PATH}"
+if [ -z "${MD_DOC_PATH}" ]
+then
+  echo "Merged results pipeline detected, but no markdown files found. Skipping."
+else
+  run_locally_or_in_docker 'markdownlint' "--config .markdownlint.json ${MD_DOC_PATH}"
+fi
 
 echo '=> Linting prose...'
-run_locally_or_in_docker 'vale' "--minAlertLevel error ${MD_DOC_PATH}"
+run_locally_or_in_docker 'vale' "--minAlertLevel error --output=doc/.vale/vale.tmpl ${MD_DOC_PATH}"
 
 if [ $ERRORCODE -ne 0 ]
 then
diff --git a/scripts/lint-rugged b/scripts/lint-rugged
index dfa4df8333f..d7af5499e1c 100755
--- a/scripts/lint-rugged
+++ b/scripts/lint-rugged
@@ -18,6 +18,11 @@ ALLOWED = [
   # Needed to detect Rugged enabled: https://gitlab.com/gitlab-org/gitlab/issues/35371
   'lib/gitlab/config_checker/puma_rugged_checker.rb',
 
+  # Needed for GPG/X509 commit signature API
+  #
+  'app/models/commit.rb',
+  'lib/api/entities/commit_signature.rb',
+
   # Needed for logging
   'config/initializers/peek.rb',
   'config/initializers/lograge.rb',
diff --git a/scripts/pack-test-mapping b/scripts/pack-test-mapping
new file mode 100755
index 00000000000..58ace3eca67
--- /dev/null
+++ b/scripts/pack-test-mapping
@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_packer'
+
+unpacked_json_mapping, packed_json_mapping = ARGV.shift(2)
+unless packed_json_mapping && unpacked_json_mapping
+  puts "usage: #{__FILE__} <unpacked_json_mapping> <packed_json_mapping>"
+  exit 1
+end
+
+puts "Compressing #{unpacked_json_mapping}"
+
+mapping = JSON.parse(File.read(unpacked_json_mapping))
+packed_mapping = Tooling::TestMapPacker.new.pack(mapping)
+
+puts "Writing packed #{packed_json_mapping}"
+File.write(packed_json_mapping, JSON.generate(packed_mapping))
+puts "Saved #{packed_json_mapping}."
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index 82be2d3a691..bfc35a6abde 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -52,10 +52,10 @@ gitlab:
     resources:
       requests:
         cpu: 855m
-        memory: 1071M
+        memory: 1285M
       limits:
         cpu: 1282m
-        memory: 1606M
+        memory: 1927M
     hpa:
       targetAverageValue: 650m
   task-runner:
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index b8cbe625e5b..59e7d183ae7 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -75,7 +75,7 @@ function kubectl_cleanup_release() {
   kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \
     | grep "${release}" \
     | awk '{print $1}' \
-    | xargs kubectl --namespace "${namespace}" delete \
+    | xargs kubectl --namespace "${namespace}" delete --ignore-not-found \
     || true
 }
 
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 3812a8b8ef7..5f003d032b7 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -48,12 +48,49 @@ function update_tests_metadata() {
   fi
 }
 
+function retrieve_tests_mapping() {
+  mkdir -p crystalball/
+
+  if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+    (wget -O "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+  fi
+
+  scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
+}
+
+function update_tests_mapping() {
+  if ! crystalball_rspec_data_exists; then
+    echo "No crystalball rspec data found."
+    return 0
+  fi
+
+  scripts/generate-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" crystalball/rspec*.yml
+
+  scripts/pack-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+
+  gzip "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+
+  if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz"
+    else
+      echo "Not uploading report to S3 as the pipeline is not a scheduled one."
+    fi
+  fi
+
+  rm -f crystalball/rspec*.yml
+}
+
+function crystalball_rspec_data_exists() {
+  compgen -G "crystalball/rspec*.yml" > /dev/null;
+}
+
 function rspec_simple_job() {
   local rspec_opts="${1}"
 
   export NO_KNAPSACK="1"
 
-  bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}
+  bin/rspec -Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}
 }
 
 function rspec_paralellized_job() {
@@ -106,7 +143,7 @@ function rspec_paralellized_job() {
 
   export MEMORY_TEST_PATH="tmp/memory_test/${report_name}_memory.csv"
 
-  knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
+  knapsack rspec "-Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
 
   date
 }
diff --git a/scripts/unpack-test-mapping b/scripts/unpack-test-mapping
new file mode 100755
index 00000000000..c0f706c3f9f
--- /dev/null
+++ b/scripts/unpack-test-mapping
@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_packer'
+
+packed_json_mapping, unpacked_json_mapping = ARGV.shift(2)
+unless packed_json_mapping && unpacked_json_mapping
+  puts "usage: #{__FILE__} <packed_json_mapping> <unpacked_json_mapping>"
+  exit 1
+end
+
+packed_mapping = JSON.parse(File.read(packed_json_mapping))
+mapping = Tooling::TestMapPacker.new.unpack(packed_mapping)
+
+puts "Writing unpacked #{unpacked_json_mapping}"
+File.write(unpacked_json_mapping, JSON.generate(mapping))
+puts "Saved #{unpacked_json_mapping}."
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 9d188fc7b77..3829bcdf24e 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -31,8 +31,8 @@ function install_api_client_dependencies_with_apt() {
 }
 
 function install_gitlab_gem() {
-  gem install httparty --no-document --version 0.17.3
-  gem install gitlab --no-document --version 4.13.0
+  gem install httparty --no-document --version 0.18.1
+  gem install gitlab --no-document --version 4.14.1
 }
 
 function install_tff_gem() {
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index 9541d2468b1..1f73753be82 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -40,12 +40,6 @@ tests = [
   },
 
   {
-    explanation: 'EE lib should map to respective spec',
-    source: 'ee/lib/flipper_session.rb',
-    expected: ['ee/spec/lib/flipper_session_spec.rb']
-  },
-
-  {
     explanation: 'Tooling should map to respective spec',
     source: 'tooling/lib/tooling/test_file_finder.rb',
     expected: ['spec/tooling/lib/tooling/test_file_finder_spec.rb']