Add latest changes from gitlab-org/gitlab@12-6-stable-ee

9 files changed, 157 insertions, 39 deletions

diff --git a/.gitlab/ci/cache-repo.gitlab-ci.yml b/.gitlab/ci/cache-repo.gitlab-ci.yml
new file mode 100644
index 00000000000..f856afd3a02
--- /dev/null
+++ b/.gitlab/ci/cache-repo.gitlab-ci.yml
@@ -0,0 +1,33 @@
+# Builds a cached .tar.gz of the master branch with full history and
+# uploads it to Google Cloud Storage. This archive is downloaded by a
+# script defined by a CI/CD variable named CI_PRE_CLONE_SCRIPT. This has
+# two benefits:
+#
+# 1. It speeds up builds. A 800 MB download only takes seconds.
+# 2. It significantly reduces load on the file server. Smaller deltas
+#    means less time spent in git pack-objects.
+#
+# Since the destination directory of the archive depends on the project
+# ID, this is only run on GitLab.com.
+#
+# CI_REPO_CACHE_CREDENTIALS contains the Google Cloud service account
+# JSON for uploading to the gitlab-ci-git-repo-cache bucket. These
+# credentials are stored in the Production vault.
+#
+# Note that this bucket should be located in the same continent as the
+# runner, or network egress charges will apply:
+# https://cloud.google.com/storage/pricing
+cache-repo:
+  extends:
+    - .only:variables_refs-canonical-dot-com-schedules
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
+  stage: sync
+  allow_failure: true
+  variables:
+    GIT_DEPTH: 0
+    TAR_FILENAME: /tmp/gitlab-master.tar
+  script:
+    - gcloud auth activate-service-account --key-file=$CI_REPO_CACHE_CREDENTIALS
+    - tar cf $TAR_FILENAME .
+    - gzip $TAR_FILENAME
+    - gsutil cp $TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/gitlab-master.tar.gz
diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml
index 07375fca611..cd0e4085e10 100644
--- a/.gitlab/ci/docs.gitlab-ci.yml
+++ b/.gitlab/ci/docs.gitlab-ci.yml
@@ -24,7 +24,8 @@
     - apk add --update openssl
     - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs
     - chmod 755 trigger-build-docs
-    - gem install gitlab --no-document
+    - gem install httparty --no-document --version 0.17.3
+    - gem install gitlab --no-document --version 4.13.0
 
 # Always trigger a docs build in gitlab-docs only on docs-only branches.
 # Useful to preview the docs changes live.
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index 0b72461a9fd..6578eec8234 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -13,7 +13,7 @@
     - .default-before_script
     - .assets-compile-cache
     - .only:changes-code-backstage-qa
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-19.03.1
   stage: test
   dependencies: ["setup-test-env"]
   needs: ["setup-test-env"]
@@ -74,7 +74,6 @@ gitlab:assets:compile pull-cache:
     - .default-before_script
     - .assets-compile-cache
     - .only:changes-code-backstage-qa
-    - .use-pg9
   stage: prepare
   script:
     - node --version
@@ -83,6 +82,7 @@ gitlab:assets:compile pull-cache:
     - retry bundle exec rake gitlab:assets:compile
     - scripts/clean-old-cached-assets
   variables:
+    SETUP_DB: "false"
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
   cache:
@@ -244,6 +244,12 @@ webpack-dev-server:
   dependencies: ["setup-test-env", "compile-assets pull-cache"]
   variables:
     WEBPACK_MEMORY_TEST: "true"
+    WEBPACK_VENDOR_DLL: "true"
   script:
-    - node --version
+    - yarn webpack-vendor
     - node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js
+  artifacts:
+    name: webpack-dev-server
+    expire_in: 31d
+    paths:
+      - webpack-dev-server.json
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml
index d746d8fe030..9ebd28c7258 100644
--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -93,7 +93,7 @@
   - "config.ru"
   - "{package.json,yarn.lock}"
   - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
-  - "doc/api/graphql/**/*"
+  - "doc/api/graphql/reference/*" # Files in this folder are auto-generated
 
 .backstage-patterns: &backstage-patterns
   - "Dangerfile"
@@ -139,7 +139,7 @@
       - "config.ru"
       - "{package.json,yarn.lock}"
       - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
-      - "doc/api/graphql/**/*"
+      - "doc/api/graphql/reference/*" # Files in this folder are auto-generated
       # Backstage changes
       - "Dangerfile"
       - "danger/**/*"
@@ -163,7 +163,7 @@
       - "config.ru"
       - "{package.json,yarn.lock}"
       - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
-      - "doc/api/graphql/**/*"
+      - "doc/api/graphql/reference/*" # Files in this folder are auto-generated
       # QA changes
       - ".dockerignore"
       - "qa/**/*"
@@ -183,7 +183,7 @@
       - "config.ru"
       - "{package.json,yarn.lock}"
       - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
-      - "doc/api/graphql/**/*"
+      - "doc/api/graphql/reference/*" # Files in this folder are auto-generated
       # Backstage changes
       - "Dangerfile"
       - "danger/**/*"
@@ -202,7 +202,7 @@
     - name: redis:alpine
 
 .use-pg10:
-  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
+  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-golang-1.12-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
   services:
     - name: postgres:10.9
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
@@ -213,15 +213,15 @@
     - name: postgres:9.6
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
-    - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12
+    - name: elasticsearch:5.6.12
 
 .use-pg10-ee:
-  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
+  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-golang-1.12-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
   services:
     - name: postgres:10.9
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
-    - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12
+    - name: elasticsearch:5.6.12
 
 .only-ee:
   only:
diff --git a/.gitlab/ci/notifications.gitlab-ci.yml b/.gitlab/ci/notifications.gitlab-ci.yml
new file mode 100644
index 00000000000..4271e709f45
--- /dev/null
+++ b/.gitlab/ci/notifications.gitlab-ci.yml
@@ -0,0 +1,23 @@
+.notify:
+  image: ruby:2.6-alpine
+  stage: notification
+  dependencies: []
+  cache: {}
+  before_script:
+    - apk update && apk add git curl bash
+    - source scripts/utils.sh
+    - source scripts/notifications.sh
+    - install_gitlab_gem
+  variables:
+    COMMIT_NOTES_URL: "https://${CI_SERVER_HOST}/${CI_PROJECT_PATH}/commit/${CI_COMMIT_SHA}#notes-list"
+
+schedule:package-and-qa:notify-failure:
+  extends:
+    - .only:variables_refs-canonical-dot-com-schedules
+    - .notify
+  script:
+    - 'export NOTIFICATION_MESSAGE=":skull_and_crossbones: Scheduled QA against master failed! :skull_and_crossbones: See ${CI_PIPELINE_URL}. For downstream pipelines, see ${COMMIT_NOTES_URL}"'
+    - 'notify_on_job_failure schedule:package-and-qa qa-master "${NOTIFICATION_MESSAGE}" ci_failing'
+  needs: ["schedule:package-and-qa"]
+  allow_failure: true
+  when: always
diff --git a/.gitlab/ci/releases.gitlab-ci.yml b/.gitlab/ci/releases.gitlab-ci.yml
index 1ddc4e90fcf..d4e0236f3a8 100644
--- a/.gitlab/ci/releases.gitlab-ci.yml
+++ b/.gitlab/ci/releases.gitlab-ci.yml
@@ -1,22 +1,36 @@
 ---
 
-# Syncs any changes pushed to a stable branch to the corresponding CE stable
-# branch. We run this prior to any tests so that random failures don't prevent a
-# sync.
-sync-stable-branch:
+# Syncs any changes pushed to a stable branch to the corresponding
+# gitlab-foss/CE stable branch. We run this prior to any tests so that random
+# failures don't prevent a sync.
+.merge-train-sync:
   # We don't need/want any global before/after commands, so we overwrite these
   # settings.
   image: alpine:edge
   stage: sync
-  # This job should only run on EE stable branches on the canonical GitLab.com
-  # repository.
-  only:
-    variables:
-      - $CI_SERVER_HOST == "gitlab.com"
-    refs:
-      - /^[\d-]+-stable-ee$/@gitlab-org/gitlab
   before_script:
     - apk add --no-cache --update curl bash
   after_script: []
   script:
     - bash scripts/sync-stable-branch.sh
+  only:
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com"
+
+sync-stable-branch:
+  extends: .merge-train-sync
+  variables:
+    SOURCE_PROJECT: gitlab-org/gitlab
+    TARGET_PROJECT: gitlab-org/gitlab-foss
+  only:
+    refs:
+      - /^[\d-]+-stable-ee$/@gitlab-org/gitlab
+
+sync-security-branch:
+  extends: .merge-train-sync
+  variables:
+    SOURCE_PROJECT: gitlab-org/security/gitlab
+    TARGET_PROJECT: gitlab-org/security/gitlab-foss
+  only:
+    refs:
+      - /^[\d-]+-stable-ee$/@gitlab-org/security/gitlab
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index fbb7826b6f2..4ff14b660b3 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -20,6 +20,7 @@ code_quality:
   variables:
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
+    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:12-5-stable"
   script:
     - |
       if ! docker info &>/dev/null; then
@@ -27,14 +28,17 @@ code_quality:
           export DOCKER_HOST='tcp://localhost:2375'
         fi
       fi
+    - docker pull --quiet "$CODE_QUALITY_IMAGE"
     - docker run
         --env SOURCE_CODE="$PWD"
         --volume "$PWD":/code
         --volume /var/run/docker.sock:/var/run/docker.sock
-        "registry.gitlab.com/gitlab-org/security-products/codequality:12-0-stable" /code
+        "$CODE_QUALITY_IMAGE" /code
   artifacts:
     reports:
       codequality: gl-code-quality-report.json
+    paths:
+      - gl-code-quality-report.json
     expire_in: 1 week
   dependencies: []
   except:
@@ -165,7 +169,6 @@ dependency_scanning:
           DS_ANALYZER_IMAGE_TAG \
           DS_DEFAULT_ANALYZERS \
           DS_EXCLUDED_PATHS \
-          DEP_SCAN_DISABLE_REMOTE_CHECKS \
           DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
           DS_PULL_ANALYZER_IMAGE_TIMEOUT \
           DS_RUN_ANALYZER_TIMEOUT \
@@ -231,9 +234,3 @@ dast:
       - gl-dast-report.json
     reports:
       dast: gl-dast-report.json
-  only:
-    variables:
-      - $GITLAB_FEATURES =~ /\bdast\b/
-  except:
-    variables:
-      - $DAST_DISABLED
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 4ed9ac03d0c..49447bc629b 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -23,8 +23,10 @@ build-qa-image:
   stage: prepare
   script:
     - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"'
+    - export QA_MASTER_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:master"
     - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}"
-    - time docker build --cache-from gitlab/gitlab-${GITLAB_EDITION}-qa:nightly --tag ${QA_IMAGE} --file ./qa/Dockerfile ./
+    - time docker pull "${QA_MASTER_IMAGE}"
+    - time docker build --cache-from "${QA_MASTER_IMAGE}" --tag ${QA_IMAGE} --file ./qa/Dockerfile ./
     - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY}
     - time docker push ${QA_IMAGE}
 
@@ -94,10 +96,7 @@ schedule:review-build-cng:
   variables:
     HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
     DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
-    # v2.4.4 + two improvements:
-    # - Allow to pass an EE license when installing the chart: https://gitlab.com/gitlab-org/charts/gitlab/merge_requests/1008
-    # - Allow to customize the livenessProbe for `gitlab-shell`: https://gitlab.com/gitlab-org/charts/gitlab/merge_requests/1021
-    GITLAB_HELM_CHART_REF: "6c655ed77e60f1f7f533afb97bef8c9cb7dc61eb"
+    GITLAB_HELM_CHART_REF: "v2.5.1"
     GITLAB_EDITION: "ce"
   environment:
     name: review/${CI_COMMIT_REF_NAME}
@@ -135,13 +134,11 @@ review-deploy:
     - .review-deploy-base
     - .only-review
     - .only:changes-code-qa
-  needs: ["review-build-cng"]
 
 schedule:review-deploy:
   extends:
     - .review-deploy-base
     - .only-review-schedules
-  needs: ["schedule:review-build-cng"]
 
 .base-review-stop:
   extends:
@@ -280,7 +277,7 @@ parallel-spec-reports:
     - .only-review
     - .only:changes-code-qa
   image: ruby:2.6-alpine
-  stage: post-test
+  stage: post-qa
   dependencies: ["review-qa-all"]
   variables:
     NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
diff --git a/.gitlab/issue_templates/Productivity Improvement.md b/.gitlab/issue_templates/Productivity Improvement.md
new file mode 100644
index 00000000000..89505cd85b4
--- /dev/null
+++ b/.gitlab/issue_templates/Productivity Improvement.md
@@ -0,0 +1,47 @@
+## What is the productivity problem to solve?
+
+<!--
+Please describe the productivity problem that needs to be solved backed by charts from
+https://about.gitlab.com/handbook/engineering/quality/engineering-productivity-team/#engineering-productivity-team-metrics.
+-->
+
+### Problem identification checklist
+
+- [ ] The root cause of the problem is identified.
+- [ ] The surface of the problem is as small as possible.
+
+## What are the potential solutions?
+
+<!--
+Please provide potential solutions here. Example solutions could be:
+
+- Dogfood a feature.
+- Refactor/improve some workflow code.
+- Throw more money at the problem.
+
+Please provide pros/cons and a weight estimate for each solution.
+-->
+
+- [ ] All potential solutions are listed.
+- [ ] A solution has been chosen for the first iteration: `PUT THE CHOSEN SOLUTION HERE`
+
+## Who and when will the solution be implemented?
+
+<!--
+For history reason, please list the person that will implement the solution and
+the planned milestone/date.
+-->
+
+## Verify that the solution has improved the situation
+
+<!--
+Ideally, looking at the charts from the first part, we should see an improvement
+after the implementation is merged/deployed/released.
+-->
+
+- [ ] The solution improved the situation.
+  - If yes, check this box and close the issue. Well done! :tada:
+  - Otherwise, create a new "Productivity Improvement" issue. You can re-use the description from this issue, but obviously another solution should be chosen this time.
+
+/label ~"Engineering Productivity" ~meta
+/cc @gl-quality/eng-prod