Add latest changes from gitlab-org/gitlab@13-3-stable-ee

5 files changed, 165 insertions, 26 deletions

diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
new file mode 100644
index 00000000000..e87f0f28d01
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
@@ -0,0 +1,146 @@
+stages:
+    - build
+    - test
+    - deploy
+    - fuzz
+
+variables:
+    FUZZAPI_PROFILE: Quick
+    FUZZAPI_VERSION: latest
+    FUZZAPI_CONFIG: "/app/.gitlab-api-fuzzing.yml"
+    FUZZAPI_TIMEOUT: 30
+    FUZZAPI_REPORT: gl-api-fuzzing-report.xml
+    #
+    FUZZAPI_D_NETWORK: testing-net
+    #
+    # Wait up to 5 minutes for API Fuzzer and target url to become
+    # available (non 500 response to HTTP(s))
+    FUZZAPI_SERVICE_START_TIMEOUT: "300"
+    #
+
+apifuzzer_fuzz:
+    stage: fuzz
+    image: docker:19.03.12
+    variables:
+        DOCKER_DRIVER: overlay2
+        DOCKER_TLS_CERTDIR: ""
+        FUZZAPI_PROJECT: $CI_PROJECT_PATH
+        FUZZAPI_API: http://apifuzzer:80
+    allow_failure: true
+    rules:
+        - if: $API_FUZZING_DISABLED
+          when: never
+        - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+              $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+          when: never
+        - if: $FUZZAPI_HAR == null &&
+              $FUZZAPI_OPENAPI == null &&
+              $FUZZAPI_D_WORKER_IMAGE == null
+          when: never
+        - if: $FUZZAPI_D_WORKER_IMAGE == null &&
+              $FUZZAPI_TARGET_URL == null
+          when: never
+        - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+    services:
+        - docker:19.03.12-dind
+    script:
+        #
+        - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+        #
+        - docker network create --driver bridge $FUZZAPI_D_NETWORK
+        #
+        # Run user provided pre-script
+        - sh -c "$FUZZAPI_PRE_SCRIPT"
+        #
+        # Start peach testing engine container
+        - |
+            docker run -d \
+            --name apifuzzer \
+            --network $FUZZAPI_D_NETWORK \
+            -e Proxy:Port=8000 \
+            -e TZ=America/Los_Angeles \
+            -e FUZZAPI_API=http://127.0.0.1:80 \
+            -e FUZZAPI_PROJECT \
+            -e FUZZAPI_PROFILE \
+            -e FUZZAPI_CONFIG \
+            -e FUZZAPI_REPORT \
+            -e FUZZAPI_HAR \
+            -e FUZZAPI_OPENAPI \
+            -e FUZZAPI_TARGET_URL \
+            -e FUZZAPI_OVERRIDES_FILE \
+            -e FUZZAPI_OVERRIDES_ENV \
+            -e FUZZAPI_OVERRIDES_CMD \
+            -e FUZZAPI_OVERRIDES_INTERVAL \
+            -e FUZZAPI_TIMEOUT \
+            -e FUZZAPI_VERBOSE \
+            -e FUZZAPI_SERVICE_START_TIMEOUT \
+            -e GITLAB_FEATURES \
+            -v $CI_PROJECT_DIR:/app \
+            -p 80:80 \
+            -p 8000:8000 \
+            -p 514:514 \
+            --restart=no \
+            registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src:${FUZZAPI_VERSION}-engine
+        #
+        # Start target container
+        - |
+            if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \
+                docker run -d \
+                    --name target \
+                    --network $FUZZAPI_D_NETWORK \
+                    $FUZZAPI_D_TARGET_ENV \
+                    $FUZZAPI_D_TARGET_PORTS \
+                    $FUZZAPI_D_TARGET_VOLUME \
+                    --restart=no \
+                    $FUZZAPI_D_TARGET_IMAGE \
+                ; fi
+        #
+        # Start worker container
+        - |
+            if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \
+                echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE" \
+                docker run \
+                    --name worker \
+                    --network $FUZZAPI_D_NETWORK \
+                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_PROJECT \
+                    -e FUZZAPI_PROFILE \
+                    -e FUZZAPI_AUTOMATION_CMD \
+                    -e FUZZAPI_CONFIG \
+                    -e FUZZAPI_REPORT \
+                    -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \
+                    $FUZZAPI_D_WORKER_ENV \
+                    $FUZZAPI_D_WORKER_PORTS \
+                    $FUZZAPI_D_WORKER_VOLUME \
+                    --restart=no \
+                    $FUZZAPI_D_WORKER_IMAGE \
+                ; fi
+        #
+        # Wait for testing to complete if api fuzzer is scanning
+        - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi
+        #
+        # Run user provided pre-script
+        - sh -c "$FUZZAPI_POST_SCRIPT"
+        #
+    after_script:
+        #
+        # Shutdown all containers
+        - echo "Stopping all containers"
+        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi
+        - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker stop worker; fi
+        - docker stop apifuzzer
+        #
+        # Save docker logs
+        - docker logs apifuzzer &> gl-api_fuzzing-logs.log
+        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi
+        - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker logs worker &> gl-api_fuzzing-worker-logs.log; fi
+        #
+    artifacts:
+        when: always
+        paths:
+            - ./gl-api_fuzzing*.log
+            - ./gl-api_fuzzing*.zip
+        reports:
+            junit: $FUZZAPI_REPORT
+
+# end
diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
index 2fab8b95a3d..3f47e575afd 100644
--- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
@@ -3,22 +3,26 @@
 variables:
   # Which branch we want to run full fledged long running fuzzing jobs.
   # All others will run fuzzing regression
-  COVERAGE_FUZZING_BRANCH: "$CI_DEFAULT_BRANCH"
-  # This is using semantic version and will always download latest v1 gitlab-cov-fuzz release
-  COVERAGE_FUZZING_VERSION: v1
+  COVFUZZ_BRANCH: "$CI_DEFAULT_BRANCH"
+  # This is using semantic version and will always download latest v2 gitlab-cov-fuzz release
+  COVFUZZ_VERSION: v2
   # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries
   # to their own servers
-  COVERAGE_FUZZING_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw"
+  COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw"
+
 
 .fuzz_base:
   stage: fuzz
   allow_failure: true
   before_script:
+    - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN
+    - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN
+    - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID
     - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi
-    - wget -O gitlab-cov-fuzz "${COVERAGE_FUZZING_URL_PREFIX}"/"${COVERAGE_FUZZING_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64
+    - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64
     - chmod a+x gitlab-cov-fuzz
     - export REGRESSION=true
-    - if [[ $CI_COMMIT_BRANCH = $COVERAGE_FUZZING_BRANCH ]]; then REGRESSION=false; fi;
+    - if [[ $CI_COMMIT_BRANCH = $COVFUZZ_BRANCH ]]; then REGRESSION=false; fi;
   artifacts:
     paths:
       - corpus
@@ -28,7 +32,7 @@ variables:
       coverage_fuzzing: gl-coverage-fuzzing-report.json
     when: always
   rules:
-    - if: $COVERAGE_FUZZING_DISABLED
+    - if: $COVFUZZ_DISABLED
       when: never
     - if: $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/
     - if: $CI_RUNNER_EXECUTABLE_ARCH == "linux"
diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
index 37f6cd216ca..d5275c57ef8 100644
--- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
@@ -111,6 +111,7 @@ gemnasium-dependency_scanning:
         - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}'
         - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}'
         - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}'
+        - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}'
 
 gemnasium-maven-dependency_scanning:
   extends: .ds-analyzer
@@ -144,8 +145,8 @@ gemnasium-python-dependency_scanning:
         - '{Pipfile,*/Pipfile,*/*/Pipfile}'
         - '{requires.txt,*/requires.txt,*/*/requires.txt}'
         - '{setup.py,*/setup.py,*/*/setup.py}'
-      # Support passing of $PIP_REQUIREMENTS_FILE
-      # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning
+        # Support passing of $PIP_REQUIREMENTS_FILE
+        # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning
     - if: $CI_COMMIT_BRANCH &&
           $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
           $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ &&
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index f0e2f48dd5c..6eb17341472 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -52,8 +52,7 @@ sast:
   rules:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
-    - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/
+    - if: $CI_COMMIT_BRANCH
   script:
     - /analyzer run
 
@@ -65,7 +64,6 @@ bandit-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /bandit/
       exists:
         - '**/*.py'
@@ -106,7 +104,6 @@ flawfinder-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /flawfinder/
       exists:
         - '**/*.c'
@@ -120,7 +117,6 @@ kubesec-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /kubesec/ &&
           $SCAN_KUBERNETES_MANIFESTS == 'true'
 
@@ -132,7 +128,6 @@ gosec-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /gosec/
       exists:
         - '**/*.go'
@@ -145,7 +140,6 @@ nodejs-scan-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/
       exists:
         - 'package.json'
@@ -158,7 +152,6 @@ phpcs-security-audit-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/
       exists:
         - '**/*.php'
@@ -171,7 +164,6 @@ pmd-apex-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/
       exists:
         - '**/*.cls'
@@ -184,7 +176,6 @@ secrets-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /secrets/
 
 security-code-scan-sast:
@@ -195,7 +186,6 @@ security-code-scan-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/
       exists:
         - '**/*.csproj'
@@ -209,7 +199,6 @@ sobelow-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /sobelow/
       exists:
         - 'mix.exs'
@@ -222,7 +211,6 @@ spotbugs-sast:
     - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsast\b/ &&
           $SAST_DEFAULT_ANALYZERS =~ /spotbugs/
       exists:
         - '**/*.groovy'
diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
index 441a57048e1..b897c7b482f 100644
--- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
@@ -7,6 +7,8 @@
 variables:
   SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
   SECRETS_ANALYZER_VERSION: "3"
+  SECRET_DETECTION_EXCLUDED_PATHS: ""
+
 
 .secret-analyzer:
   stage: test
@@ -21,8 +23,7 @@ secret_detection_default_branch:
   rules:
     - if: $SECRET_DETECTION_DISABLED
       when: never
-    - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsecret_detection\b/
+    - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH
   script:
     - /analyzer run
 
@@ -31,8 +32,7 @@ secret_detection:
   rules:
     - if: $SECRET_DETECTION_DISABLED
       when: never
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bsecret_detection\b/
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
   script:
     - git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME
     - export SECRET_DETECTION_COMMIT_TO=$(git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME | tail -n 1)