Add latest changes from gitlab-org/gitlab@master

4 files changed, 282 insertions, 10 deletions

diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb
index 9d1aeb75ccc..93f4bebd41d 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/external.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb
@@ -82,18 +82,18 @@ module Gitlab
             end
 
             def validation_service_timeout
-              timeout = ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i
+              timeout = Gitlab::CurrentSettings.external_pipeline_validation_service_timeout || ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i
               return timeout if timeout > 0
 
               DEFAULT_VALIDATION_REQUEST_TIMEOUT
             end
 
             def validation_service_url
-              ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+              Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL']
             end
 
             def validation_service_token
-              ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN']
+              Gitlab::CurrentSettings.external_pipeline_validation_service_token || ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN']
             end
 
             def validation_service_payload
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
index 654a03ced5f..bf42cd52605 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
@@ -12,7 +12,7 @@ stages:
 
 variables:
     FUZZAPI_PROFILE: Quick
-    FUZZAPI_VERSION: latest
+    FUZZAPI_VERSION: "1.6"
     FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
     FUZZAPI_TIMEOUT: 30
     FUZZAPI_REPORT: gl-api-fuzzing-report.json
@@ -45,7 +45,7 @@ apifuzzer_fuzz:
         entrypoint: ["/bin/bash", "-l", "-c"]
     variables:
         FUZZAPI_PROJECT: $CI_PROJECT_PATH
-        FUZZAPI_API: http://localhost:80
+        FUZZAPI_API: http://localhost:5000
         FUZZAPI_NEW_REPORT: 1
         FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log
         TZ: America/Los_Angeles
@@ -107,7 +107,7 @@ apifuzzer_fuzz_dnd:
         DOCKER_DRIVER: overlay2
         DOCKER_TLS_CERTDIR: ""
         FUZZAPI_PROJECT: $CI_PROJECT_PATH
-        FUZZAPI_API: http://apifuzzer:80
+        FUZZAPI_API: http://apifuzzer:5000
     allow_failure: true
     rules:
         - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
@@ -142,6 +142,7 @@ apifuzzer_fuzz_dnd:
             -e TZ=America/Los_Angeles \
             -e GITLAB_FEATURES \
             -p 80:80 \
+            -p 5000:5000 \
             -p 8000:8000 \
             -p 514:514 \
             --restart=no \
@@ -168,7 +169,7 @@ apifuzzer_fuzz_dnd:
                 docker run \
                     --name worker \
                     --network $FUZZAPI_D_NETWORK \
-                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_API=http://apifuzzer:5000 \
                     -e FUZZAPI_PROJECT \
                     -e FUZZAPI_PROFILE \
                     -e FUZZAPI_CONFIG \
@@ -211,7 +212,7 @@ apifuzzer_fuzz_dnd:
                     --name worker \
                     --network $FUZZAPI_D_NETWORK \
                     -e TZ=America/Los_Angeles \
-                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_API=http://apifuzzer:5000 \
                     -e FUZZAPI_PROJECT \
                     -e FUZZAPI_PROFILE \
                     -e FUZZAPI_CONFIG \
@@ -237,6 +238,7 @@ apifuzzer_fuzz_dnd:
                     -v $CI_PROJECT_DIR:/app \
                     -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \
                     -p 81:80 \
+                    -p 5001:5000 \
                     -p 8001:8000 \
                     -p 515:514 \
                     --restart=no \
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..215029dc952
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
@@ -0,0 +1,270 @@
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/
+
+# Configure the scanning tool through the environment variables.
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables
+# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+
+variables:
+    FUZZAPI_PROFILE: Quick
+    FUZZAPI_VERSION: latest
+    FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
+    FUZZAPI_TIMEOUT: 30
+    FUZZAPI_REPORT: gl-api-fuzzing-report.json
+    FUZZAPI_REPORT_ASSET_PATH: assets
+    #
+    FUZZAPI_D_NETWORK: testing-net
+    #
+    # Wait up to 5 minutes for API Fuzzer and target url to become
+    # available (non 500 response to HTTP(s))
+    FUZZAPI_SERVICE_START_TIMEOUT: "300"
+    #
+    FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine
+    #
+
+apifuzzer_fuzz_unlicensed:
+    stage: fuzz
+    allow_failure: true
+    rules:
+        - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null'
+        - when: never
+    script:
+        - |
+            echo "Error: Your GitLab project is not licensed for API Fuzzing."
+        - exit 1
+
+apifuzzer_fuzz:
+    stage: fuzz
+    image:
+        name: $FUZZAPI_IMAGE
+        entrypoint: ["/bin/bash", "-l", "-c"]
+    variables:
+        FUZZAPI_PROJECT: $CI_PROJECT_PATH
+        FUZZAPI_API: http://localhost:80
+        FUZZAPI_NEW_REPORT: 1
+        FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log
+        TZ: America/Los_Angeles
+    allow_failure: true
+    rules:
+        - if: $FUZZAPI_D_TARGET_IMAGE
+          when: never
+        - if: $FUZZAPI_D_WORKER_IMAGE
+          when: never
+        - if: $API_FUZZING_DISABLED
+          when: never
+        - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+                $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+          when: never
+        - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+    script:
+        #
+        # Validate options
+        - |
+            if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+                echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+                echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+                exit 1; \
+            fi
+        #
+        # Run user provided pre-script
+        - sh -c "$FUZZAPI_PRE_SCRIPT"
+        #
+        # Make sure asset path exists
+        - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
+        #
+        # Start API Security background process
+        - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER &
+        - APISEC_PID=$!
+        #
+        # Start scanning
+        - worker-entry
+        #
+        # Run user provided post-script
+        - sh -c "$FUZZAPI_POST_SCRIPT"
+        #
+        # Shutdown API Security
+        - kill $APISEC_PID
+        - wait $APISEC_PID
+        #
+    artifacts:
+        when: always
+        paths:
+            - $FUZZAPI_REPORT_ASSET_PATH
+            - $FUZZAPI_REPORT
+            - $FUZZAPI_LOG_SCANNER
+        reports:
+            api_fuzzing: $FUZZAPI_REPORT
+
+apifuzzer_fuzz_dnd:
+    stage: fuzz
+    image: docker:19.03.12
+    variables:
+        DOCKER_DRIVER: overlay2
+        DOCKER_TLS_CERTDIR: ""
+        FUZZAPI_PROJECT: $CI_PROJECT_PATH
+        FUZZAPI_API: http://apifuzzer:80
+    allow_failure: true
+    rules:
+        - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
+          when: never
+        - if: $API_FUZZING_DISABLED
+          when: never
+        - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+                $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+          when: never
+        - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+    services:
+        - docker:19.03.12-dind
+    script:
+        #
+        #
+        - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+        #
+        - docker network create --driver bridge $FUZZAPI_D_NETWORK
+        #
+        # Run user provided pre-script
+        - sh -c "$FUZZAPI_PRE_SCRIPT"
+        #
+        # Make sure asset path exists
+        - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
+        #
+        # Start peach testing engine container
+        - |
+            docker run -d \
+            --name apifuzzer \
+            --network $FUZZAPI_D_NETWORK \
+            -e Proxy:Port=8000 \
+            -e TZ=America/Los_Angeles \
+            -e GITLAB_FEATURES \
+            -p 80:80 \
+            -p 8000:8000 \
+            -p 514:514 \
+            --restart=no \
+            $FUZZAPI_IMAGE \
+            dotnet /peach/Peach.Web.dll
+        #
+        # Start target container
+        - |
+            if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \
+                docker run -d \
+                    --name target \
+                    --network $FUZZAPI_D_NETWORK \
+                    $FUZZAPI_D_TARGET_ENV \
+                    $FUZZAPI_D_TARGET_PORTS \
+                    $FUZZAPI_D_TARGET_VOLUME \
+                    --restart=no \
+                    $FUZZAPI_D_TARGET_IMAGE \
+                ; fi
+        #
+        # Start worker container if provided
+        - |
+            if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \
+                echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \
+                docker run \
+                    --name worker \
+                    --network $FUZZAPI_D_NETWORK \
+                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_PROJECT \
+                    -e FUZZAPI_PROFILE \
+                    -e FUZZAPI_CONFIG \
+                    -e FUZZAPI_REPORT \
+                    -e FUZZAPI_REPORT_ASSET_PATH \
+                    -e FUZZAPI_NEW_REPORT=1 \
+                    -e FUZZAPI_HAR \
+                    -e FUZZAPI_OPENAPI \
+                    -e FUZZAPI_POSTMAN_COLLECTION \
+                    -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
+                    -e FUZZAPI_TARGET_URL \
+                    -e FUZZAPI_OVERRIDES_FILE \
+                    -e FUZZAPI_OVERRIDES_ENV \
+                    -e FUZZAPI_OVERRIDES_CMD \
+                    -e FUZZAPI_OVERRIDES_INTERVAL \
+                    -e FUZZAPI_TIMEOUT \
+                    -e FUZZAPI_VERBOSE \
+                    -e FUZZAPI_SERVICE_START_TIMEOUT \
+                    -e FUZZAPI_HTTP_USERNAME \
+                    -e FUZZAPI_HTTP_PASSWORD \
+                    -e CI_PROJECT_URL \
+                    -e CI_JOB_ID \
+                    -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \
+                    $FUZZAPI_D_WORKER_ENV \
+                    $FUZZAPI_D_WORKER_PORTS \
+                    $FUZZAPI_D_WORKER_VOLUME \
+                    --restart=no \
+                    $FUZZAPI_D_WORKER_IMAGE \
+                ; fi
+        #
+        # Start API Fuzzing provided worker if no other worker present
+        - |
+            if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \
+                if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+                    echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+                    echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+                    exit 1; \
+                fi; \
+                docker run \
+                    --name worker \
+                    --network $FUZZAPI_D_NETWORK \
+                    -e TZ=America/Los_Angeles \
+                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_PROJECT \
+                    -e FUZZAPI_PROFILE \
+                    -e FUZZAPI_CONFIG \
+                    -e FUZZAPI_REPORT \
+                    -e FUZZAPI_REPORT_ASSET_PATH \
+                    -e FUZZAPI_NEW_REPORT=1 \
+                    -e FUZZAPI_HAR \
+                    -e FUZZAPI_OPENAPI \
+                    -e FUZZAPI_POSTMAN_COLLECTION \
+                    -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
+                    -e FUZZAPI_TARGET_URL \
+                    -e FUZZAPI_OVERRIDES_FILE \
+                    -e FUZZAPI_OVERRIDES_ENV \
+                    -e FUZZAPI_OVERRIDES_CMD \
+                    -e FUZZAPI_OVERRIDES_INTERVAL \
+                    -e FUZZAPI_TIMEOUT \
+                    -e FUZZAPI_VERBOSE \
+                    -e FUZZAPI_SERVICE_START_TIMEOUT \
+                    -e FUZZAPI_HTTP_USERNAME \
+                    -e FUZZAPI_HTTP_PASSWORD \
+                    -e CI_PROJECT_URL \
+                    -e CI_JOB_ID \
+                    -v $CI_PROJECT_DIR:/app \
+                    -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \
+                    -p 81:80 \
+                    -p 8001:8000 \
+                    -p 515:514 \
+                    --restart=no \
+                    $FUZZAPI_IMAGE \
+                    worker-entry \
+                ; fi
+        #
+        # Propagate exit code from api fuzzing scanner (if any)
+        - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi
+        #
+        # Run user provided post-script
+        - sh -c "$FUZZAPI_POST_SCRIPT"
+        #
+    after_script:
+        #
+        # Shutdown all containers
+        - echo "Stopping all containers"
+        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi
+        - docker stop worker
+        - docker stop apifuzzer
+        #
+        # Save docker logs
+        - docker logs apifuzzer &> gl-api_fuzzing-logs.log
+        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi
+        - docker logs worker &> gl-api_fuzzing-worker-logs.log
+        #
+    artifacts:
+        when: always
+        paths:
+            - ./gl-api_fuzzing*.log
+            - ./gl-api_fuzzing*.zip
+            - $FUZZAPI_REPORT_ASSET_PATH
+            - $FUZZAPI_REPORT
+        reports:
+            api_fuzzing: $FUZZAPI_REPORT
+
+# end
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 90dc80a3fc0..a1f0d2e5a48 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -160,7 +160,7 @@ mobsf-android-sast:
   services:
     # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile
     # Unfortunately, we need to keep track of mobsf version in 2 different places for now.
-    - name: opensecurity/mobile-security-framework-mobsf:v3.3.3
+    - name: opensecurity/mobile-security-framework-mobsf:v3.4.0
       alias: mobsf
   image:
     name: "$SAST_ANALYZER_IMAGE"
@@ -186,7 +186,7 @@ mobsf-ios-sast:
   services:
     # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile
     # Unfortunately, we need to keep track of mobsf version in 2 different places for now.
-    - name: opensecurity/mobile-security-framework-mobsf:v3.3.3
+    - name: opensecurity/mobile-security-framework-mobsf:v3.4.0
       alias: mobsf
   image:
     name: "$SAST_ANALYZER_IMAGE"