diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-21 02:50:22 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-21 02:50:22 +0300 |
commit | 9dc93a4519d9d5d7be48ff274127136236a3adb3 (patch) | |
tree | 70467ae3692a0e35e5ea56bcb803eb512a10bedb /lib/gitlab/ci/templates | |
parent | 4b0f34b6d759d6299322b3a54453e930c6121ff0 (diff) |
Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43
Diffstat (limited to 'lib/gitlab/ci/templates')
13 files changed, 603 insertions, 44 deletions
diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 5ebbbf15682..2ff36bcc657 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -113,9 +113,10 @@ promoteBeta: promoteProduction: extends: .promote_job stage: production - # We only allow production promotion on `master` because - # it has its own production scoped secret variables + # We only allow production promotion on the default branch because + # it has its own production scoped secret variables. only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH script: - bundle exec fastlane promote_beta_to_production diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index 15cdbf63cb1..d0c63ab6edf 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -1,27 +1,31 @@ -docker-build-master: - # Official docker image. - image: docker:latest - stage: build - services: - - docker:dind - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - docker build --pull -t "$CI_REGISTRY_IMAGE" . - - docker push "$CI_REGISTRY_IMAGE" - only: - - master - +# Build a Docker image with CI/CD and push to the GitLab registry. +# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html +# +# This template uses one generic job with conditional builds +# for the default branch and all other (MR) branches. docker-build: - # Official docker image. + # Use the official docker image. image: docker:latest stage: build services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) script: - - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . - - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" - except: - - master + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag="" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag=":$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile diff --git a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml new file mode 100644 index 00000000000..90812083917 --- /dev/null +++ b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml @@ -0,0 +1,9 @@ +# This file is a template demonstrating the `script` keyword. +# Learn more about this keyword here: https://docs.gitlab.com/ee/ci/yaml/README.html#script + +# After committing this template, visit CI/CD > Jobs to see the script output. + +job: + script: + # provide a shell script as argument for this keyword. + - echo "Hello World" diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml new file mode 100644 index 00000000000..c7fb1321055 --- /dev/null +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml @@ -0,0 +1,91 @@ +# This template is provided and maintained by Indeni, an official Technology Partner with GitLab. +# See https://about.gitlab.com/partners/technology-partners/#security for more information. + +# For more information about Indeni Cloudrail: https://indeni.com/cloudrail/ +# +# This file shows an example of using Indeni Cloudrail with GitLab CI/CD. +# It is not designed to be included in an existing CI/CD configuration with the "include:" keyword. +# Documentation about this integration: https://indeni.com/doc-indeni-cloudrail/integrate-with-ci-cd/gitlab-instructions +# +# For an example of this used in a GitLab repository, see: https://gitlab.com/indeni/cloudrail-demo/-/blob/master/.gitlab-ci.yml + +# The sast-report output complies with GitLab's format. This report displays Cloudrail's +# results in the Security tab in the pipeline view, if you have that feature enabled +# (GitLab Ultimate only). Otherwise, Cloudrail generates a JUnit report, which displays +# in the "Test summary" in merge requests. + +# Note that Cloudrail's input is the Terraform plan. That is why we've included in this +# template an example of doing that. You are welcome to replace it with your own way +# of generating a Terraform plan. + +# Before you can use this template, get a Cloudrail API key from the Cloudrail web +# user interface. Save it as a CI/CD variable named CLOUDRAIL_API_KEY in your project +# settings. + +variables: + TEST_ROOT: ${CI_PROJECT_DIR}/my_folder_with_terraform_content + +default: + before_script: + - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content + +stages: + - init_and_plan + - cloudrail + +init_and_plan: + stage: init_and_plan + image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.tf' + script: + - terraform init + - terraform plan -out=plan.out + artifacts: + name: "$CI_COMMIT_BRANCH-terraform_plan" + paths: + - ./**/plan.out + - ./**/.terraform + +cloudrail_scan: + stage: cloudrail + image: indeni/cloudrail-cli:1.2.44 + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.tf' + script: + - | + if [[ "${GITLAB_FEATURES}" == *"security_dashboard"* ]]; then + echo "You are licensed for GitLab Security Dashboards. Your scan results will display in the Security Dashboard." + cloudrail run --tf-plan plan.out \ + --directory . \ + --api-key ${CLOUDRAIL_API_KEY} \ + --origin ci \ + --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \ + --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \ + --output-format json-gitlab-sast \ + --output-file ${CI_PROJECT_DIR}/cloudrail-sast-report.json \ + --auto-approve + else + echo "Your scan results will display in the GitLab Test results visualization panel." + cloudrail run --tf-plan plan.out \ + --directory . \ + --api-key ${CLOUDRAIL_API_KEY} \ + --origin ci \ + --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \ + --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \ + --output-format junit \ + --output-file ${CI_PROJECT_DIR}/cloudrail-junit-report.xml \ + --auto-approve + fi + artifacts: + reports: + sast: cloudrail-sast-report.json + junit: cloudrail-junit-report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 5edb26a0b56..01907ef9e2e 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -20,15 +20,48 @@ performance: fi - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS fi - mv sitespeed-results/data/performance.json browser-performance.json artifacts: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 1c25d9d583b..196d42f3e3a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.12-dind + - docker:20.10.6-dind script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index fd6c51ea350..b29342216fc 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -36,6 +36,7 @@ code_quality: REPORT_STDOUT \ REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ + CODECLIMATE_PREFIX \ ) \ --volume "$PWD":/code \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 654a03ced5f..bf42cd52605 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -12,7 +12,7 @@ stages: variables: FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: latest + FUZZAPI_VERSION: "1.6" FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml FUZZAPI_TIMEOUT: 30 FUZZAPI_REPORT: gl-api-fuzzing-report.json @@ -45,7 +45,7 @@ apifuzzer_fuzz: entrypoint: ["/bin/bash", "-l", "-c"] variables: FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:80 + FUZZAPI_API: http://localhost:5000 FUZZAPI_NEW_REPORT: 1 FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log TZ: America/Los_Angeles @@ -107,7 +107,7 @@ apifuzzer_fuzz_dnd: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + FUZZAPI_API: http://apifuzzer:5000 allow_failure: true rules: - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null @@ -142,6 +142,7 @@ apifuzzer_fuzz_dnd: -e TZ=America/Los_Angeles \ -e GITLAB_FEATURES \ -p 80:80 \ + -p 5000:5000 \ -p 8000:8000 \ -p 514:514 \ --restart=no \ @@ -168,7 +169,7 @@ apifuzzer_fuzz_dnd: docker run \ --name worker \ --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_API=http://apifuzzer:5000 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ @@ -211,7 +212,7 @@ apifuzzer_fuzz_dnd: --name worker \ --network $FUZZAPI_D_NETWORK \ -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_API=http://apifuzzer:5000 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ @@ -237,6 +238,7 @@ apifuzzer_fuzz_dnd: -v $CI_PROJECT_DIR:/app \ -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ -p 81:80 \ + -p 5001:5000 \ -p 8001:8000 \ -p 515:514 \ --restart=no \ diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..215029dc952 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -0,0 +1,270 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + FUZZAPI_PROFILE: Quick + FUZZAPI_VERSION: latest + FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml + FUZZAPI_TIMEOUT: 30 + FUZZAPI_REPORT: gl-api-fuzzing-report.json + FUZZAPI_REPORT_ASSET_PATH: assets + # + FUZZAPI_D_NETWORK: testing-net + # + # Wait up to 5 minutes for API Fuzzer and target url to become + # available (non 500 response to HTTP(s)) + FUZZAPI_SERVICE_START_TIMEOUT: "300" + # + FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + # + +apifuzzer_fuzz_unlicensed: + stage: fuzz + allow_failure: true + rules: + - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' + - when: never + script: + - | + echo "Error: Your GitLab project is not licensed for API Fuzzing." + - exit 1 + +apifuzzer_fuzz: + stage: fuzz + image: + name: $FUZZAPI_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://localhost:80 + FUZZAPI_NEW_REPORT: 1 + FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log + TZ: America/Los_Angeles + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE + when: never + - if: $FUZZAPI_D_WORKER_IMAGE + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + script: + # + # Validate options + - | + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & + - APISEC_PID=$! + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # + artifacts: + when: always + paths: + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT + - $FUZZAPI_LOG_SCANNER + reports: + api_fuzzing: $FUZZAPI_REPORT + +apifuzzer_fuzz_dnd: + stage: fuzz + image: docker:19.03.12 + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://apifuzzer:80 + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + services: + - docker:19.03.12-dind + script: + # + # + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # + - docker network create --driver bridge $FUZZAPI_D_NETWORK + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # + # Start peach testing engine container + - | + docker run -d \ + --name apifuzzer \ + --network $FUZZAPI_D_NETWORK \ + -e Proxy:Port=8000 \ + -e TZ=America/Los_Angeles \ + -e GITLAB_FEATURES \ + -p 80:80 \ + -p 8000:8000 \ + -p 514:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + dotnet /peach/Peach.Web.dll + # + # Start target container + - | + if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ + docker run -d \ + --name target \ + --network $FUZZAPI_D_NETWORK \ + $FUZZAPI_D_TARGET_ENV \ + $FUZZAPI_D_TARGET_PORTS \ + $FUZZAPI_D_TARGET_VOLUME \ + --restart=no \ + $FUZZAPI_D_TARGET_IMAGE \ + ; fi + # + # Start worker container if provided + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ + echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ + -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ + $FUZZAPI_D_WORKER_ENV \ + $FUZZAPI_D_WORKER_PORTS \ + $FUZZAPI_D_WORKER_VOLUME \ + --restart=no \ + $FUZZAPI_D_WORKER_IMAGE \ + ; fi + # + # Start API Fuzzing provided worker if no other worker present + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e TZ=America/Los_Angeles \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ + -v $CI_PROJECT_DIR:/app \ + -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ + -p 81:80 \ + -p 8001:8000 \ + -p 515:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + worker-entry \ + ; fi + # + # Propagate exit code from api fuzzing scanner (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + after_script: + # + # Shutdown all containers + - echo "Stopping all containers" + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi + - docker stop worker + - docker stop apifuzzer + # + # Save docker logs + - docker logs apifuzzer &> gl-api_fuzzing-logs.log + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi + - docker logs worker &> gl-api_fuzzing-worker-logs.log + # + artifacts: + when: always + paths: + - ./gl-api_fuzzing*.log + - ./gl-api_fuzzing*.zip + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT + reports: + api_fuzzing: $FUZZAPI_REPORT + +# end diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 64001c2828a..c628e30b2c7 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -6,14 +6,10 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" CS_MAJOR_VERSION: 3 -container_scanning: +.cs_common: stage: test image: "$CS_ANALYZER_IMAGE" variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image - # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes - CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details @@ -21,19 +17,44 @@ container_scanning: # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. - CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION + CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependencies: [] + +container_scanning: + extends: .cs_common + variables: + # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image + # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes + CLAIR_DB_IMAGE_TAG: "latest" + CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" + CS_PROJECT: 'klar' services: - name: $CLAIR_DB_IMAGE alias: clair-vulnerabilities-db script: - /analyzer run + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CS_MAJOR_VERSION =~ /^[0-3]$/ + +container_scanning_new: + extends: .cs_common + variables: + CS_PROJECT: 'container-scanning' + script: + - gtcs scan artifacts: - reports: - container_scanning: gl-container-scanning-report.json - dependencies: [] + paths: [gl-container-scanning-report.json] rules: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CS_MAJOR_VERSION !~ /^[0-3]$/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index fc1acd09714..533f8bb25f8 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -1,3 +1,16 @@ +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST.latest.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST: +# +# stages: +# - build +# - test +# - deploy +# - dast + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ # Configure the scanning tool through the environment variables. @@ -9,6 +22,19 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_PROFILE: Full + DAST_API_VERSION: latest + DAST_API_CONFIG: .gitlab-dast-api.yml + DAST_API_TIMEOUT: 30 + DAST_API_REPORT: gl-dast-api-report.json + DAST_API_REPORT_ASSET_PATH: assets + # + # Wait up to 5 minutes for API Security and target url to become + # available (non 500 response to HTTP(s)) + DAST_API_SERVICE_START_TIMEOUT: "300" + # + DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine dast: stage: dast @@ -25,6 +51,11 @@ dast: reports: dast: gl-dast-report.json rules: + - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION || + $DAST_API_OPENAPI || + $DAST_API_POSTMAN_COLLECTION || + $DAST_API_HAR ) + when: never - if: $DAST_DISABLED when: never - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && @@ -40,4 +71,72 @@ dast: - if: $CI_COMMIT_BRANCH && $DAST_WEBSITE - if: $CI_COMMIT_BRANCH && + $DAST_API_BETA == null && $DAST_API_SPECIFICATION + +dast_api: + stage: dast + image: + name: $DAST_API_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + API_SECURITY_MODE: DAST + DAST_API_NEW_REPORT: 1 + DAST_API_PROJECT: $CI_PROJECT_PATH + DAST_API_API: http://127.0.0.1:5000 + DAST_API_LOG_SCANNER: gl-dast-api-scanner.log + TZ: America/Los_Angeles + allow_failure: true + rules: + - if: $DAST_API_BETA == null + when: never + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && + $DAST_API_SPECIFICATION == null && + $DAST_API_OPENAPI == null && + $DAST_API_POSTMAN_COLLECTION == null && + $DAST_API_HAR == null + when: never + - if: $DAST_API_SPECIFICATION == null && + $DAST_API_OPENAPI == null && + $DAST_API_POSTMAN_COLLECTION == null && + $DAST_API_HAR == null + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdast\b/ + script: + # + # Run user provided pre-script + - sh -c "$DAST_API_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $DAST_API_REPORT_ASSET_PATH + # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER & + - APISEC_PID=$! + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$DAST_API_POST_SCRIPT" + # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # + artifacts: + when: always + paths: + - $DAST_API_REPORT_ASSET_PATH + - $DAST_API_REPORT + - $DAST_API_LOG_SCANNER + - gl-*.log + reports: + dast: $DAST_API_REPORT diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 9693a4fbca2..3ebccfbba4a 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -160,7 +160,7 @@ mobsf-android-sast: services: # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 + - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" @@ -186,7 +186,7 @@ mobsf-ios-sast: services: # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 + - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" @@ -303,6 +303,10 @@ semgrep-sast: $SAST_EXPERIMENTAL_FEATURES == 'true' exists: - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' sobelow-sast: extends: .sast-analyzer @@ -348,3 +352,4 @@ spotbugs-sast: - '**/*.groovy' - '**/*.java' - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index e591e3cc1e2..404d4a4c6db 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -18,9 +18,32 @@ performance: - docker:stable-dind script: - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: |