diff options
Diffstat (limited to 'lib/gitlab/ci')
6 files changed, 380 insertions, 20 deletions
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 6077d70a080..2d2032b1d8c 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -120,7 +120,7 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - job_variables: variables_value.to_h, + job_variables: variables_entry.value_with_data, root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..70f85382967 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,244 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ +# +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + DS_EXCLUDED_ANALYZERS: "" + DS_EXCLUDED_PATHS: "spec, test, tests, tmp" + DS_MAJOR_VERSION: 3 + +dependency_scanning: + stage: test + script: + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json + dependencies: [] + rules: + - when: never + +.ds-analyzer: + extends: dependency_scanning + allow_failure: true + variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/$DS_ANALYZER_NAME:$DS_MAJOR_VERSION" + # DS_ANALYZER_NAME is an undocumented variable used in job definitions + # to inject the analyzer name in the image name. + DS_ANALYZER_NAME: "" + image: + name: "$DS_ANALYZER_IMAGE$DS_IMAGE_SUFFIX" + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + script: + - /analyzer run + +.cyclonedx-reports: + artifacts: + paths: + - "**/gl-sbom-*.cdx.json" + +.gemnasium-shared-rule: + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' + +gemnasium-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium" + GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-shared-rule, exists] + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-shared-rule, exists] + +.gemnasium-maven-shared-rule: + exists: + - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' + - '{build.sbt,*/build.sbt,*/*/build.sbt}' + - '{pom.xml,*/pom.xml,*/*/pom.xml}' + +gemnasium-maven-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium-maven" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-maven-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-maven-shared-rule, exists] + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-maven-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-maven-shared-rule, exists] + +.gemnasium-python-shared-rule: + exists: + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' + +gemnasium-python-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium-python" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-python-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-python-shared-rule, exists] + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-python-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-python-shared-rule, exists] + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE + +bundler-audit-dependency_scanning: + extends: .ds-analyzer + variables: + DS_ANALYZER_NAME: "bundler-audit" + DS_MAJOR_VERSION: 2 + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/347491" + - exit 1 + rules: + - when: never + +retire-js-dependency_scanning: + extends: .ds-analyzer + variables: + DS_ANALYZER_NAME: "retire.js" + DS_MAJOR_VERSION: 2 + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/289830" + - exit 1 + rules: + - when: never diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..e47f669c2e2 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,48 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml + +# Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html +# +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + + LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. + LICENSE_MANAGEMENT_VERSION: 4 + +license_scanning: + stage: test + image: + name: "$SECURE_ANALYZERS_PREFIX/license-finder:$LICENSE_MANAGEMENT_VERSION" + entrypoint: [""] + variables: + LM_REPORT_VERSION: '2.1' + SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD + allow_failure: true + script: + - /run.sh analyze . + artifacts: + reports: + license_scanning: gl-license-scanning-report.json + dependencies: [] + rules: + - if: $LICENSE_MANAGEMENT_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\blicense_scanning\b/ + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\blicense_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f7b1d12b3b3 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,68 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# CS_DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables + +variables: + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + +container_scanning: + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" + stage: test + variables: + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting + GIT_STRATEGY: none + allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + dependencies: [] + script: + - gtcs scan + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb index a77ca71b9cf..16e3afd8620 100644 --- a/lib/gitlab/ci/variables/helpers.rb +++ b/lib/gitlab/ci/variables/helpers.rb @@ -6,24 +6,24 @@ module Gitlab module Helpers class << self def merge_variables(current_vars, new_vars) - current_vars = transform_from_yaml_variables(current_vars) - new_vars = transform_from_yaml_variables(new_vars) + return current_vars if new_vars.blank? - transform_to_yaml_variables( - current_vars.merge(new_vars) - ) - end + current_vars = transform_to_array(current_vars) if current_vars.is_a?(Hash) + new_vars = transform_to_array(new_vars) if new_vars.is_a?(Hash) - def transform_to_yaml_variables(vars) - vars.to_h.map do |key, value| - { key: key.to_s, value: value } - end + (new_vars + current_vars).uniq { |var| var[:key] } end - def transform_from_yaml_variables(vars) - return vars.stringify_keys.transform_values(&:to_s) if vars.is_a?(Hash) + def transform_to_array(vars) + return [] if vars.blank? - vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] } + vars.map do |key, data| + if data.is_a?(Hash) + { key: key.to_s, **data.except(:key) } + else + { key: key.to_s, value: data } + end + end end def inherit_yaml_variables(from:, to:, inheritance:) @@ -35,7 +35,7 @@ module Gitlab def apply_inheritance(variables, inheritance) case inheritance when true then variables - when false then {} + when false then [] when Array then variables.select { |var| inheritance.include?(var[:key]) } end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 4bd1ac3b67f..f203f88442d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -43,7 +43,7 @@ module Gitlab end def root_variables - @root_variables ||= transform_to_yaml_variables(variables) + @root_variables ||= transform_to_array(variables) end def jobs @@ -70,7 +70,7 @@ module Gitlab environment: job[:environment_name], coverage_regex: job[:coverage], # yaml_variables is calculated with using job_variables in Seed::Build - job_variables: transform_to_yaml_variables(job[:job_variables]), + job_variables: transform_to_array(job[:job_variables]), root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], @@ -114,7 +114,7 @@ module Gitlab Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: root_variables, - to: transform_to_yaml_variables(job[:job_variables]), + to: job[:job_variables], inheritance: job.fetch(:root_variables_inheritance, true) ) end @@ -137,8 +137,8 @@ module Gitlab job[:release] end - def transform_to_yaml_variables(variables) - ::Gitlab::Ci::Variables::Helpers.transform_to_yaml_variables(variables) + def transform_to_array(variables) + ::Gitlab::Ci::Variables::Helpers.transform_to_array(variables) end end end |