From 9f46488805e86b1bc341ea1620b866016c2ce5ed Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 20 May 2020 14:34:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-0-stable-ee --- lib/gitlab/ci/ansi2html.rb | 2 +- lib/gitlab/ci/ansi2json/state.rb | 2 +- lib/gitlab/ci/build/artifacts/metadata.rb | 4 +- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 2 +- lib/gitlab/ci/config/entry/artifacts.rb | 10 +- lib/gitlab/ci/config/entry/reports.rb | 4 +- lib/gitlab/ci/config/entry/trigger.rb | 9 +- lib/gitlab/ci/cron_parser.rb | 11 +- lib/gitlab/ci/features.rb | 18 ++ lib/gitlab/ci/parsers.rb | 8 +- lib/gitlab/ci/parsers/accessibility/pa11y.rb | 33 +++ lib/gitlab/ci/parsers/terraform/tfplan.rb | 35 +++ lib/gitlab/ci/parsers/test/junit.rb | 8 +- lib/gitlab/ci/pipeline/chain/command.rb | 15 ++ lib/gitlab/ci/pipeline/chain/sequence.rb | 3 + .../ci/pipeline/seed/build/resource_group.rb | 1 - lib/gitlab/ci/reports/accessibility_reports.rb | 46 ++++ .../ci/reports/accessibility_reports_comparer.rb | 55 +++++ lib/gitlab/ci/reports/terraform_reports.rb | 27 +++ lib/gitlab/ci/reports/test_reports.rb | 6 + lib/gitlab/ci/reports/test_suite.rb | 19 +- lib/gitlab/ci/status/build/failed.rb | 4 +- lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml | 3 +- lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml | 31 +-- .../Jobs/Browser-Performance-Testing.gitlab-ci.yml | 14 +- lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml | 5 +- .../ci/templates/Jobs/Code-Quality.gitlab-ci.yml | 11 +- .../Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml | 44 ++-- lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 154 ++++++------- .../ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml | 30 +++ lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml | 16 +- lib/gitlab/ci/templates/Scala.gitlab-ci.yml | 8 +- .../Security/Container-Scanning.gitlab-ci.yml | 21 +- .../ci/templates/Security/DAST.gitlab-ci.yml | 26 ++- .../Security/Dependency-Scanning.gitlab-ci.yml | 120 ++++++---- .../Security/License-Management.gitlab-ci.yml | 34 +-- .../Security/License-Scanning.gitlab-ci.yml | 25 ++- .../ci/templates/Security/SAST.gitlab-ci.yml | 214 +++++++++++------- .../Security/Secure-Binaries.gitlab-ci.yml | 246 +++++++++++++++++++++ lib/gitlab/ci/templates/Terraform.gitlab-ci.yml | 2 +- .../templates/Verify/Accessibility.gitlab-ci.yml | 4 +- .../Workflows/Branch-Pipelines.gitlab-ci.yml | 7 + .../Workflows/MergeRequest-Pipelines.gitlab-ci.yml | 8 + lib/gitlab/ci/yaml_processor.rb | 2 +- 44 files changed, 971 insertions(+), 376 deletions(-) create mode 100644 lib/gitlab/ci/features.rb create mode 100644 lib/gitlab/ci/parsers/accessibility/pa11y.rb create mode 100644 lib/gitlab/ci/parsers/terraform/tfplan.rb create mode 100644 lib/gitlab/ci/reports/accessibility_reports.rb create mode 100644 lib/gitlab/ci/reports/accessibility_reports_comparer.rb create mode 100644 lib/gitlab/ci/reports/terraform_reports.rb create mode 100644 lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml create mode 100644 lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml create mode 100644 lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml create mode 100644 lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml (limited to 'lib/gitlab/ci') diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 3a05feee156..e145bd2e9df 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -353,7 +353,7 @@ module Gitlab def restore_state(new_state, stream) state = Base64.urlsafe_decode64(new_state) - state = JSON.parse(state, symbolize_names: true) + state = Gitlab::Json.parse(state, symbolize_names: true) return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 7e1a8102a35..38d36e6950c 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -90,7 +90,7 @@ module Gitlab decoded_state = Base64.urlsafe_decode64(state) return unless decoded_state.present? - JSON.parse(decoded_state) + Gitlab::Json.parse(decoded_state) end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 1c3ce08be76..c5afb16ab1a 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -32,7 +32,7 @@ module Gitlab raise ParserError, 'Errors field not found!' unless errors begin - JSON.parse(errors) + Gitlab::Json.parse(errors) rescue JSON::ParserError raise ParserError, 'Invalid errors field!' end @@ -71,7 +71,7 @@ module Gitlab next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN - entries[path] = JSON.parse(meta, symbolize_names: true) + entries[path] = Gitlab::Json.parse(meta, symbolize_names: true) rescue JSON::ParserError, Encoding::CompatibilityError next end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 80e69cdcc95..ef354832e8e 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -50,7 +50,7 @@ module Gitlab end def basename - (directory? && !blank_node?) ? name + '/' : name + directory? && !blank_node? ? name + '/' : name end def name diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 241c73db3bb..a9a9636637f 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as].freeze + ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude].freeze EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" @@ -35,6 +35,8 @@ module Gitlab }, if: :expose_as_present? validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present? validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present? + validates :exclude, array_of_strings: true, if: :exclude_enabled? + validates :exclude, absence: { message: 'feature is disabled' }, unless: :exclude_enabled? validates :reports, type: Hash validates :when, inclusion: { in: %w[on_success on_failure always], @@ -50,8 +52,6 @@ module Gitlab end def expose_as_present? - return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true) - # This duplicates the `validates :config, type: Hash` above, # but Validatable currently doesn't halt the validation # chain if it encounters a validation error. @@ -59,6 +59,10 @@ module Gitlab !@config[:expose_as].nil? end + + def exclude_enabled? + ::Gitlab::Ci::Features.artifacts_exclude_enabled? + end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 8ccee3b5b2b..1a871e043a6 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif - dotenv cobertura terraform].freeze + dotenv cobertura terraform accessibility cluster_applications].freeze attributes ALLOWED_KEYS @@ -37,6 +37,8 @@ module Gitlab validates :dotenv, array_of_strings_or_string: true validates :cobertura, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true + validates :accessibility, array_of_strings_or_string: true + validates :cluster_applications, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index 7202784842a..c6ba53adfd7 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -25,8 +25,7 @@ module Gitlab strategy :CrossProjectTrigger, if: -> (config) { !config.key?(:include) } strategy :SameProjectTrigger, if: -> (config) do - ::Feature.enabled?(:ci_parent_child_pipeline, default_enabled: true) && - config.key?(:include) + config.key?(:include) end class CrossProjectTrigger < ::Gitlab::Config::Entry::Node @@ -72,11 +71,7 @@ module Gitlab class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors - if ::Feature.enabled?(:ci_parent_child_pipeline, default_enabled: true) - ['config must specify either project or include'] - else - ['config must specify project'] - end + ['config must specify either project or include'] end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 1d7e7ea0f9a..efd48a9b29f 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -12,8 +12,11 @@ module Gitlab end def next_time_from(time) - @cron_line ||= try_parse_cron(@cron, @cron_timezone) - @cron_line.next_time(time).utc.in_time_zone(Time.zone) if @cron_line.present? + cron_line.next_time(time).utc.in_time_zone(Time.zone) if cron_line.present? + end + + def previous_time_from(time) + cron_line.previous_time(time).utc.in_time_zone(Time.zone) if cron_line.present? end def cron_valid? @@ -49,6 +52,10 @@ module Gitlab def try_parse_cron(cron, cron_timezone) Fugit::Cron.parse("#{cron} #{cron_timezone}") end + + def cron_line + @cron_line ||= try_parse_cron(@cron, @cron_timezone) + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb new file mode 100644 index 00000000000..48f3d4fdd2f --- /dev/null +++ b/lib/gitlab/ci/features.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + ## + # Ci::Features is a class that aggregates all CI/CD feature flags in one place. + # + module Features + def self.artifacts_exclude_enabled? + ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: false) + end + + def self.ensure_scheduling_type_enabled? + ::Feature.enabled?(:ci_ensure_scheduling_type, default_enabled: true) + end + end + end +end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index a44105d53c2..0e44475607b 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -3,14 +3,14 @@ module Gitlab module Ci module Parsers - prepend_if_ee('::EE::Gitlab::Ci::Parsers') # rubocop: disable Cop/InjectEnterpriseEditionModule - ParserNotFoundError = Class.new(ParserError) def self.parsers { junit: ::Gitlab::Ci::Parsers::Test::Junit, - cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura + cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura, + terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan, + accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y } end @@ -22,3 +22,5 @@ module Gitlab end end end + +Gitlab::Ci::Parsers.prepend_if_ee('::EE::Gitlab::Ci::Parsers') diff --git a/lib/gitlab/ci/parsers/accessibility/pa11y.rb b/lib/gitlab/ci/parsers/accessibility/pa11y.rb new file mode 100644 index 00000000000..953b5a91258 --- /dev/null +++ b/lib/gitlab/ci/parsers/accessibility/pa11y.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Accessibility + class Pa11y + def parse!(json_data, accessibility_report) + root = Gitlab::Json.parse(json_data).with_indifferent_access + + parse_all(root, accessibility_report) + rescue JSON::ParserError => e + accessibility_report.set_error_message("JSON parsing failed: #{e}") + rescue StandardError => e + accessibility_report.set_error_message("Pa11y parsing failed: #{e}") + end + + private + + def parse_all(root, accessibility_report) + return unless root.present? + + root.dig("results").each do |url, value| + accessibility_report.add_url(url, value) + end + + accessibility_report + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb new file mode 100644 index 00000000000..26a18c6603e --- /dev/null +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Terraform + class Tfplan + TfplanParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + + def parse!(json_data, terraform_reports, artifact:) + tfplan = Gitlab::Json.parse(json_data).tap do |parsed_data| + parsed_data['job_path'] = Gitlab::Routing.url_helpers.project_job_path( + artifact.job.project, artifact.job + ) + end + + raise TfplanParserError, 'Tfplan missing required key' unless valid_supported_keys?(tfplan) + + terraform_reports.add_plan(artifact.filename, tfplan) + rescue JSON::ParserError + raise TfplanParserError, 'JSON parsing failed' + rescue + raise TfplanParserError, 'Tfplan parsing failed' + end + + private + + def valid_supported_keys?(tfplan) + tfplan.keys == %w[create update delete job_path] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 33140b4c7fd..5746f38ae5b 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -15,10 +15,10 @@ module Gitlab test_case = create_test_case(test_case, args) test_suite.add_test_case(test_case) end - rescue Nokogiri::XML::SyntaxError - raise JunitParserError, "XML parsing failed" - rescue - raise JunitParserError, "JUnit parsing failed" + rescue Nokogiri::XML::SyntaxError => e + test_suite.set_suite_error("JUnit XML parsing failed: #{e}") + rescue StandardError => e + test_suite.set_suite_error("JUnit data parsing failed: #{e}") end private diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index fa46114615c..73187401903 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -76,6 +76,21 @@ module Gitlab def parent_pipeline bridge&.parent_pipeline end + + def duration_histogram + strong_memoize(:duration_histogram) do + name = :gitlab_ci_pipeline_creation_duration_seconds + comment = 'Pipeline creation duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def observe_creation_duration(duration) + duration_histogram.observe({}, duration.seconds) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 99780409085..a7c671e76d3 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -10,6 +10,7 @@ module Gitlab @command = command @sequence = sequence @completed = [] + @start = Time.now end def build! @@ -24,6 +25,8 @@ module Gitlab @pipeline.tap do yield @pipeline, self if block_given? + + @command.observe_creation_duration(Time.now - @start) end end diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb index 3bec6d1e8b6..c0641d9ff0a 100644 --- a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb @@ -16,7 +16,6 @@ module Gitlab end def to_resource - return unless Feature.enabled?(:ci_resource_group, build.project, default_enabled: true) return unless resource_group_key.present? resource_group = build.project.resource_groups diff --git a/lib/gitlab/ci/reports/accessibility_reports.rb b/lib/gitlab/ci/reports/accessibility_reports.rb new file mode 100644 index 00000000000..1901ba3b102 --- /dev/null +++ b/lib/gitlab/ci/reports/accessibility_reports.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class AccessibilityReports + attr_reader :urls, :error_message + + def initialize + @urls = {} + @error_message = nil + end + + def add_url(url, data) + if url.empty? + set_error_message("Empty URL detected in gl-accessibility.json") + else + urls[url] = data + end + end + + def scans_count + @urls.size + end + + def passes_count + @urls.count { |url, errors| errors.empty? } + end + + # rubocop: disable CodeReuse/ActiveRecord + def errors_count + @urls.sum { |url, errors| errors.size } + end + # rubocop: enable CodeReuse/ActiveRecord + + def set_error_message(error) + @error_message = error + end + + def all_errors + @urls.values.flatten + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb new file mode 100644 index 00000000000..fa6337166d5 --- /dev/null +++ b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class AccessibilityReportsComparer + include Gitlab::Utils::StrongMemoize + + STATUS_SUCCESS = 'success' + STATUS_FAILED = 'failed' + + attr_reader :base_reports, :head_reports + + def initialize(base_reports, head_reports) + @base_reports = base_reports || AccessibilityReports.new + @head_reports = head_reports + end + + def status + head_reports.errors_count.positive? ? STATUS_FAILED : STATUS_SUCCESS + end + + def existing_errors + strong_memoize(:existing_errors) do + base_reports.all_errors + end + end + + def new_errors + strong_memoize(:new_errors) do + head_reports.all_errors - base_reports.all_errors + end + end + + def resolved_errors + strong_memoize(:resolved_errors) do + base_reports.all_errors - head_reports.all_errors + end + end + + def errors_count + head_reports.errors_count + end + + def resolved_count + resolved_errors.size + end + + def total_count + existing_errors.size + new_errors.size + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/terraform_reports.rb b/lib/gitlab/ci/reports/terraform_reports.rb new file mode 100644 index 00000000000..f955d007daf --- /dev/null +++ b/lib/gitlab/ci/reports/terraform_reports.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TerraformReports + attr_reader :plans + + def initialize + @plans = {} + end + + def pick(keys) + terraform_plans = plans.select do |key| + keys.include?(key) + end + + { plans: terraform_plans } + end + + def add_plan(name, plan) + plans[name] = plan + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb index 72323c4343d..86ba725c71e 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -42,6 +42,12 @@ module Gitlab self end + def suite_errors + test_suites.each_with_object({}) do |(name, suite), errors| + errors[suite.name] = suite.suite_error if suite.suite_error + end + end + TestCase::STATUS_TYPES.each do |status_type| define_method("#{status_type}_count") do # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index cf43c5313c0..8bbf2e0f6cf 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -7,6 +7,7 @@ module Gitlab attr_reader :name attr_reader :test_cases attr_reader :total_time + attr_reader :suite_error def initialize(name = nil) @name = name @@ -25,12 +26,16 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def total_count + return 0 if suite_error + test_cases.values.sum(&:count) end # rubocop: enable CodeReuse/ActiveRecord def total_status - if failed_count > 0 || error_count > 0 + if suite_error + TestCase::STATUS_ERROR + elsif failed_count > 0 || error_count > 0 TestCase::STATUS_FAILED else TestCase::STATUS_SUCCESS @@ -49,14 +54,22 @@ module Gitlab TestCase::STATUS_TYPES.each do |status_type| define_method("#{status_type}") do - test_cases[status_type] || {} + return {} if suite_error || test_cases[status_type].nil? + + test_cases[status_type] end define_method("#{status_type}_count") do - test_cases[status_type]&.length.to_i + return 0 if suite_error || test_cases[status_type].nil? + + test_cases[status_type].length end end + def set_suite_error(msg) + @suite_error = msg + end + private def existing_key?(test_case) diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index b0b01538a30..76ad113aad9 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -29,8 +29,6 @@ module Gitlab private_constant :REASONS - prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed') # rubocop: disable Cop/InjectEnterpriseEditionModule - def status_tooltip base_message end @@ -65,3 +63,5 @@ module Gitlab end end end + +Gitlab::Ci::Status::Build::Failed.prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed') diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index a9f29bda9b9..5017037fb5a 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -48,7 +48,6 @@ variables: POSTGRES_PASSWORD: testing-password POSTGRES_ENABLED: "true" POSTGRES_DB: $CI_ENVIRONMENT_SLUG - POSTGRES_VERSION: 9.6.2 DOCKER_DRIVER: overlay2 @@ -159,5 +158,5 @@ include: - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml index a41b399032f..82b2f5c035e 100644 --- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml @@ -5,32 +5,9 @@ stages: - deploy - production +variables: + AUTO_DEVOPS_PLATFORM_TARGET: ECS + include: - template: Jobs/Build.gitlab-ci.yml - -.deploy_to_ecs: - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest - script: - - ecs update-task-definition - -review: - extends: .deploy_to_ecs - stage: review - environment: - name: review/$CI_COMMIT_REF_NAME - only: - refs: - - branches - - tags - except: - refs: - - master - -production: - extends: .deploy_to_ecs - stage: production - environment: - name: production - only: - refs: - - master + - template: Jobs/Deploy/ECS.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index d85078c0a40..adbf9731e43 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -30,11 +30,9 @@ performance: paths: - performance.json - sitespeed-results/ - only: - refs: - - branches - - tags - kubernetes: active - except: - variables: - - $PERFORMANCE_DISABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$PERFORMANCE_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 3949b87bbda..787f07521e0 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -15,6 +15,5 @@ build: export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} fi - /build/build.sh - only: - - branches - - tags + rules: + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 9c4699f1f44..24e75c56a75 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -26,10 +26,7 @@ code_quality: codequality: gl-code-quality-report.json expire_in: 1 week dependencies: [] - only: - refs: - - branches - - tags - except: - variables: - - $CODE_QUALITY_DISABLED + rules: + - if: '$CODE_QUALITY_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 3cf4910fe86..5174aed04ba 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.10.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0" dast_environment_deploy: extends: .dast-auto-deploy @@ -18,17 +18,16 @@ dast_environment_deploy: on_stop: stop_dast_environment artifacts: paths: [environment_url.txt] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ - kubernetes: active - except: - variables: - - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - - $DAST_WEBSITE # we don't need to create a review app if a URL is already given + rules: + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + when: never + - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + when: never + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + when: never + - if: $CI_COMMIT_BRANCH && + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ stop_dast_environment: extends: .dast-auto-deploy @@ -42,14 +41,13 @@ stop_dast_environment: name: dast-default action: stop needs: ["dast"] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ - kubernetes: active - except: - variables: - - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - - $DAST_WEBSITE + rules: + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + when: never + - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + when: never + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + when: never + - if: $CI_COMMIT_BRANCH && + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 9bf0d31409a..b4e5a41a34d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,8 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.13.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0" + +include: + - template: Jobs/Deploy/ECS.gitlab-ci.yml review: extends: .auto-deploy @@ -18,16 +21,14 @@ review: on_stop: stop_review artifacts: paths: [environment_url.txt] - only: - refs: - - branches - - tags - kubernetes: active - except: - refs: - - master - variables: - - $REVIEW_DISABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' stop_review: extends: .auto-deploy @@ -41,18 +42,16 @@ stop_review: name: review/$CI_COMMIT_REF_NAME action: stop dependencies: [] - when: manual allow_failure: true - only: - refs: - - branches - - tags - kubernetes: active - except: - refs: - - master - variables: - - $REVIEW_DISABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + when: manual # Staging deploys are disabled by default since # continuous deployment to production is enabled by default @@ -73,12 +72,12 @@ staging: environment: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN - only: - refs: - - master - kubernetes: active - variables: - - $STAGING_ENABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$STAGING_ENABLED' # Canaries are disabled by default, but if you want them, # and know what the downsides are, you can enable this by setting @@ -97,13 +96,13 @@ canary: environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN - when: manual - only: - refs: - - master - kubernetes: active - variables: - - $CANARY_ENABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$CANARY_ENABLED' + when: manual .production: &production_template extends: .auto-deploy @@ -126,32 +125,33 @@ canary: production: <<: *production_template - only: - refs: - - master - kubernetes: active - except: - variables: - - $STAGING_ENABLED - - $CANARY_ENABLED - - $INCREMENTAL_ROLLOUT_ENABLED - - $INCREMENTAL_ROLLOUT_MODE + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$STAGING_ENABLED' + when: never + - if: '$CANARY_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' production_manual: <<: *production_template - when: manual allow_failure: false - only: - refs: - - master - kubernetes: active - variables: - - $STAGING_ENABLED - - $CANARY_ENABLED - except: - variables: - - $INCREMENTAL_ROLLOUT_ENABLED - - $INCREMENTAL_ROLLOUT_MODE + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE' + when: never + - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + when: manual + - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + when: manual # This job implements incremental rollout on for every push to `master`. @@ -176,29 +176,29 @@ production_manual: .manual_rollout_template: &manual_rollout_template <<: *rollout_template stage: production - when: manual - # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4) - only: - refs: - - master - kubernetes: active - variables: - - $INCREMENTAL_ROLLOUT_MODE == "manual" - - $INCREMENTAL_ROLLOUT_ENABLED - except: - variables: - - $INCREMENTAL_ROLLOUT_MODE == "timed" + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax + - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' + when: manual .timed_rollout_template: &timed_rollout_template <<: *rollout_template - when: delayed - start_in: 5 minutes - only: - refs: - - master - kubernetes: active - variables: - - $INCREMENTAL_ROLLOUT_MODE == "timed" + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' + when: delayed + start_in: 5 minutes timed rollout 10%: <<: *timed_rollout_template diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml new file mode 100644 index 00000000000..642f0ebeaf7 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -0,0 +1,30 @@ +.deploy_to_ecs: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + script: + - ecs update-task-definition + +review_ecs: + extends: .deploy_to_ecs + stage: review + environment: + name: review/$CI_COMMIT_REF_NAME + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + +production_ecs: + extends: .deploy_to_ecs + stage: production + environment: + name: production + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' diff --git a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml index a0ddd273552..3b87d53f165 100644 --- a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml @@ -1,10 +1,12 @@ test: - services: - - "postgres:${POSTGRES_VERSION}" variables: + POSTGRES_VERSION: 9.6.16 POSTGRES_DB: test + services: + - "postgres:${POSTGRES_VERSION}" stage: test image: gliderlabs/herokuish:latest + needs: [] script: - | if [ -z ${KUBERNETES_PORT+x} ]; then @@ -15,9 +17,7 @@ test: - export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}" - cp -R . /tmp/app - /bin/herokuish buildpack test - only: - - branches - - tags - except: - variables: - - $TEST_DISABLED + rules: + - if: '$TEST_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index b4208ed9d7d..e081e20564a 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -1,7 +1,7 @@ -# Official Java image. Look for the different tagged releases at -# https://hub.docker.com/r/library/java/tags/ . A Java image is not required +# Official OpenJDK Java image. Look for the different tagged releases at +# https://hub.docker.com/_/openjdk/ . A Java image is not required # but an image with a JVM speeds up the build a bit. -image: java:8 +image: openjdk:8 before_script: # Enable the usage of sources over https @@ -14,7 +14,7 @@ before_script: - apt-get update -yqq - apt-get install sbt -yqq # Log the sbt version - - sbt sbt-version + - sbt sbtVersion test: script: diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 6efb6b4e273..21bcdd8d9b5 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,16 +1,20 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + CS_MAJOR_VERSION: 2 container_scanning: stage: test - image: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION + image: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "arminc/clair-db:$CLAIR_DB_IMAGE_TAG" + CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details @@ -25,11 +29,8 @@ container_scanning: reports: container_scanning: gl-container-scanning-report.json dependencies: [] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ - except: - variables: - - $CONTAINER_SCANNING_DISABLED + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 0e3d7660bdf..07399216597 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -12,11 +12,14 @@ stages: variables: DAST_VERSION: 1 + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" dast: stage: dast image: - name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" variables: GIT_STRATEGY: none allow_failure: true @@ -27,12 +30,15 @@ dast: artifacts: reports: dast: gl-dast-report.json - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ - except: - variables: - - $DAST_DISABLED - - $DAST_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + rules: + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && $DAST_WEBSITE == null && + $DAST_API_SPECIFICATION == null + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 0ecf37b37a3..616966b4f04 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -5,11 +5,16 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SECURITY_SCANNER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products" - DS_ANALYZER_IMAGE_PREFIX: "$SECURITY_SCANNER_IMAGE_PREFIX/analyzers" + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + + # Deprecated, use SECURE_ANALYZERS_PREFIX instead + DS_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX" + DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_MAJOR_VERSION: 2 - DS_DISABLE_DIND: "false" + DS_DISABLE_DIND: "true" dependency_scanning: stage: test @@ -21,7 +26,6 @@ dependency_scanning: services: - docker:stable-dind script: - - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | if ! docker info &>/dev/null; then if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then @@ -68,28 +72,25 @@ dependency_scanning: ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ - "$SECURITY_SCANNER_IMAGE_PREFIX/dependency-scanning:$DS_VERSION" /code + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_MAJOR_VERSION" /code artifacts: reports: dependency_scanning: gl-dependency-scanning-report.json dependencies: [] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ - except: - variables: - - $DEPENDENCY_SCANNING_DISABLED - - $DS_DISABLE_DIND == 'true' + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'true' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ .ds-analyzer: extends: dependency_scanning services: [] - except: - variables: - - $DEPENDENCY_SCANNING_DISABLED - - $DS_DISABLE_DIND == 'false' + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ script: - /analyzer run @@ -97,48 +98,81 @@ gemnasium-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php|\bgo\b/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' gemnasium-maven-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(java|scala)\b/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ + exists: + - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.sbt,*/build.sbt,*/*/build.sbt}' + - '{pom.xml,*/pom.xml,*/*/pom.xml}' gemnasium-python-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ + exists: + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && + $PIP_REQUIREMENTS_FILE bundler-audit-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/bundler-audit:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' retire-js-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/retire.js:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /retire.js/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /retire.js/ + exists: + - '{package.json,*/package.json,*/*/package.json}' diff --git a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml index 58fd018a82d..87f78d0c887 100644 --- a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml @@ -1,29 +1,13 @@ # Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/14624 # Please, use License-Scanning.gitlab-ci.yml template instead -variables: - LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. +include: + - template: License-Scanning.gitlab-ci.yml -license_management: - stage: test - image: - name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" - entrypoint: [""] - variables: - SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD - allow_failure: true - script: - - echo "This template is deprecated, please use License-Scanning.gitlab-ci.yml template instead." - - /run.sh analyze . - artifacts: - reports: - license_management: gl-license-management-report.json - dependencies: [] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\blicense_management\b/ - except: - variables: - - $LICENSE_MANAGEMENT_DISABLED +license_scanning: + before_script: + - | + echo "As of GitLab 12.8, we deprecated the License-Management.gitlab.ci.yml template. + Please replace it with the License-Scanning.gitlab-ci.yml template instead. + For more details visit + https://docs.gitlab.com/ee/user/compliance/license_compliance/#migration-from-license_management-to-license_scanning" diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 2333fb4e947..b86014c1ebc 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -5,29 +5,30 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. + LICENSE_MANAGEMENT_VERSION: 3 license_scanning: stage: test image: - name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" + name: "$SECURE_ANALYZERS_PREFIX/license-finder:$LICENSE_MANAGEMENT_VERSION" entrypoint: [""] variables: + LM_REPORT_FILE: gl-license-scanning-report.json SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD allow_failure: true script: - /run.sh analyze . - after_script: - - mv gl-license-management-report.json gl-license-scanning-report.json artifacts: reports: - license_scanning: gl-license-scanning-report.json + license_scanning: $LM_REPORT_FILE dependencies: [] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\blicense_scanning\b/ - except: - variables: - - $LICENSE_MANAGEMENT_DISABLED + rules: + - if: $LICENSE_MANAGEMENT_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\blicense_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 03b9720747d..47f68118ee0 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -5,10 +5,16 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + + # Deprecated, use SECURE_ANALYZERS_PREFIX instead + SAST_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec" SAST_ANALYZER_IMAGE_TAG: 2 - SAST_DISABLE_DIND: "false" + SAST_DISABLE_DIND: "true" SCAN_KUBERNETES_MANIFESTS: "false" sast: @@ -17,19 +23,18 @@ sast: artifacts: reports: sast: gl-sast-report.json - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'true' + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bsast\b/ image: docker:stable variables: + SEARCH_MAX_DEPTH: 4 DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: - - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | if ! docker info &>/dev/null; then if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then @@ -41,19 +46,16 @@ sast: $(awk 'BEGIN{for(v in ENVIRON) print v}' | grep -v -E '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | awk '{printf " -e %s", $0}') \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code - except: - variables: - - $SAST_DISABLED - - $SAST_DISABLE_DIND == 'true' + "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_ANALYZER_IMAGE_TAG" /app/bin/run /code .sast-analyzer: extends: sast services: [] - except: - variables: - - $SAST_DISABLED - - $SAST_DISABLE_DIND == 'false' + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ script: - /analyzer run @@ -61,49 +63,65 @@ bandit-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /bandit/&& - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bpython\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /bandit/ + exists: + - '**/*.py' brakeman-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /brakeman/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bruby\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /brakeman/ + exists: + - '**/*.rb' eslint-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /eslint/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjavascript\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /eslint/ + exists: + - '**/*.html' + - '**/*.js' flawfinder-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /(c(\+\+)?,)|(c(\+\+)?$)/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ + exists: + - '**/*.c' + - '**/*.cpp' kubesec-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && $SCAN_KUBERNETES_MANIFESTS == 'true' @@ -111,87 +129,117 @@ gosec-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /gosec/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bgo\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /gosec/ + exists: + - '**/*.go' nodejs-scan-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjavascript\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ + exists: + - '**/*.js' phpcs-security-audit-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bphp\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ + exists: + - '**/*.php' pmd-apex-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bapex\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ + exists: + - '**/*.cls' secrets-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /secrets/ security-code-scan-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\#|visual basic\b)/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ + exists: + - '**/*.csproj' + - '**/*.vbproj' sobelow-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /sobelow/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\belixir\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /sobelow/ + exists: + - '**/*.ex' + - '**/*.exs' spotbugs-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(groovy|java|scala)\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' tslint-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /tslint/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\btypescript\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /tslint/ + exists: + - '**/*.ts' diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml new file mode 100644 index 00000000000..b6c05c61db1 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -0,0 +1,246 @@ +# This template should be used when Security Products (https://about.gitlab.com/handbook/engineering/development/secure/#security-products) +# have to be downloaded and stored locally. +# +# Usage: +# +# ``` +# include: +# - template: Secure-Binaries.gitlab-ci.yml +# ``` +# +# Docs: https://docs.gitlab.com/ee/topics/airgap/ + + +variables: + SECURE_BINARIES_ANALYZERS: >- + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec, + bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, + klar, clair-vulnerabilities-db, + license-finder, + dast + + SECURE_BINARIES_DOWNLOAD_IMAGES: "true" + SECURE_BINARIES_PUSH_IMAGES: "true" + SECURE_BINARIES_SAVE_ARTIFACTS: "false" + + SECURE_BINARIES_ANALYZER_VERSION: "2" + +.download_images: + allow_failure: true + image: docker:stable + only: + refs: + - branches + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + services: + - docker:stable-dind + script: + - docker info + - env + - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"registry.gitlab.com/gitlab-org/security-products/analyzers/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi + - docker pull ${SECURE_BINARIES_IMAGE} + - mkdir -p output/$(dirname ${CI_JOB_NAME}) + - | + if [ "$SECURE_BINARIES_SAVE_ARTIFACTS" = "true" ]; then + docker save ${SECURE_BINARIES_IMAGE} | gzip > output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz + sha256sum output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz > output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz.sha256sum + fi + - | + if [ "$SECURE_BINARIES_PUSH_IMAGES" = "true" ]; then + docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + docker tag ${SECURE_BINARIES_IMAGE} ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION} + docker push ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION} + fi + + artifacts: + paths: + - output/ + +# +# SAST jobs +# + +bandit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbandit\b/ + +brakeman: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbrakeman\b/ + +gosec: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgosec\b/ + +spotbugs: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bspotbugs\b/ + +flawfinder: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bflawfinder\b/ + +phpcs-security-audit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bphpcs-security-audit\b/ + +security-code-scan: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecurity-code-scan\b/ + +nodejs-scan: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bnodejs-scan\b/ + +eslint: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\beslint\b/ + +tslint: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\btslint\b/ + +secrets: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ + +sobelow: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsobelow\b/ + +pmd-apex: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ + +kubesec: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bkubesec\b/ +# +# Container Scanning jobs +# + +klar: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/ + +clair-vulnerabilities-db: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bclair-vulnerabilities-db\b/ + variables: + SECURE_BINARIES_IMAGE: arminc/clair-db + SECURE_BINARIES_ANALYZER_VERSION: latest + +# +# Dependency Scanning jobs +# + +bundler-audit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbundler-audit\b/ + +retire.js: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bretire\.js\b/ + +gemnasium: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium\b/ + +gemnasium-maven: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-maven\b/ + +gemnasium-python: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-python\b/ + +# +# License Scanning +# + +license-finder: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\blicense-finder\b/ + +# +# DAST +# + +dast: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "1" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 83483108fde..a0832718214 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -19,7 +19,7 @@ cache: - .terraform before_script: - - alias convert_report="jq -r '([.resource_changes[].change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" + - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" - terraform --version - terraform init diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 5d9d3c74def..e8a99a6ea06 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -8,12 +8,14 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.2 + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.3 script: /gitlab-accessibility.sh $a11y_urls allow_failure: true artifacts: when: always expose_as: 'Accessibility Reports' paths: ['reports/'] + reports: + accessibility: reports/gl-accessibility.json rules: - if: $a11y_urls diff --git a/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml new file mode 100644 index 00000000000..05635cf71be --- /dev/null +++ b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml @@ -0,0 +1,7 @@ +# Read more on when to use this template at +# https://docs.gitlab.com/ee/ci/yaml/#workflowrules + +workflow: + rules: + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml new file mode 100644 index 00000000000..50ff4c1f60b --- /dev/null +++ b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml @@ -0,0 +1,8 @@ +# Read more on when to use this template at +# https://docs.gitlab.com/ee/ci/yaml/#workflowrules + +workflow: + rules: + - if: $CI_MERGE_REQUEST_IID + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 933504ea82f..5816ac3bc54 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -157,7 +157,7 @@ module Gitlab return unless job[:stage] unless job[:stage].is_a?(String) && job[:stage].in?(@stages) - raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" + raise ValidationError, "#{name} job: chosen stage does not exist; available stages are #{@stages.join(", ")}" end end -- cgit v1.2.3