diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-18 11:17:02 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-18 11:17:02 +0300 |
| commit | b39512ed755239198a9c294b6a45e65c05900235 (patch) | |
| tree | d234a3efade1de67c46b9e5a38ce813627726aa7 /lib/gitlab/ci | |
| parent | d31474cf3b17ece37939d20082b07f6657cc79a9 (diff) | |
Add latest changes from gitlab-org/gitlab@15-3-stable-eev15.3.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
82 files changed, 1112 insertions, 365 deletions
diff --git a/lib/gitlab/ci/artifacts/logger.rb b/lib/gitlab/ci/artifacts/logger.rb new file mode 100644 index 00000000000..628f4129df4 --- /dev/null +++ b/lib/gitlab/ci/artifacts/logger.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Artifacts + module Logger + def log_artifacts_filesize(artifact_file) + return if artifact_file.nil? + + unless artifact_file.is_a?(::Ci::Artifactable) + raise ArgumentError, "unknown artifact file class `#{artifact_file.class}`" + end + + ::Gitlab::ApplicationContext.push(artifact: artifact_file) + end + + def log_artifacts_context(job) + ::Gitlab::ApplicationContext.push( + namespace: job&.project&.namespace, + project: job&.project, + job: job + ) + end + + def log_build_dependencies(size:, count: 0) + ::Gitlab::ApplicationContext.push( + artifacts_dependencies_size: size, + artifacts_dependencies_count: count + ) + end + + def self.log_created(artifact) + payload = Gitlab::ApplicationContext.current.merge( + message: 'Artifact created', + job_artifact_id: artifact.id, + size: artifact.size, + type: artifact.file_type, + build_id: artifact.job_id, + project_id: artifact.project_id + ) + + Gitlab::AppLogger.info(payload) + end + + def self.log_deleted(job_artifacts, method) + Array(job_artifacts).each do |artifact| + payload = Gitlab::ApplicationContext.current.merge( + message: 'Artifact deleted', + job_artifact_id: artifact.id, + expire_at: artifact.expire_at, + size: artifact.size, + type: artifact.file_type, + build_id: artifact.job_id, + project_id: artifact.project_id, + method: method + ) + + Gitlab::AppLogger.info(payload) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/artifacts/metrics.rb b/lib/gitlab/ci/artifacts/metrics.rb index 03459c4bf36..59930426cd5 100644 --- a/lib/gitlab/ci/artifacts/metrics.rb +++ b/lib/gitlab/ci/artifacts/metrics.rb @@ -6,6 +6,13 @@ module Gitlab class Metrics include Gitlab::Utils::StrongMemoize + def self.build_completed_report_type_counter(report_type) + name = "artifact_report_#{report_type}_builds_completed_total".to_sym + comment = "Number of completed builds with #{report_type} report artifacts" + + ::Gitlab::Metrics.counter(name, comment) + end + def increment_destroyed_artifacts_count(size) destroyed_artifacts_counter.increment({}, size.to_i) end diff --git a/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb new file mode 100644 index 00000000000..690a47097c6 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Artifacts + module Adapters + class ZipStream + MAX_DECOMPRESSED_SIZE = 100.megabytes + MAX_FILES_PROCESSED = 50 + + attr_reader :stream + + InvalidStreamError = Class.new(StandardError) + + def initialize(stream) + raise InvalidStreamError, "Stream is required" unless stream + + @stream = stream + @files_processed = 0 + end + + def each_blob + Zip::InputStream.open(stream) do |zio| + while entry = zio.get_next_entry + break if at_files_processed_limit? + next unless should_process?(entry) + + @files_processed += 1 + + yield entry.get_input_stream.read + end + end + end + + private + + def should_process?(entry) + file?(entry) && !too_large?(entry) + end + + def file?(entry) + # Check the file name as a workaround for incorrect + # file type detection when using InputStream + # https://github.com/rubyzip/rubyzip/issues/533 + entry.file? && !entry.name.end_with?('/') + end + + def too_large?(entry) + entry.size > MAX_DECOMPRESSED_SIZE + end + + def at_files_processed_limit? + @files_processed >= MAX_FILES_PROCESSED + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/releaser.rb b/lib/gitlab/ci/build/releaser.rb index 9720bb1123a..09717516aa4 100644 --- a/lib/gitlab/ci/build/releaser.rb +++ b/lib/gitlab/ci/build/releaser.rb @@ -5,7 +5,7 @@ module Gitlab module Build class Releaser BASE_COMMAND = 'release-cli create' - SINGLE_FLAGS = %i[name description tag_name ref released_at].freeze + SINGLE_FLAGS = %i[name description tag_name tag_message ref released_at].freeze ARRAY_FLAGS = %i[milestones].freeze attr_reader :config diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index 2d4f9cf635b..dee95534b07 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -6,7 +6,7 @@ module Gitlab class Rules include ::Gitlab::Utils::StrongMemoize - Result = Struct.new(:when, :start_in, :allow_failure, :variables) do + Result = Struct.new(:when, :start_in, :allow_failure, :variables, :errors) do def build_attributes { when: self.when, @@ -38,6 +38,8 @@ module Gitlab else Result.new('never') end + rescue Rule::Clause::ParseError => e + Result.new('never', nil, nil, nil, [e.message]) end private diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb index 6d4bbbb8c21..503f2a87361 100644 --- a/lib/gitlab/ci/build/rules/rule/clause.rb +++ b/lib/gitlab/ci/build/rules/rule/clause.rb @@ -11,6 +11,7 @@ module Gitlab # Used for job's inclusion rules configuration. # UnknownClauseError = Class.new(StandardError) + ParseError = Class.new(StandardError) def self.fabricate(type, value) "#{self}::#{type.to_s.camelize}".safe_constantize&.new(value) diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 1bcd87c9d93..1034f5eacef 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -11,10 +11,12 @@ module Gitlab end def satisfied_by?(pipeline, context) - return true unless pipeline&.modified_paths + modified_paths = find_modified_paths(pipeline) + + return true unless modified_paths expanded_globs = expand_globs(context) - pipeline.modified_paths.any? do |path| + modified_paths.any? do |path| expanded_globs.any? do |glob| File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) end @@ -33,13 +35,31 @@ module Gitlab def paths strong_memoize(:paths) do - if @globs.is_a?(Array) - @globs - else - Array(@globs[:paths]) - end + Array(@globs[:paths]) end end + + def find_modified_paths(pipeline) + return unless pipeline + return pipeline.modified_paths unless ::Feature.enabled?(:ci_rules_changes_compare, pipeline.project) + + compare_to_sha = find_compare_to_sha(pipeline) + + if compare_to_sha + pipeline.modified_paths_since(compare_to_sha) + else + pipeline.modified_paths + end + end + + def find_compare_to_sha(pipeline) + return unless @globs.include?(:compare_to) + + commit = pipeline.project.commit(@globs[:compare_to]) + raise Rules::Rule::Clause::ParseError, 'rules:changes:compare_to is not a valid ref' unless commit + + commit.sha + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 15a4ff91c1b..438fa1cb3b2 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -81,6 +81,10 @@ module Gitlab root.jobs_value end + def workflow_rules + root.workflow_entry.rules_value + end + def normalized_jobs @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 96ac959a3f4..613f7ff3370 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -8,37 +8,13 @@ module Gitlab # Entry that represents a Docker image. # class Image < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - include ::Gitlab::Config::Entry::Configurable - - ALLOWED_KEYS = %i[name entrypoint ports pull_policy].freeze - LEGACY_ALLOWED_KEYS = %i[name entrypoint ports].freeze + include ::Gitlab::Ci::Config::Entry::Imageable validations do - validates :config, hash_or_string: true - validates :config, allowed_keys: ALLOWED_KEYS, if: :ci_docker_image_pull_policy_enabled? - validates :config, allowed_keys: LEGACY_ALLOWED_KEYS, unless: :ci_docker_image_pull_policy_enabled? - validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? - - validates :name, type: String, presence: true - validates :entrypoint, array_of_strings: true, allow_nil: true - end - - entry :ports, Entry::Ports, - description: 'Ports used to expose the image' - - entry :pull_policy, Entry::PullPolicy, - description: 'Pull policy for the image' - - attributes :ports, :pull_policy - - def name - value[:name] - end - - def entrypoint - value[:entrypoint] + validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS, + if: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: IMAGEABLE_LEGACY_ALLOWED_KEYS, + unless: :ci_docker_image_pull_policy_enabled? end def value @@ -55,18 +31,6 @@ module Gitlab {} end end - - def with_image_ports? - opt(:with_image_ports) - end - - def ci_docker_image_pull_policy_enabled? - ::Feature.enabled?(:ci_docker_image_pull_policy) - end - - def skip_config_hash_validation? - true - end end end end diff --git a/lib/gitlab/ci/config/entry/imageable.rb b/lib/gitlab/ci/config/entry/imageable.rb new file mode 100644 index 00000000000..f045ee3d549 --- /dev/null +++ b/lib/gitlab/ci/config/entry/imageable.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Represents Imageable concern shared by Image and Service. + module Imageable + extend ActiveSupport::Concern + + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable + + IMAGEABLE_ALLOWED_KEYS = %i[name entrypoint ports pull_policy].freeze + IMAGEABLE_LEGACY_ALLOWED_KEYS = %i[name entrypoint ports].freeze + + included do + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, hash_or_string: true + validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? + + validates :name, type: String, presence: true + validates :entrypoint, array_of_strings: true, allow_nil: true + end + + attributes :ports, :pull_policy + + entry :ports, Entry::Ports, + description: 'Ports used to expose the image/service' + + entry :pull_policy, Entry::PullPolicy, + description: 'Pull policy for the image/service' + end + + def name + value[:name] + end + + def entrypoint + value[:entrypoint] + end + + def with_image_ports? + opt(:with_image_ports) + end + + def ci_docker_image_pull_policy_enabled? + ::Feature.enabled?(:ci_docker_image_pull_policy) + end + + def skip_config_hash_validation? + true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 78794f524f4..975da8662e1 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -120,7 +120,7 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - job_variables: variables_value.to_h, + job_variables: variables_entry.value_with_data, root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, diff --git a/lib/gitlab/ci/config/entry/release.rb b/lib/gitlab/ci/config/entry/release.rb index 7e504c24ade..2be0eae120b 100644 --- a/lib/gitlab/ci/config/entry/release.rb +++ b/lib/gitlab/ci/config/entry/release.rb @@ -12,8 +12,8 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[tag_name name description ref released_at milestones assets].freeze - attributes %i[tag_name name ref milestones assets].freeze + ALLOWED_KEYS = %i[tag_name tag_message name description ref released_at milestones assets].freeze + attributes %i[tag_name tag_message name ref milestones assets].freeze attr_reader :released_at # Attributable description conflicts with @@ -31,6 +31,7 @@ module Gitlab validations do validates :config, allowed_keys: ALLOWED_KEYS validates :tag_name, type: String, presence: true + validates :tag_message, type: String, allow_blank: true validates :description, type: String, presence: true validates :milestones, array_of_strings_or_string: true, allow_blank: true validate do diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index d5d204bb995..f77876cc926 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -17,7 +17,7 @@ module Gitlab dast performance browser_performance load_performance license_scanning metrics lsif dotenv terraform accessibility requirements coverage_fuzzing api_fuzzing cluster_image_scanning - coverage_report].freeze + coverage_report cyclonedx].freeze attributes ALLOWED_KEYS @@ -48,6 +48,7 @@ module Gitlab validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true validates :requirements, array_of_strings_or_string: true + validates :cyclonedx, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/rules/rule/changes.rb b/lib/gitlab/ci/config/entry/rules/rule/changes.rb index a56b928450a..107e7c228af 100644 --- a/lib/gitlab/ci/config/entry/rules/rule/changes.rb +++ b/lib/gitlab/ci/config/entry/rules/rule/changes.rb @@ -30,7 +30,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[paths].freeze + ALLOWED_KEYS = %i[paths compare_to].freeze REQUIRED_KEYS = %i[paths].freeze attributes ALLOWED_KEYS @@ -43,6 +43,7 @@ module Gitlab validates :paths, array_of_strings: true, length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + validates :compare_to, type: String, allow_nil: true end end end diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 1a35f7de6cf..0e19447dff8 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -7,41 +7,28 @@ module Gitlab ## # Entry that represents a configuration of Docker service. # - # TODO: remove duplication with Image superclass by defining a common - # Imageable concern. - # https://gitlab.com/gitlab-org/gitlab/issues/208774 class Service < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Ci::Config::Entry::Imageable - ALLOWED_KEYS = %i[name entrypoint command alias ports variables pull_policy].freeze - LEGACY_ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze + ALLOWED_KEYS = %i[command alias variables].freeze + LEGACY_ALLOWED_KEYS = %i[command alias variables].freeze validations do - validates :config, hash_or_string: true - validates :config, allowed_keys: ALLOWED_KEYS, if: :ci_docker_image_pull_policy_enabled? - validates :config, allowed_keys: LEGACY_ALLOWED_KEYS, unless: :ci_docker_image_pull_policy_enabled? - validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? - validates :name, type: String, presence: true - validates :entrypoint, array_of_strings: true, allow_nil: true + validates :config, allowed_keys: ALLOWED_KEYS + IMAGEABLE_ALLOWED_KEYS, + if: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: LEGACY_ALLOWED_KEYS + IMAGEABLE_LEGACY_ALLOWED_KEYS, + unless: :ci_docker_image_pull_policy_enabled? validates :command, array_of_strings: true, allow_nil: true validates :alias, type: String, allow_nil: true validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? } end - entry :ports, Entry::Ports, - description: 'Ports used to expose the service' - - entry :pull_policy, Entry::PullPolicy, - description: 'Pull policy for the service' - entry :variables, ::Gitlab::Ci::Config::Entry::Variables, description: 'Environment variables available for this service.', inherit: false - attributes :ports, :pull_policy, :variables + attributes :variables def alias value[:alias] @@ -51,14 +38,6 @@ module Gitlab value[:command] end - def name - value[:name] - end - - def entrypoint - value[:entrypoint] - end - def value if string? { name: @config } @@ -70,18 +49,6 @@ module Gitlab {} end end - - def with_image_ports? - opt(:with_image_ports) - end - - def ci_docker_image_pull_policy_enabled? - ::Feature.enabled?(:ci_docker_image_pull_policy) - end - - def skip_config_hash_validation? - true - end end end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index efb469ee32a..3130aec0446 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -18,7 +18,9 @@ module Gitlab end def value - @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } + @config.to_h do |key, data| + [key.to_s, expand_data(data)[:value]] + end end def self.default(**) @@ -26,7 +28,9 @@ module Gitlab end def value_with_data - @config.to_h { |key, value| [key.to_s, expand_value(value)] } + @config.to_h do |key, data| + [key.to_s, expand_data(data)] + end end def use_value_data? @@ -35,11 +39,11 @@ module Gitlab private - def expand_value(value) - if value.is_a?(Hash) - { value: value[:value].to_s, description: value[:description] } + def expand_data(data) + if data.is_a?(Hash) + { value: data[:value].to_s, description: data[:description] }.compact else - { value: value.to_s, description: nil } + { value: data.to_s } end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index c294291e538..d3e7210b820 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -65,7 +65,7 @@ module Gitlab fields.merge!( environment: environment.name, environment_protected: environment_protected?.to_s, - deployment_tier: build.environment_deployment_tier || environment.tier + deployment_tier: build.environment_tier ) end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 1223d664214..b52e2d8f613 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -13,7 +13,8 @@ module Gitlab accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y, codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate, sast: ::Gitlab::Ci::Parsers::Security::Sast, - secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection + secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection, + cyclonedx: ::Gitlab::Ci::Parsers::Sbom::Cyclonedx } end diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb new file mode 100644 index 00000000000..deb20a2138c --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + class Cyclonedx + SUPPORTED_SPEC_VERSIONS = %w[1.4].freeze + COMPONENT_ATTRIBUTES = %w[type name version].freeze + + def parse!(blob, sbom_report) + @report = sbom_report + @data = Gitlab::Json.parse(blob) + + return unless valid? + + parse_report + rescue JSON::ParserError => e + report.add_error("Report JSON is invalid: #{e}") + end + + private + + attr_reader :json_data, :report, :data + + def schema_validator + @schema_validator ||= Validators::CyclonedxSchemaValidator.new(data) + end + + def valid? + valid_schema? && supported_spec_version? + end + + def supported_spec_version? + return true if SUPPORTED_SPEC_VERSIONS.include?(data['specVersion']) + + report.add_error( + "Unsupported CycloneDX spec version. Must be one of: %{versions}" \ + % { versions: SUPPORTED_SPEC_VERSIONS.join(', ') } + ) + + false + end + + def valid_schema? + return true if schema_validator.valid? + + schema_validator.errors.each { |error| report.add_error(error) } + + false + end + + def parse_report + parse_metadata_properties + parse_components + end + + def parse_metadata_properties + properties = data.dig('metadata', 'properties') + source = CyclonedxProperties.parse_source(properties) + report.set_source(source) if source + end + + def parse_components + data['components']&.each do |component| + next unless supported_component_type?(component['type']) + + report.add_component(component.slice(*COMPONENT_ATTRIBUTES)) + end + end + + def supported_component_type?(type) + ::Enums::Sbom.component_types.include?(type.to_sym) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb new file mode 100644 index 00000000000..3dc73544208 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + # Parses GitLab CycloneDX metadata properties which are defined by the taxonomy at + # https://gitlab.com/gitlab-org/security-products/gitlab-cyclonedx-property-taxonomy + # + # This parser knows how to process schema version 1 and will not attempt to parse + # later versions. Each source type has it's own namespace in the property schema, + # and is also given its own parser. Properties are filtered by namespace, + # and then passed to each source parser for processing. + class CyclonedxProperties + SUPPORTED_SCHEMA_VERSION = '1' + GITLAB_PREFIX = 'gitlab:' + SOURCE_PARSERS = { + 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning + }.freeze + SUPPORTED_PROPERTIES = %w[ + meta:schema_version + dependency_scanning:category + dependency_scanning:input_file:path + dependency_scanning:source_file:path + dependency_scanning:package_manager:name + dependency_scanning:language:name + ].freeze + + def self.parse_source(...) + new(...).parse_source + end + + def initialize(properties) + @properties = properties + end + + def parse_source + return unless properties.present? + return unless supported_schema_version? + + source + end + + private + + attr_reader :properties + + def property_data + @property_data ||= properties + .each_with_object({}) { |property, data| parse_property(property, data) } + end + + def parse_property(property, data) + name = property['name'] + value = property['value'] + + # The specification permits the name or value to be absent. + return unless name.present? && value.present? + return unless name.start_with?(GITLAB_PREFIX) + + namespaced_name = name.delete_prefix(GITLAB_PREFIX) + + return unless SUPPORTED_PROPERTIES.include?(namespaced_name) + + parse_name_value_pair(namespaced_name, value, data) + end + + def parse_name_value_pair(name, value, data) + # Each namespace in the property name reflects a key in the hash. + # A property with the name `dependency_scanning:input_file:path` + # and the value `package-lock.json` should be transformed into + # this data: + # {"dependency_scanning": {"input_file": {"path": "package-lock.json"}}} + keys = name.split(':') + + # Remove last item from the keys and use it to create + # the initial object. + last = keys.pop + + # Work backwards. For each key, create a new hash wrapping the previous one. + # Using `dependency_scanning:input_file:path` as an example: + # + # 1. memo = { "path" => "package-lock.json" } (arguments given to reduce) + # 2. memo = { "input_file" => memo } + # 3. memo = { "dependency_scanning" => memo } + property = keys.reverse.reduce({ last => value }) do |memo, key| + { key => memo } + end + + data.deep_merge!(property) + end + + def schema_version + @schema_version ||= property_data.dig('meta', 'schema_version') + end + + def supported_schema_version? + schema_version == SUPPORTED_SCHEMA_VERSION + end + + def source + @source ||= property_data + .slice(*SOURCE_PARSERS.keys) + .lazy + .filter_map { |namespace, data| SOURCE_PARSERS[namespace].source(data) } + .first + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb new file mode 100644 index 00000000000..ad04b3257f9 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Source + class DependencyScanning + REQUIRED_ATTRIBUTES = [ + %w[input_file path] + ].freeze + + def self.source(...) + new(...).source + end + + def initialize(data) + @data = data + end + + def source + return unless required_attributes_present? + + { + 'type' => :dependency_scanning, + 'data' => data, + 'fingerprint' => fingerprint + } + end + + private + + attr_reader :data + + def required_attributes_present? + REQUIRED_ATTRIBUTES.all? do |keys| + data.dig(*keys).present? + end + end + + def fingerprint + Digest::SHA256.hexdigest(data.to_json) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb b/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb new file mode 100644 index 00000000000..9d56e001c2f --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Validators + class CyclonedxSchemaValidator + SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'cyclonedx_report.json').freeze + + def initialize(report_data) + @report_data = report_data + end + + def valid? + errors.empty? + end + + def errors + @errors ||= pretty_errors + end + + private + + def raw_errors + JSONSchemer.schema(SCHEMA_PATH).validate(@report_data) + end + + def pretty_errors + raw_errors.map { |error| JSONSchemer::Errors.pretty(error) } + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index ee7733a081d..c075ada725a 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -38,13 +38,14 @@ module Gitlab def initialize(report_type, report_version) @report_type = report_type.to_sym @report_version = report_version.to_s + @supported_versions = SUPPORTED_VERSIONS[@report_type] end delegate :validate, to: :schemer private - attr_reader :report_type, :report_version + attr_reader :report_type, :report_version, :supported_versions def schemer JSONSchemer.schema(pathname) @@ -60,10 +61,24 @@ module Gitlab report_declared_version = File.join(root_path, report_version, file_name) return report_declared_version if File.file?(report_declared_version) + if latest_vendored_patch_version + latest_vendored_patch_version_file = File.join(root_path, latest_vendored_patch_version, file_name) + return latest_vendored_patch_version_file if File.file?(latest_vendored_patch_version) + end + earliest_supported_version = SUPPORTED_VERSIONS[report_type].min File.join(root_path, earliest_supported_version, file_name) end + def latest_vendored_patch_version + ::Security::ReportSchemaVersionMatcher.new( + report_declared_version: report_version, + supported_versions: supported_versions + ).call + rescue ArgumentError + nil + end + def file_name report_type == :api_fuzzing ? "dast-report-format.json" : "#{report_type.to_s.dasherize}-report-format.json" end @@ -79,29 +94,85 @@ module Gitlab @warnings = [] @deprecation_warnings = [] - populate_errors - populate_warnings + populate_schema_version_errors + populate_validation_errors populate_deprecation_warnings end - def valid? - errors.empty? + def populate_schema_version_errors + add_schema_version_errors if add_schema_version_error? end - def populate_errors - schema_validation_errors = schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + def add_schema_version_errors + if report_version.nil? + template = _("Report version not provided,"\ + " %{report_type} report type supports versions: %{supported_schema_versions}."\ + " GitLab will attempt to validate this report against the earliest supported versions of this report"\ + " type, to show all the errors but will not ingest the report") + message = format(template, report_type: report_type, supported_schema_versions: supported_schema_versions) + else + template = _("Version %{report_version} for report type %{report_type} is unsupported, supported versions"\ + " for this report type are: %{supported_schema_versions}."\ + " GitLab will attempt to validate this report against the earliest supported versions of this report"\ + " type, to show all the errors but will not ingest the report") + message = format(template, report_version: report_version, report_type: report_type, supported_schema_versions: supported_schema_versions) + end - log_warnings(problem_type: 'schema_validation_fails') unless schema_validation_errors.empty? + log_warnings(problem_type: 'using_unsupported_schema_version') + add_message_as(level: :error, message: message) + end + + def add_schema_version_error? + !report_uses_supported_schema_version? && + !report_uses_deprecated_schema_version? && + !report_uses_supported_major_and_minor_schema_version? + end + + def report_uses_deprecated_schema_version? + DEPRECATED_VERSIONS[report_type].include?(report_version) + end + + def report_uses_supported_schema_version? + SUPPORTED_VERSIONS[report_type].include?(report_version) + end - if Feature.enabled?(:enforce_security_report_validation, @project) - @errors += schema_validation_errors + def report_uses_supported_major_and_minor_schema_version? + if !find_latest_patch_version.nil? + add_supported_major_minor_behavior_warning + true else - @warnings += schema_validation_errors + false end end - def populate_warnings - add_unsupported_report_version_message if !report_uses_supported_schema_version? && !report_uses_deprecated_schema_version? + def find_latest_patch_version + ::Security::ReportSchemaVersionMatcher.new( + report_declared_version: report_version, + supported_versions: SUPPORTED_VERSIONS[report_type] + ).call + rescue ArgumentError + nil + end + + def add_supported_major_minor_behavior_warning + template = _("This report uses a supported MAJOR.MINOR schema version but the PATCH version doesn't match"\ + " any vendored schema version. Validation will be attempted against version"\ + " %{find_latest_patch_version}") + + message = format(template, find_latest_patch_version: find_latest_patch_version) + + add_message_as( + level: :warning, + message: message + ) + end + + def populate_validation_errors + schema_validation_errors = schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + + log_warnings(problem_type: 'schema_validation_fails') unless schema_validation_errors.empty? + + @errors += schema_validation_errors end def populate_deprecation_warnings @@ -111,10 +182,19 @@ module Gitlab def add_deprecated_report_version_message log_warnings(problem_type: 'using_deprecated_schema_version') - message = "Version #{report_version} for report type #{report_type} has been deprecated, supported versions for this report type are: #{supported_schema_versions}" + template = _("Version %{report_version} for report type %{report_type} has been deprecated,"\ + " supported versions for this report type are: %{supported_schema_versions}."\ + " GitLab will attempt to parse and ingest this report if valid.") + + message = format(template, report_version: report_version, report_type: report_type, supported_schema_versions: supported_schema_versions) + add_message_as(level: :deprecation_warning, message: message) end + def valid? + errors.empty? + end + def log_warnings(problem_type:) Gitlab::AppLogger.info( message: 'security report schema validation problem', @@ -127,34 +207,6 @@ module Gitlab ) end - def add_unsupported_report_version_message - log_warnings(problem_type: 'using_unsupported_schema_version') - - if Feature.enabled?(:enforce_security_report_validation, @project) - handle_unsupported_report_version(treat_as: :error) - else - handle_unsupported_report_version(treat_as: :warning) - end - end - - def report_uses_deprecated_schema_version? - DEPRECATED_VERSIONS[report_type].include?(report_version) - end - - def report_uses_supported_schema_version? - SUPPORTED_VERSIONS[report_type].include?(report_version) - end - - def handle_unsupported_report_version(treat_as:) - if report_version.nil? - message = "Report version not provided, #{report_type} report type supports versions: #{supported_schema_versions}" - else - message = "Version #{report_version} for report type #{report_type} is unsupported, supported versions for this report type are: #{supported_schema_versions}" - end - - add_message_as(level: treat_as, message: message) - end - def supported_schema_versions SUPPORTED_VERSIONS[report_type].join(", ") end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index f637001f9f8..9c12d46cede 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -40,7 +40,13 @@ module Gitlab ::Ci::Pipeline .id_in(pipeline_ids) .with_only_interruptible_builds - .each { |cancelable| cancelable.auto_cancel_running(pipeline) } + .each do |cancelable_pipeline| + # cascade_to_children not needed because we iterate through descendants here + cancelable_pipeline.cancel_running( + auto_canceled_by_pipeline_id: pipeline.id, + cascade_to_children: false + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 8177502be1d..6e95c7988fc 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -101,7 +101,8 @@ module Gitlab ref: pipeline.ref, type: pipeline.source }, - builds: builds_validation_payload + builds: builds_validation_payload, + total_builds_count: current_user.pipelines.jobs_count_in_alive_pipelines } end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 6efb3a4f16a..c4f06c4686d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -14,11 +14,9 @@ module Gitlab return false unless regexp - if ::Feature.enabled?(:ci_fix_rules_if_comparison_with_regexp_variable) - # All variables are evaluated as strings, even if they are regexp strings. - # So, we need to convert them to regexp objects. - regexp = Lexeme::Pattern.build_and_evaluate(regexp, variables) - end + # All variables are evaluated as strings, even if they are regexp strings. + # So, we need to convert them to regexp objects. + regexp = Lexeme::Pattern.build_and_evaluate(regexp, variables) regexp.scan(text.to_s).present? end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index a72e5dbc822..99d9206da74 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -14,11 +14,9 @@ module Gitlab return true unless regexp - if ::Feature.enabled?(:ci_fix_rules_if_comparison_with_regexp_variable) - # All variables are evaluated as strings, even if they are regexp strings. - # So, we need to convert them to regexp objects. - regexp = Lexeme::Pattern.build_and_evaluate(regexp, variables) - end + # All variables are evaluated as strings, even if they are regexp strings. + # So, we need to convert them to regexp objects. + regexp = Lexeme::Pattern.build_and_evaluate(regexp, variables) regexp.scan(text.to_s).empty? end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 901208f325a..93106b96af2 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -54,9 +54,11 @@ module Gitlab end def errors - return unless included? - strong_memoize(:errors) do + # We check rules errors before checking "included?" because rules affects its inclusion status. + next rules_errors if rules_errors + next unless included? + [needs_errors, variable_expansion_errors].compact.flatten end end @@ -168,6 +170,12 @@ module Gitlab end end + def rules_errors + strong_memoize(:rules_errors) do + ["Failed to parse rule for #{name}: #{rules_result.errors.join(', ')}"] if rules_result.errors.present? + end + end + def evaluate_context strong_memoize(:evaluate_context) do Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index c8795840e5f..6bcc71a808b 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -30,7 +30,7 @@ module Gitlab end def deployment_tier - job.environment_deployment_tier + job.environment_tier_from_options end def expanded_environment_name diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index bc56fe9bef9..7cf6466cf4b 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -36,7 +36,7 @@ module Gitlab def errors strong_memoize(:errors) do - seeds.flat_map(&:errors).compact + @builds.flat_map(&:errors).compact end end diff --git a/lib/gitlab/ci/reports/coverage_report_generator.rb b/lib/gitlab/ci/reports/coverage_report_generator.rb index 76992a48b0a..6d57e05aa63 100644 --- a/lib/gitlab/ci/reports/coverage_report_generator.rb +++ b/lib/gitlab/ci/reports/coverage_report_generator.rb @@ -20,7 +20,7 @@ module Gitlab coverage_report.tap do |coverage_report| report_builds.find_each do |build| - build.each_report(::Ci::JobArtifact::COVERAGE_REPORT_FILE_TYPES) do |file_type, blob| + build.each_report(::Ci::JobArtifact.file_types_for_report(:coverage)) do |file_type, blob| Gitlab::Ci::Parsers.fabricate!(file_type).parse!( blob, coverage_report, diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb new file mode 100644 index 00000000000..86b9be274cc --- /dev/null +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Sbom + class Component + attr_reader :component_type, :name, :version + + def initialize(component = {}) + @component_type = component['type'] + @name = component['name'] + @version = component['version'] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/sbom/report.rb b/lib/gitlab/ci/reports/sbom/report.rb new file mode 100644 index 00000000000..dc6b3153e51 --- /dev/null +++ b/lib/gitlab/ci/reports/sbom/report.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Sbom + class Report + attr_reader :components, :source, :errors + + def initialize + @components = [] + @errors = [] + end + + def add_error(error) + errors << error + end + + def set_source(source) + self.source = Source.new(source) + end + + def add_component(component) + components << Component.new(component) + end + + private + + attr_writer :source + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/sbom/reports.rb b/lib/gitlab/ci/reports/sbom/reports.rb new file mode 100644 index 00000000000..efb772cb818 --- /dev/null +++ b/lib/gitlab/ci/reports/sbom/reports.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Sbom + class Reports + attr_reader :reports + + def initialize + @reports = [] + end + + def add_report(report) + @reports << report + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb new file mode 100644 index 00000000000..60bf30b65a5 --- /dev/null +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Sbom + class Source + attr_reader :source_type, :data, :fingerprint + + def initialize(source = {}) + @source_type = source['type'] + @data = source['data'] + @fingerprint = source['fingerprint'] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb index 8773ecbf09e..dab24bfd501 100644 --- a/lib/gitlab/ci/runner_releases.rb +++ b/lib/gitlab/ci/runner_releases.rb @@ -36,6 +36,9 @@ module Gitlab reset_backoff! extract_releases(response) + rescue Errno::ETIMEDOUT + @backoff_expire_time = next_backoff.from_now + break nil end end @@ -74,7 +77,7 @@ module Gitlab releases = response.parsed_response .map { |release| parse_runner_release(release) } .select(&:valid?) - .sort! + .sort return if releases.empty? && response.parsed_response.present? diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index 10a89bb15d4..03130addd6a 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -3,57 +3,70 @@ module Gitlab module Ci class RunnerUpgradeCheck - include Singleton + def initialize(gitlab_version, runner_releases_store = nil) + @gitlab_version = ::Gitlab::VersionInfo.parse(gitlab_version, parse_suffix: true) + @releases_store = runner_releases_store + end + + def check_runner_upgrade_suggestion(runner_version) + check_runner_upgrade_suggestions(runner_version).first + end - def check_runner_upgrade_status(runner_version) + private + + def runner_releases_store + @releases_store ||= RunnerReleases.instance + end + + def add_suggestion(suggestions, runner_version, version, status) + return false unless version && version > runner_version + + suggestions[version] = status + true + end + + def check_runner_upgrade_suggestions(runner_version) runner_version = ::Gitlab::VersionInfo.parse(runner_version, parse_suffix: true) - return { invalid_version: runner_version } unless runner_version.valid? - return { error: runner_version } unless runner_releases_store.releases + return { runner_version => :invalid_version } unless runner_version.valid? + return { runner_version => :error } unless runner_releases_store.releases - # Recommend update if outside of backport window - recommended_version = recommendation_if_outside_backport_window(runner_version) - return { recommended: recommended_version } if recommended_version + suggestions = {} - # Recommend patch update if there's a newer release in a same minor branch as runner - recommended_version = recommended_runner_release_update(runner_version) - return { recommended: recommended_version } if recommended_version + # Recommend update if outside of backport window + unless add_recommendation_if_outside_backport_window(runner_version, suggestions) + # Recommend patch update if there's a newer release in a same minor branch as runner + add_recommended_runner_release_update(runner_version, suggestions) + end # Consider update if there's a newer release within the currently deployed GitLab version - available_version = available_runner_release(runner_version) - return { available: available_version } if available_version + add_available_runner_release(runner_version, suggestions) - { not_available: runner_version } - end + suggestions[runner_version] = :not_available if suggestions.empty? - private + suggestions + end - def recommended_runner_release_update(runner_version) + def add_recommended_runner_release_update(runner_version, suggestions) recommended_release = runner_releases_store.releases_by_minor[runner_version.without_patch] - return recommended_release if recommended_release && recommended_release > runner_version + return true if add_suggestion(suggestions, runner_version, recommended_release, :recommended) # Consider the edge case of pre-release runner versions that get registered, but are never published. # In this case, suggest the latest compatible runner version - latest_release = runner_releases_store.releases_by_minor.values.select { |v| v < gitlab_version }.max - latest_release if latest_release && latest_release > runner_version - end - - def available_runner_release(runner_version) - available_release = runner_releases_store.releases_by_minor[gitlab_version.without_patch] - available_release if available_release && available_release > runner_version + latest_release = runner_releases_store.releases_by_minor.values.select { |v| v < @gitlab_version }.max + add_suggestion(suggestions, runner_version, latest_release, :recommended) end - def gitlab_version - @gitlab_version ||= ::Gitlab::VersionInfo.parse(::Gitlab::VERSION, parse_suffix: true) - end - - def runner_releases_store - RunnerReleases.instance + def add_available_runner_release(runner_version, suggestions) + available_version = runner_releases_store.releases_by_minor[@gitlab_version.without_patch] + unless suggestions.include?(available_version) + add_suggestion(suggestions, runner_version, available_version, :available) + end end - def recommendation_if_outside_backport_window(runner_version) - return if runner_releases_store.releases.empty? - return if runner_version >= runner_releases_store.releases.last # return early if runner version is too new + def add_recommendation_if_outside_backport_window(runner_version, suggestions) + return false if runner_releases_store.releases.empty? + return false if runner_version >= runner_releases_store.releases.last # return early if runner version is too new minor_releases_with_index = runner_releases_store.releases_by_minor.keys.each_with_index.to_h runner_minor_version_index = minor_releases_with_index[runner_version.without_patch] @@ -62,14 +75,15 @@ module Gitlab outside_window = minor_releases_with_index.count - runner_minor_version_index > 3 if outside_window - recommended_release = runner_releases_store.releases_by_minor[gitlab_version.without_patch] - - recommended_release if recommended_release && recommended_release > runner_version + recommended_version = runner_releases_store.releases_by_minor[@gitlab_version.without_patch] + return add_suggestion(suggestions, runner_version, recommended_version, :recommended) end else # If unknown runner version, then recommend the latest version for the GitLab instance - recommended_runner_release_update(gitlab_version) + return add_recommended_runner_release_update(@gitlab_version, suggestions) end + + false end end end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index 263fd9d1052..d66d4b20bba 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -16,11 +16,7 @@ module Gitlab def details_path return unless can?(user, :read_pipeline, downstream_pipeline) - if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project) - project_job_path(subject.project, subject) - else - project_pipeline_path(downstream_project, downstream_pipeline) - end + project_pipeline_path(downstream_project, downstream_pipeline) end def has_action? diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml index 71f38ededd9..bb88bee9137 100644 --- a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -39,7 +39,7 @@ cache: terraform_apply: stage: provision - image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + image: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/5-minute-production-app/deploy-template/stable" extends: .needs_aws_vars resource_group: terraform before_script: @@ -53,7 +53,7 @@ terraform_apply: deploy: stage: deploy - image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + image: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/5-minute-production-app/deploy-template/stable" extends: .needs_aws_vars resource_group: deploy before_script: @@ -74,7 +74,7 @@ terraform_destroy: variables: GIT_STRATEGY: none stage: destroy - image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + image: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/5-minute-production-app/deploy-template/stable" before_script: - cp /*.tf . - cp /deploy.sh . diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index 35401e62fe2..4e011bb325d 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -6,25 +6,86 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Dart.gitlab-ci.yml -# https://hub.docker.com/r/google/dart -image: google/dart:2.8.4 +# https://hub.docker.com/_/dart +image: dart:2.17 variables: - # Use to learn more: - # pub run test --help + # To learn more go to https://dart.dev/tools/dart-test + # Or run `dart test --help` PUB_VARS: "--platform vm --timeout 30s --concurrency=6 --test-randomize-ordering-seed=random --reporter=expanded" +.use-pub-cache-bin: + # Define commands that need to be executed before each job. + before_script: + # Set PUB_CACHE either here or in the CI/CD Settings if you have multiple jobs that use dart commands. + # PUB_CACHE is used by the `dart pub` command, it needs to be set so package dependencies are stored at the project-level for CI/CD operations. + - export PUB_CACHE=".pub-cache" + - export PATH="$PATH:$HOME/$PUB_CACHE/bin" + +# Cache generated files and plugins between builds. +.upload-cache: + cache: + when: 'on_success' + paths: + - .pub-cache/bin/ + - .pub-cache/global_packages/ + - .pub-cache/hosted/ + - .dart_tool/ + - .packages + # Cache downloaded dependencies and plugins between builds. # To keep cache across branches add 'key: "$CI_JOB_NAME"' -cache: - paths: - - .pub-cache/global_packages +.download-cache: + cache: + paths: + - .dart_tool/ + - .packages + policy: pull + +install-dependencies: + stage: .pre + extends: + - .use-pub-cache-bin + - .upload-cache + script: + - dart pub get --no-precompile + +build: + stage: build + needs: + - install-dependencies + extends: + - .use-pub-cache-bin + - .upload-cache + script: + - dart pub get --offline --precompile -before_script: - - export PATH="$PATH:$HOME/.pub-cache/bin" - - pub get --no-precompile +unit-test: + stage: test + needs: + - build + extends: + - .use-pub-cache-bin + - .download-cache + script: + - dart test $PUB_VARS + +lint-test: + stage: test + needs: + - install-dependencies + extends: + - .use-pub-cache-bin + - .download-cache + script: + - dart analyze . -test: +format-test: stage: test + needs: + - install-dependencies + extends: + - .use-pub-cache-bin + - .download-cache script: - - pub run test $PUB_VARS + - dart format --set-exit-if-changed bin/ lib/ test/ diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 464b81965f2..dc55277318b 100644 --- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -42,6 +42,7 @@ lint-test-job: # This job also runs in the test stage. deploy-job: # This job runs in the deploy stage. stage: deploy # It only runs when *both* jobs in the test stage complete successfully. + environment: production script: - echo "Deploying application..." - echo "Application successfully deployed." diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml index 7f33d048c1e..34988fcdcde 100644 --- a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml @@ -31,7 +31,7 @@ default: init_and_plan: stage: build - image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 + image: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/0.13" rules: - if: $SAST_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 8c63019d743..ce227bad19a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -3,7 +3,7 @@ variables: build: stage: build - image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' + image: '${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' variables: DOCKER_TLS_CERTDIR: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 8c63019d743..ce227bad19a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -3,7 +3,7 @@ variables: build: stage: build - image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' + image: '${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' variables: DOCKER_TLS_CERTDIR: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml index 11f8376f0b4..6e8cf15204a 100644 --- a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml @@ -2,7 +2,7 @@ stages: - provision cloud_formation: - image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-cloudformation:latest' + image: '${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cloud-deploy/aws-cloudformation:latest' stage: provision script: - gl-cloudformation create-stack diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 86e3ace84c5..e278539d214 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.29" + CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:0.85.29" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index b41e92e3a56..f0ddc4b4916 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,8 +1,8 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" + image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" .common_rules: &common_rules - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME @@ -57,7 +57,7 @@ stop_dast_environment: when: always .ecs_image: - image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + image: '${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cloud-deploy/aws-ecs:latest' .ecs_rules: &ecs_rules - if: $AUTO_DEVOPS_PLATFORM_TARGET != "ECS" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index a9d9c400a34..7cbc8e40b47 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 3 diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index f9c0d4333ff..1a2a8b4edb4 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,8 +1,8 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" + image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 36f1b6981c4..cb8818357a2 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,8 +1,8 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" + image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml index ab3bc511cba..8a349f751ea 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml @@ -3,7 +3,7 @@ stages: - production .push-and-deploy: - image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ec2:latest' + image: '${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cloud-deploy/aws-ec2:latest' script: - gl-ec2 push-to-s3 - gl-ec2 deploy-to-ec2 diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index c2d31fd9669..43dc44312da 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -7,9 +7,8 @@ # then result in potentially breaking your future pipelines. # # More about including CI templates: https://docs.gitlab.com/ee/ci/yaml/#includetemplate - .ecs_image: - image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + image: '${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cloud-deploy/aws-ecs:latest' .deploy_to_ecs: extends: .ecs_image diff --git a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml index d55c126eeb7..b7735068680 100644 --- a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml @@ -3,9 +3,8 @@ # # To use, set the CI variable MIGRATE_HELM_2TO3 to "true". # For more details, go to https://docs.gitlab.com/ee/topics/autodevops/upgrading_auto_deploy_dependencies.html#helm-v3 - .helm-2to3-migrate: - image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/helm-2to3-2.17.0-3.5.3-kube-1.16.15-alpine-3.12 + image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/helm-install-image/releases/helm-2to3-2.17.0-3.5.3-kube-1.16.15-alpine-3.12" # NOTE: We use the deploy stage because: # - It exists in all versions of Auto DevOps. # - It is _empty_. @@ -54,7 +53,7 @@ done .helm-2to3-cleanup: - image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/helm-2to3-2.17.0-3.5.3-kube-1.16.15-alpine-3.12 + image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/helm-install-image/releases/helm-2to3-2.17.0-3.5.3-kube-1.16.15-alpine-3.12" stage: cleanup environment: action: prepare diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml index f7945b46a59..f8668699fe5 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. LICENSE_MANAGEMENT_VERSION: 4 diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml index b6358eb0831..c195ecd8ee5 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SAST_IMAGE_SUFFIX: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" @@ -31,7 +31,7 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index b6358eb0831..0513aae00a8 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SAST_IMAGE_SUFFIX: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" @@ -31,11 +31,14 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ when: never - - if: $CI_COMMIT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index be41553450c..dd164c00724 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SAST_IMAGE_SUFFIX: "" SAST_EXCLUDED_ANALYZERS: "" diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index f8e6e152ab9..c6938920ea4 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SAST_IMAGE_SUFFIX: "" SAST_EXCLUDED_ANALYZERS: "" @@ -46,7 +46,7 @@ bandit-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.py' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -67,7 +67,7 @@ brakeman-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.rb' - '**/Gemfile' @@ -90,7 +90,7 @@ eslint-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.html' - '**/*.js' @@ -119,7 +119,7 @@ flawfinder-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.c' - '**/*.cc' @@ -151,7 +151,7 @@ kubesec-sast: - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ when: never # Add the job to merge request pipelines if there's an open merge request. - - if: $CI_MERGE_REQUEST_IID && + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $SCAN_KUBERNETES_MANIFESTS == 'true' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never @@ -171,7 +171,7 @@ gosec-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.go' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -196,7 +196,7 @@ mobsf-android-sast: - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ when: never # Add the job to merge request pipelines if there's an open merge request. - - if: $CI_MERGE_REQUEST_IID && + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: - '**/*.apk' @@ -218,7 +218,7 @@ mobsf-ios-sast: - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ when: never # Add the job to merge request pipelines if there's an open merge request. - - if: $CI_MERGE_REQUEST_IID && + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: - '**/*.ipa' @@ -244,7 +244,7 @@ nodejs-scan-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/package.json' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -265,7 +265,7 @@ phpcs-security-audit-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.php' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -286,7 +286,7 @@ pmd-apex-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.cls' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -307,7 +307,7 @@ security-code-scan-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.csproj' - '**/*.vbproj' @@ -331,7 +331,7 @@ semgrep-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.py' - '**/*.js' @@ -366,7 +366,7 @@ sobelow-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - 'mix.exs' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -391,7 +391,7 @@ spotbugs-sast: when: never - if: $SAST_DISABLED when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.groovy' - '**/*.java' diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 3f18237a525..b7a9dbf7bc6 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -5,7 +5,7 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SECRET_DETECTION_IMAGE_SUFFIX: "" SECRETS_ANALYZER_VERSION: "4" diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml index e81e06d1a1d..e6eba6f6406 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml @@ -5,8 +5,9 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SECRET_DETECTION_IMAGE_SUFFIX: "" + SECRETS_ANALYZER_VERSION: "4" SECRET_DETECTION_EXCLUDED_PATHS: "" @@ -28,7 +29,7 @@ secret_detection: rules: - if: $SECRET_DETECTION_DISABLED when: never - - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. diff --git a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml index 64a063388b2..30767e66649 100644 --- a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml @@ -3,31 +3,45 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml -# Use this template to run MATLAB and Simulink as part of your CI/CD pipeline. The template has three jobs: +# Use this template to run MATLAB and Simulink as part of your CI/CD pipeline. The template includes three jobs: # - `command`: Run MATLAB scripts, functions, and statements. # - `test`: Run tests authored using the MATLAB unit testing framework or Simulink Test. # - `test_artifacts`: Run MATLAB and Simulink tests, and generate test and coverage artifacts. # +# The jobs in the template use the `matlab -batch` syntax to start MATLAB. The `-batch` option is supported +# in MATLAB R2019a and later. +# # You can copy and paste one or more jobs in this template into your `.gitlab-ci.yml` file. # You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. # -# - To run MATLAB and Simulink, MATLAB must be installed on the runner that will run the jobs. -# The runner will use the topmost MATLAB version on the system path. -# The build fails if the operating system cannot find MATLAB on the path. -# - The jobs in this template use the `matlab -batch` syntax to start MATLAB. The `-batch` option is supported -# in MATLAB R2019a and later. + +# Your runner must use the Docker executor to run MATLAB within a container. The [MATLAB Container on Docker Hub][1] +# lets you run your build using MATLAB R2020b or a later release. If your build requires additional toolboxes, use a +# custom MATLAB container instead. For more information on how to create and use a custom MATLAB container, +# see [Create a Custom MATLAB Container][2]. +# +# [1] https://www.mathworks.com/help/cloudcenter/ug/matlab-container-on-docker-hub.html +# [2] https://www.mathworks.com/help/cloudcenter/ug/create-a-custom-matlab-container.html + +# The jobs in this template incorporate the contents of a hidden `.matlab_defaults` job. You need to +# configure this job before running the `command`, `test`, and `test_artifacts` jobs. To configure the job: +# - Specify the name of the MATLAB container image you want to use. +# - Set the `MLM_LICENSE_FILE` environment variable using the port number and DNS address for your network license manager. +# +.matlab_defaults: + image: + name: mathworks/matlab:latest # Replace the value with the name of the MATLAB container image you want to use + entrypoint: [""] + variables: + MLM_LICENSE_FILE: 27000@MyLicenseServer # Replace the value with the port number and DNS address for your network license manager # The `command` job runs MATLAB scripts, functions, and statements. To use the job in your pipeline, # substitute `mycommand` with the code you want to run. # command: + extends: .matlab_defaults script: matlab -batch mycommand -# If the value of `mycommand` is the name of a MATLAB script or function, do not specify the file extension. -# For example, to run a script named `myscript.m` in the root of your repository, specify `mycommand` like this: -# -# "myscript" -# # If you specify more than one script, function, or statement, use a comma or semicolon to separate them. # For example, to run `myscript.m` in a folder named `myfolder` located in the root of the repository, # you can specify `mycommand` like this: @@ -36,51 +50,51 @@ command: # # MATLAB exits with exit code 0 if the specified script, function, or statement executes successfully without # error. Otherwise, MATLAB terminates with a nonzero exit code, which causes the job to fail. To have the -# job fail in certain conditions, use the [`assert`][1] or [`error`][2] functions. +# job fail in certain conditions, use the [`assert`][3] or [`error`][4] functions. # -# [1] https://www.mathworks.com/help/matlab/ref/assert.html -# [2] https://www.mathworks.com/help/matlab/ref/error.html +# [3] https://www.mathworks.com/help/matlab/ref/assert.html +# [4] https://www.mathworks.com/help/matlab/ref/error.html -# The `test` job runs the MATLAB and Simulink tests in your project. It calls the [`runtests`][3] function -# to run the tests and then the [`assertSuccess`][4] method to fail the job if any of the tests fail. +# The `test` job runs the MATLAB and Simulink tests in your project. It calls the [`runtests`][5] function +# to run the tests and then the [`assertSuccess`][6] method to fail the job if any of the tests fail. # test: + extends: .matlab_defaults script: matlab -batch "results = runtests('IncludeSubfolders',true), assertSuccess(results);" -# By default, the job includes any files in your [MATLAB Project][5] that have a `Test` label. If your repository +# By default, the job includes any files in your [MATLAB Project][7] that have a `Test` label. If your repository # does not have a MATLAB project, then the job includes all tests in the root of your repository or in any of # its subfolders. # -# [3] https://www.mathworks.com/help/matlab/ref/runtests.html -# [4] https://www.mathworks.com/help/matlab/ref/matlab.unittest.testresult.assertsuccess.html -# [5] https://www.mathworks.com/help/matlab/projects.html +# [5] https://www.mathworks.com/help/matlab/ref/runtests.html +# [6] https://www.mathworks.com/help/matlab/ref/matlab.unittest.testresult.assertsuccess.html +# [7] https://www.mathworks.com/help/matlab/projects.html # The `test_artifacts` job runs your tests and additionally generates test and coverage artifacts. -# It uses the plugin classes in the [`matlab.unittest.plugins`][6] package to generate a JUnit test results +# It uses the plugin classes in the [`matlab.unittest.plugins`][8] package to generate a JUnit test results # report and a Cobertura code coverage report. Like the `test` job, this job runs all the tests in your # project and fails the build if any of the tests fail. # test_artifacts: + extends: .matlab_defaults script: | - matlab -batch " - import matlab.unittest.TestRunner - import matlab.unittest.Verbosity - import matlab.unittest.plugins.CodeCoveragePlugin - import matlab.unittest.plugins.XMLPlugin - import matlab.unittest.plugins.codecoverage.CoberturaFormat - - suite = testsuite(pwd,'IncludeSubfolders',true); - - [~,~] = mkdir('artifacts'); - - runner = TestRunner.withTextOutput('OutputDetail',Verbosity.Detailed); - runner.addPlugin(XMLPlugin.producingJUnitFormat('artifacts/results.xml')) - runner.addPlugin(CodeCoveragePlugin.forFolder(pwd,'IncludingSubfolders',true, ... - 'Producing',CoberturaFormat('artifacts/cobertura.xml'))) - - results = runner.run(suite) - assertSuccess(results);" - + cat <<- 'BLOCK' > runAllTests.m + import matlab.unittest.TestRunner + import matlab.unittest.Verbosity + import matlab.unittest.plugins.CodeCoveragePlugin + import matlab.unittest.plugins.XMLPlugin + import matlab.unittest.plugins.codecoverage.CoberturaFormat + suite = testsuite(pwd,'IncludeSubfolders',true); + [~,~] = mkdir('artifacts') + runner = TestRunner.withTextOutput('OutputDetail',Verbosity.Detailed); + runner.addPlugin(XMLPlugin.producingJUnitFormat('artifacts/results.xml')) + % Replace `pwd` with the location of the folder containing source code + runner.addPlugin(CodeCoveragePlugin.forFolder(pwd,'IncludingSubfolders',true, ... + 'Producing',CoberturaFormat('artifacts/cobertura.xml'))) + results = runner.run(suite) + assertSuccess(results); + BLOCK + matlab -batch runAllTests artifacts: reports: junit: "./artifacts/results.xml" @@ -92,7 +106,7 @@ test_artifacts: # You can modify the contents of the `test_artifacts` job depending on your goals. For more # information on how to customize the test runner and generate various test and coverage artifacts, -# see [Generate Artifacts Using MATLAB Unit Test Plugins][7]. +# see [Generate Artifacts Using MATLAB Unit Test Plugins][9]. # -# [6] https://www.mathworks.com/help/matlab/ref/matlab.unittest.plugins-package.html -# [7] https://www.mathworks.com/help/matlab/matlab_prog/generate-artifacts-using-matlab-unit-test-plugins.html +# [8] https://www.mathworks.com/help/matlab/ref/matlab.unittest.plugins-package.html +# [9] https://www.mathworks.com/help/matlab/matlab_prog/generate-artifacts-using-matlab-unit-test-plugins.html diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index cfc4a1d904a..591eebf9cd6 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -6,7 +6,7 @@ --- # All available Hugo versions are listed here: # https://gitlab.com/pages/hugo/container_registry -image: registry.gitlab.com/pages/hugo:latest +image: "${CI_TEMPLATE_REGISTRY_HOST}/pages/hugo:latest" variables: GIT_SUBMODULE_STRATEGY: recursive diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 2fd5b409f5e..cdfa4556769 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" # FUZZAPI_VERSION: "2" FUZZAPI_IMAGE_SUFFIX: "" diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 450969fcdab..8d6c191edc4 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" # FUZZAPI_VERSION: "2" FUZZAPI_IMAGE_SUFFIX: "" diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index bec269e2933..3d7883fb87a 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:5 + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" container_scanning: image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index 893098d33c4..1b33596baa0 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" # DAST_API_VERSION: "2" DAST_API_IMAGE_SUFFIX: "" diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index 3acc3b06031..8aabf20c5df 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" # DAST_API_VERSION: "2" DAST_API_IMAGE_SUFFIX: "" diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 4a72f5e72b1..1bd527a6ec0 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -10,7 +10,7 @@ stages: - dast variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" DAST_API_VERSION: "2" DAST_API_IMAGE_SUFFIX: "" DAST_API_IMAGE: api-security diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index c71a1b1873a..701e08ba56d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -13,7 +13,7 @@ variables: DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml index d27a08db181..5b6af37977e 100644 --- a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml @@ -15,7 +15,7 @@ variables: validation: stage: dast image: - name: "registry.gitlab.com/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION" + name: "$CI_TEMPLATE_REGISTRY_HOST/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION" variables: GIT_STRATEGY: none allow_failure: false diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 3bc44fe5e1b..40060e96dff 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 10549b56856..9d3b1f4316e 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" dast: stage: dast @@ -39,6 +39,8 @@ dast: - if [ -z "$DAST_WEBSITE$DAST_API_SPECIFICATION" ]; then echo "Either DAST_WEBSITE or DAST_API_SPECIFICATION must be set. See https://docs.gitlab.com/ee/user/application_security/dast/#configuration for more details." && exit 1; fi - /analyze artifacts: + paths: + - dast_artifacts/* reports: dast: gl-dast-report.json rules: @@ -55,3 +57,7 @@ dast: $GITLAB_FEATURES =~ /\bdast\b/ - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ + after_script: + # Remove any debug.log files because they might contain secrets. + - rm -f /zap/wrk/**/debug.log + - cp -r /zap/wrk dast_artifacts diff --git a/lib/gitlab/ci/templates/Security/Fortify-FoD-sast.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Fortify-FoD-sast.gitlab-ci.yml new file mode 100644 index 00000000000..acd532132f4 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Fortify-FoD-sast.gitlab-ci.yml @@ -0,0 +1,52 @@ +# This template is provided and maintained by Fortify, an official Technology Partner with GitLab. +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. + +################################################################################################################################################ +# Fortify lets you build secure software fast with an appsec platform that automates testing throughout the DevSecOps pipeline. Fortify static,# +# dynamic, interactive, and runtime security testing is available on premises or as a service. To learn more about Fortify, start a free trial # +# or contact our sales team, visit microfocus.com/appsecurity. # +# # +# Use this pipeline template as a basis for integrating Fortify on Demand Static Application Security Testing(SAST) into your GitLab pipelines.# +# This template demonstrates the steps to prepare the code+dependencies and initiate a scan. As an option, it also supports waiting for the # +# SAST scan to complete and optinally failing the job. Software Composition Analysis can be also be performed in conjunection with the SAST # +# scan if that service has been purchased. Users should review inputs and environment variables below to configure scanning for an existing # +# application in your Fortify on Demand tenant. Additional information is available in the comments throughout the template and the Fortify on # +# Demand, FoD Uploader and ScanCentral Client product documentation. If you need additional assistance with configuration, feel free to create # +# a help ticket in the Fortify on Demand portal. # +################################################################################################################################################ + +fortify_fod_sast: + image: fortifydocker/fortify-ci-tools:3-jdk-8 + variables: + # Update/override PACKAGE_OPTS based on the ScanCentral Client documentation for your project's included tech stack(s). Helpful hints: + # ScanCentral Client will download dependencies for maven (-bt mvn) and gradle (-bt gradle). + # The current fortify-ci-tools image is Linux only at this time. Msbuild integration is not currently supported. + # ScanCentral has additional options that should be set for PHP and Python projects. + # For other build tools (-bt none), add your build commands to download necessary dependencies and prepare according to Fortify on Demand Packaging documentation. + # ScanCentral Client documentation is located at https://www.microfocus.com/documentation/fortify-software-security-center/ + PACKAGE_OPTS: "-bt mvn" + + # Update/override the FoDUploader environment variables as needed. For more information on FoDUploader commands, see https://github.com/fod-dev/fod-uploader-java. Helpful hints: + # Credentials (FOD_USERNAME, FOD_PAT, FOD_TENANT) are expected as GitLab CICD Variables in the template (masking recommended). + # Static scan settings should be configured in Fortify on Demand portal (Automated Audit preference strongly recommended). + # FOD_RELEASE_ID is expected as a GitLab CICD Variable. + # FOD_UPLOADER_OPTS can be adjusted to wait for scan completion/pull results (-I 1) and control whether to fail the job (-apf). + FOD_URL: "https://ams.fortify.com" + FOD_API_URL: "https://api.ams.fortify.com/" + FOD_UPLOADER_OPTS: "-ep 2 -pp 0" + FOD_NOTES: "Triggered by Gitlab Pipeline IID $CI_PIPELINE_IID: $CI_PIPELINE_URL" + + script: + # Package source code and dependencies using Fortify ScanCentral client + - 'scancentral package $PACKAGE_OPTS -o package.zip' + # Start Fortify on Demand SAST scan + - 'FoDUpload -z package.zip -aurl $FOD_API_URL -purl $FOD_URL -rid "$FOD_RELEASE" -tc "$FOD_TENANT" -uc "$FOD_USERNAME" "$FOD_PAT" $FOD_UPLOADER_OPTS -I 1 -n "$FOD_NOTES"' + # Generate GitLab reports + - 'FortifyVulnerabilityExporter FoDToGitLabSAST --fod.baseUrl=$FOD_URL --fod.tenant="$FOD_TENANT" --fod.userName="$FOD_USERNAME" --fod.password="$FOD_PAT" --fod.release.id=$FOD_RELEASE' + # Change to false to fail the entire pipeline if the scan fails and/or the result of a scan causes security policy failure (see "-apf" option in FoDUploader documentation) + allow_failure: true + # Report SAST vulnerabilities back to GitLab + artifacts: + reports: + sast: gl-fortify-sast.json diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index c414e70bfa3..fd04c86e6c7 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -16,7 +16,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SECURE_BINARIES_ANALYZERS: >- bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, gemnasium, gemnasium-maven, gemnasium-python, license-finder, @@ -246,7 +246,7 @@ dast-runner-validation: extends: .download_images variables: SECURE_BINARIES_ANALYZER_VERSION: "1" - SECURE_BINARIES_IMAGE: "registry.gitlab.com/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}" + SECURE_BINARIES_IMAGE: "${CI_TEMPLATE_REGISTRY_HOST}/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 6f9a9c5133c..3a956ebfc49 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -9,7 +9,7 @@ # There is a more opinionated template which we suggest the users to abide, # which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml image: - name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.1.9 + name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/terraform:1.1.9" variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 9ba009a5bca..4579f31d7ac 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -10,7 +10,7 @@ # which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml image: - name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest" variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 2b5e86f4066..488b035d189 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -4,7 +4,6 @@ # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/accessibility_testing.html - stages: - build - test @@ -13,7 +12,7 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.2.3 + image: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/accessibility:6.2.3" script: - /gitlab-accessibility.sh "$a11y_urls" allow_failure: true diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 9f24ba99201..32f64948635 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -22,7 +22,7 @@ module Gitlab @chunks_cache = [] @tell = 0 @size = calculate_size - yield self if block_given? + yield self if block end def close diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index a00c1da97ea..52673d03e69 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -72,24 +72,32 @@ module Gitlab Collection.new(@variables.reject(&block)) end - def expand_value(value, keep_undefined: false) + def expand_value(value, keep_undefined: false, expand_file_vars: true) value.gsub(Item::VARIABLES_REGEXP) do - match = Regexp.last_match - if match[:key] - # we matched variable - if variable = self[match[:key]] - variable.value - elsif keep_undefined - match[0] - end + match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%) + full_match = match[0] + variable_name = match[:key] + + next full_match unless variable_name # it is a ($$ / %%), so we don't touch it + + # now we know that it is a valid variable definition: $VARIABLE_NAME / %VARIABLE_NAME / ${VARIABLE_NAME} + + # we are trying to find a variable with key VARIABLE_NAME + variable = self[variable_name] + + if variable # VARIABLE_NAME is an existing variable + next variable.value unless variable.file? + + expand_file_vars ? variable.value : full_match + elsif keep_undefined + full_match # we do not touch the variable definition else - # we escape sequence - match[0] + nil # we remove the variable definition end end end - def sort_and_expand_all(keep_undefined: false) + def sort_and_expand_all(keep_undefined: false, expand_file_vars: true) sorted = Sort.new(self) return self.class.new(self, sorted.errors) unless sorted.valid? @@ -103,7 +111,8 @@ module Gitlab # expand variables as they are added variable = item.to_runner_variable - variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined) + variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, + expand_file_vars: expand_file_vars) new_collection.append(variable) end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 0217e6129ca..ea2aa8f2db8 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -25,6 +25,10 @@ module Gitlab @variable.fetch(:raw) end + def file? + @variable.fetch(:file) + end + def [](key) @variable.fetch(key) end diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb index 3a62f01e2e3..300b2708e6d 100644 --- a/lib/gitlab/ci/variables/helpers.rb +++ b/lib/gitlab/ci/variables/helpers.rb @@ -6,24 +6,22 @@ module Gitlab module Helpers class << self def merge_variables(current_vars, new_vars) - current_vars = transform_from_yaml_variables(current_vars) - new_vars = transform_from_yaml_variables(new_vars) + return current_vars if new_vars.blank? - transform_to_yaml_variables( - current_vars.merge(new_vars) - ) - end + current_vars = transform_to_array(current_vars) if current_vars.is_a?(Hash) + new_vars = transform_to_array(new_vars) if new_vars.is_a?(Hash) - def transform_to_yaml_variables(vars) - vars.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + (new_vars + current_vars).uniq { |var| var[:key] } end - def transform_from_yaml_variables(vars) - return vars.stringify_keys if vars.is_a?(Hash) - - vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] } + def transform_to_array(vars) + vars.to_h.map do |key, data| + if data.is_a?(Hash) + { key: key.to_s, **data.except(:key) } + else + { key: key.to_s, value: data } + end + end end def inherit_yaml_variables(from:, to:, inheritance:) @@ -35,7 +33,7 @@ module Gitlab def apply_inheritance(variables, inheritance) case inheritance when true then variables - when false then {} + when false then [] when Array then variables.select { |var| inheritance.include?(var[:key]) } end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 576fb509d47..f203f88442d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -39,11 +39,11 @@ module Gitlab end def workflow_rules - @workflow_rules ||= hash_config.dig(:workflow, :rules) + @workflow_rules ||= @ci_config.workflow_rules end def root_variables - @root_variables ||= transform_to_yaml_variables(variables) + @root_variables ||= transform_to_array(variables) end def jobs @@ -70,7 +70,7 @@ module Gitlab environment: job[:environment_name], coverage_regex: job[:coverage], # yaml_variables is calculated with using job_variables in Seed::Build - job_variables: transform_to_yaml_variables(job[:job_variables]), + job_variables: transform_to_array(job[:job_variables]), root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], @@ -114,7 +114,7 @@ module Gitlab Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: root_variables, - to: transform_to_yaml_variables(job[:job_variables]), + to: job[:job_variables], inheritance: job.fetch(:root_variables_inheritance, true) ) end @@ -133,16 +133,12 @@ module Gitlab @variables ||= @ci_config.variables end - def hash_config - @hash_config ||= @ci_config.to_hash - end - def release(job) job[:release] end - def transform_to_yaml_variables(variables) - ::Gitlab::Ci::Variables::Helpers.transform_to_yaml_variables(variables) + def transform_to_array(variables) + ::Gitlab::Ci::Variables::Helpers.transform_to_array(variables) end end end |