diff options
Diffstat (limited to 'lib/gitlab/ci')
49 files changed, 790 insertions, 121 deletions
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 6d152c052dc..ddf40296809 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -22,8 +22,7 @@ module Gitlab start_offset = @state.offset - @state.new_line!( - style: Style.new(@state.inherited_style)) + @state.new_line!(style: Style.new(**@state.inherited_style)) stream.each_line do |line| consume_line(line) diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index a500a0cc35d..a39afee194c 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -6,18 +6,31 @@ module Gitlab class Rules include ::Gitlab::Utils::StrongMemoize - Result = Struct.new(:when, :start_in, :allow_failure) do - def build_attributes + Result = Struct.new(:when, :start_in, :allow_failure, :variables) do + def build_attributes(seed_attributes = {}) { when: self.when, options: { start_in: start_in }.compact, - allow_failure: allow_failure + allow_failure: allow_failure, + yaml_variables: yaml_variables(seed_attributes[:yaml_variables]) }.compact end def pass? self.when != 'never' end + + private + + def yaml_variables(seed_variables) + return unless variables && seed_variables + + indexed_seed_variables = seed_variables.deep_dup.index_by { |var| var[:key] } + + variables.each_with_object(indexed_seed_variables) do |var, hash| + hash[var[0].to_s] = { key: var[0].to_s, value: var[1], public: true } + end.values + end end def initialize(rule_hashes, default_when:) @@ -32,7 +45,8 @@ module Gitlab Result.new( matched_rule.attributes[:when] || @default_when, matched_rule.attributes[:start_in], - matched_rule.attributes[:allow_failure] + matched_rule.attributes[:allow_failure], + matched_rule.attributes[:variables] ) else Result.new('never') diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index cbecce57163..9c2f6eea1dd 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -11,7 +11,7 @@ module Gitlab def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? - expanded_globs = expand_globs(pipeline, context) + expanded_globs = expand_globs(context) pipeline.modified_paths.any? do |path| expanded_globs.any? do |glob| File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) @@ -19,8 +19,7 @@ module Gitlab end end - def expand_globs(pipeline, context) - return @globs unless ::Feature.enabled?(:ci_variable_expansion_in_rules_changes, pipeline.project, default_enabled: true) + def expand_globs(context) return @globs unless context @globs.map do |glob| diff --git a/lib/gitlab/ci/config/entry/allow_failure.rb b/lib/gitlab/ci/config/entry/allow_failure.rb new file mode 100644 index 00000000000..de768c3a03b --- /dev/null +++ b/lib/gitlab/ci/config/entry/allow_failure.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents allow_failure settings. + # + class AllowFailure < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_KEYS = %i[exit_codes].freeze + attributes ALLOWED_KEYS + + validations do + validates :config, hash_or_boolean: true + validates :config, allowed_keys: ALLOWED_KEYS + validates :exit_codes, array_of_integers_or_integer: true, allow_nil: true + end + + def value + @config[:exit_codes] = Array.wrap(exit_codes) if exit_codes.present? + @config + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index 70fcc1d586a..e8e2eef281e 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -22,6 +22,7 @@ module Gitlab in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } + validates :allow_failure, boolean: true end validate on: :composed do @@ -47,7 +48,7 @@ module Gitlab inherit: false, metadata: { allowed_needs: %i[job bridge] } - attributes :when + attributes :when, :allow_failure def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -72,6 +73,10 @@ module Gitlab def bridge_needs needs_value[:bridge] if needs_value end + + def ignored? + allow_failure.nil? ? manual_action? : allow_failure + end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 1ce7060df22..85e3514499c 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -31,6 +31,7 @@ module Gitlab validates :dependencies, array_of_strings: true validates :resource_group, type: String + validates :allow_failure, hash_or_boolean: true end validates :start_in, duration: { limit: '1 week' }, if: :delayed? @@ -117,9 +118,14 @@ module Gitlab description: 'Parallel configuration for this job.', inherit: false + entry :allow_failure, ::Gitlab::Ci::Config::Entry::AllowFailure, + description: 'Indicates whether this job is allowed to fail or not.', + inherit: false + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, :resource_group, :release + :interruptible, :timeout, :resource_group, + :release, :allow_failure def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -166,11 +172,32 @@ module Gitlab release: release_value, after_script: after_script_value, ignore: ignored?, + allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, resource_group: resource_group, scheduling_type: needs_defined? ? :dag : :stage ).compact end + + def ignored? + allow_failure_defined? ? static_allow_failure : manual_action? + end + + private + + def allow_failure_criteria + return unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? + + if allow_failure_defined? && allow_failure_value.is_a?(Hash) + allow_failure_value + end + end + + def static_allow_failure + return false if allow_failure_value.is_a?(Hash) + + allow_failure_value + end end end end diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index abfffb7a5ed..46191eca842 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -8,7 +8,19 @@ module Gitlab strategy :JobString, if: -> (config) { config.is_a?(String) } strategy :JobHash, - if: -> (config) { config.is_a?(Hash) && config.key?(:job) && !(config.key?(:project) || config.key?(:ref)) } + if: -> (config) { config.is_a?(Hash) && same_pipeline_need?(config) } + + strategy :CrossPipelineDependency, + if: -> (config) { config.is_a?(Hash) && cross_pipeline_need?(config) } + + def self.same_pipeline_need?(config) + config.key?(:job) && + !(config.key?(:project) || config.key?(:ref) || config.key?(:pipeline)) + end + + def self.cross_pipeline_need?(config) + config.key?(:job) && config.key?(:pipeline) && !config.key?(:project) + end class JobString < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -50,6 +62,30 @@ module Gitlab end end + class CrossPipelineDependency < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[pipeline job artifacts].freeze + attributes :pipeline, :job, :artifacts + + validations do + validates :config, presence: true + validates :config, allowed_keys: ALLOWED_KEYS + validates :pipeline, type: String, presence: true + validates :job, type: String, presence: true + validates :artifacts, boolean: true, allow_nil: true + end + + def type + :cross_dependency + end + + def value + super.merge(artifacts: artifacts || artifacts.nil?) + end + end + class UnknownStrategy < ::Gitlab::Config::Entry::Node def type end diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index 66cd57b8cf3..dd01cfeedff 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -10,6 +10,8 @@ module Gitlab class Needs < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable + NEEDS_CROSS_PIPELINE_DEPENDENCIES_LIMIT = 5 + validations do validate do unless config.is_a?(Hash) || config.is_a?(Array) @@ -27,6 +29,15 @@ module Gitlab errors.add(:config, "uses invalid types: #{extra_keys.join(', ')}") end end + + validate on: :composed do + cross_dependencies = value[:cross_dependency].to_a + cross_pipeline_dependencies = cross_dependencies.select { |dep| dep[:pipeline] } + + if cross_pipeline_dependencies.size > NEEDS_CROSS_PIPELINE_DEPENDENCIES_LIMIT + errors.add(:config, "must be less than or equal to #{NEEDS_CROSS_PIPELINE_DEPENDENCIES_LIMIT}") + end + end end def value diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index c0315e5f901..5ef8cfbddb7 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -32,7 +32,6 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true - validates :allow_failure, boolean: true end end @@ -65,7 +64,7 @@ module Gitlab inherit: false, default: {} - attributes :extends, :rules, :allow_failure + attributes :extends, :rules end def compose!(deps = nil) @@ -141,10 +140,6 @@ module Gitlab def manual_action? self.when == 'manual' end - - def ignored? - allow_failure.nil? ? manual_action? : allow_failure - end end end end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 2d93f1ab06e..54ef84b965a 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -50,6 +50,7 @@ module Gitlab entry :variables, Entry::Variables, description: 'Environment variables that will be used.', + metadata: { use_value_data: true }, reserved: true entry :stages, Entry::Stages, diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 8ffd49b8a93..840f2d6f31a 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -6,14 +6,18 @@ module Gitlab module Entry class Rules::Rule < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable CLAUSES = %i[if changes exists].freeze - ALLOWED_KEYS = %i[if changes exists when start_in allow_failure].freeze + ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables].freeze ALLOWABLE_WHEN = %w[on_success on_failure always never manual delayed].freeze attributes :if, :changes, :exists, :when, :start_in, :allow_failure + entry :variables, Entry::Variables, + description: 'Environment variables to define for rule conditions.' + validations do validates :config, presence: true validates :config, type: { with: Hash } diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index e258f7128fc..dc164d752be 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -13,7 +13,8 @@ module Gitlab ALLOWED_VALUE_DATA = %i[value description].freeze validations do - validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA } + validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA }, if: :use_value_data? + validates :config, variables: true, unless: :use_value_data? end def value @@ -28,6 +29,10 @@ module Gitlab Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }] end + def use_value_data? + opt(:use_value_data) + end + private def expand_value(value) diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 661189eea50..af1df933b36 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -55,12 +55,8 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.manual_bridges_enabled?(project) - ::Feature.enabled?(:ci_manual_bridges, project, default_enabled: true) - end - - def self.auto_rollback_available?(project) - ::Feature.enabled?(:cd_auto_rollback, project) && project&.feature_available?(:auto_rollback) + def self.pipeline_open_merge_requests?(project) + ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: false) end def self.seed_block_run_before_workflow_rules_enabled?(project) @@ -70,6 +66,14 @@ module Gitlab def self.ci_pipeline_editor_page_enabled?(project) ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: false) end + + def self.allow_failure_with_exit_codes_enabled? + ::Feature.enabled?(:ci_allow_failure_with_exit_codes) + end + + def self.rules_variables_enabled?(project) + ::Feature.enabled?(:ci_rules_variables, project, default_enabled: false) + end end end end diff --git a/lib/gitlab/ci/limit.rb b/lib/gitlab/ci/limit.rb new file mode 100644 index 00000000000..c22a3c503d5 --- /dev/null +++ b/lib/gitlab/ci/limit.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + ## + # Abstract base class for CI/CD Quotas + # + class Limit + LimitExceededError = Class.new(StandardError) + + def initialize(_context, _resource) + end + + def enabled? + raise NotImplementedError + end + + def exceeded? + raise NotImplementedError + end + + def message + raise NotImplementedError + end + + def log_error!(extra_context = {}) + error = LimitExceededError.new(message) + # TODO: change this to Gitlab::ErrorTracking.log_exception(error, extra_context) + # https://gitlab.com/gitlab-org/gitlab/issues/32906 + ::Gitlab::ErrorTracking.track_exception(error, extra_context) + end + end + end +end diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb index e5a7151b823..062ced9e234 100644 --- a/lib/gitlab/ci/mask_secret.rb +++ b/lib/gitlab/ci/mask_secret.rb @@ -9,11 +9,7 @@ module Gitlab # We assume 'value' must be mutable, given # that frozen string is enabled. - ## - # TODO We need to remove this because it is going to change checksum of - # a trace. - # - value.gsub!(token, 'x' * token.length) + value.gsub!(token, 'x' * token.bytesize) value end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 0e44475607b..57f73c265b2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -10,7 +10,8 @@ module Gitlab junit: ::Gitlab::Ci::Parsers::Test::Junit, cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura, terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan, - accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y + accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y, + codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate } end diff --git a/lib/gitlab/ci/parsers/codequality/code_climate.rb b/lib/gitlab/ci/parsers/codequality/code_climate.rb new file mode 100644 index 00000000000..628d50b84cb --- /dev/null +++ b/lib/gitlab/ci/parsers/codequality/code_climate.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Codequality + class CodeClimate + def parse!(json_data, codequality_report) + root = Gitlab::Json.parse(json_data) + + parse_all(root, codequality_report) + rescue JSON::ParserError => e + codequality_report.set_error_message("JSON parsing failed: #{e}") + end + + private + + def parse_all(root, codequality_report) + return unless root.present? + + root.each do |degradation| + break unless codequality_report.add_degradation(degradation) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index 934c797580c..1edcbac2f25 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -5,50 +5,113 @@ module Gitlab module Parsers module Coverage class Cobertura - CoberturaParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + InvalidXMLError = Class.new(Gitlab::Ci::Parsers::ParserError) + InvalidLineInformationError = Class.new(Gitlab::Ci::Parsers::ParserError) - def parse!(xml_data, coverage_report) + GO_SOURCE_PATTERN = '/usr/local/go/src' + MAX_SOURCES = 100 + + def parse!(xml_data, coverage_report, project_path: nil, worktree_paths: nil) root = Hash.from_xml(xml_data) - parse_all(root, coverage_report) + context = { + project_path: project_path, + paths: worktree_paths&.to_set, + sources: [] + } + + parse_all(root, coverage_report, context) rescue Nokogiri::XML::SyntaxError - raise CoberturaParserError, "XML parsing failed" - rescue - raise CoberturaParserError, "Cobertura parsing failed" + raise InvalidXMLError, "XML parsing failed" end private - def parse_all(root, coverage_report) + def parse_all(root, coverage_report, context) return unless root.present? root.each do |key, value| - parse_node(key, value, coverage_report) + parse_node(key, value, coverage_report, context) end end - def parse_node(key, value, coverage_report) - return if key == 'sources' - - if key == 'class' + def parse_node(key, value, coverage_report, context) + if key == 'sources' && value['source'].present? + parse_sources(value['source'], context) + elsif key == 'package' Array.wrap(value).each do |item| - parse_class(item, coverage_report) + parse_package(item, coverage_report, context) + end + elsif key == 'class' + # This means the cobertura XML does not have classes within package nodes. + # This is possible in some cases like in simple JS project structures + # running Jest. + Array.wrap(value).each do |item| + parse_class(item, coverage_report, context) end elsif value.is_a?(Hash) - parse_all(value, coverage_report) + parse_all(value, coverage_report, context) elsif value.is_a?(Array) value.each do |item| - parse_all(item, coverage_report) + parse_all(item, coverage_report, context) end end end - def parse_class(file, coverage_report) + def parse_sources(sources, context) + return unless context[:project_path] && context[:paths] + + sources = Array.wrap(sources) + + # TODO: Go cobertura has a different format with how their packages + # are included in the filename. So we can't rely on the sources. + # We'll deal with this later. + return if sources.include?(GO_SOURCE_PATTERN) + + sources.each do |source| + source = build_source_path(source, context) + context[:sources] << source if source.present? + end + end + + def build_source_path(source, context) + # | raw source | extracted | + # |-----------------------------|------------| + # | /builds/foo/test/SampleLib/ | SampleLib/ | + # | /builds/foo/test/something | something | + # | /builds/foo/test/ | nil | + # | /builds/foo/test | nil | + source.split("#{context[:project_path]}/", 2)[1] + end + + def parse_package(package, coverage_report, context) + classes = package.dig('classes', 'class') + return unless classes.present? + + matched_filenames = Array.wrap(classes).map do |item| + parse_class(item, coverage_report, context) + end + + # Remove these filenames from the paths to avoid conflict + # with other packages that may contain the same class filenames + remove_matched_filenames(matched_filenames, context) + end + + def remove_matched_filenames(filenames, context) + return unless context[:paths] + + filenames.each { |f| context[:paths].delete(f) } + end + + def parse_class(file, coverage_report, context) return unless file["filename"].present? && file["lines"].present? parsed_lines = parse_lines(file["lines"]) + filename = determine_filename(file["filename"], context) + + coverage_report.add_file(filename, Hash[parsed_lines]) if filename - coverage_report.add_file(file["filename"], Hash[parsed_lines]) + filename end def parse_lines(lines) @@ -58,6 +121,27 @@ module Gitlab # Using `Integer()` here to raise exception on invalid values [Integer(line["number"]), Integer(line["hits"])] end + rescue + raise InvalidLineInformationError, "Line information had invalid values" + end + + def determine_filename(filename, context) + return filename unless context[:sources].any? + + full_filename = nil + + context[:sources].each_with_index do |source, index| + break if index >= MAX_SOURCES + break if full_filename = check_source(source, filename, context) + end + + full_filename + end + + def check_source(source, filename, context) + full_path = File.join(source, filename) + + return full_path if context[:paths].include?(full_path) end end end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index a864c843dd8..2ca51930c19 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -35,7 +35,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def pipelines - if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: false) + if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: true) project.all_pipelines.ci_and_parent_sources else project.ci_pipelines diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 06096a33f27..d05be54267c 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -12,7 +12,7 @@ module Gitlab :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: - :config_content, :yaml_processor_result, :stage_seeds + :config_content, :yaml_processor_result, :pipeline_seed ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/limit/deployments.rb b/lib/gitlab/ci/pipeline/chain/limit/deployments.rb new file mode 100644 index 00000000000..d684eedcaac --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/deployments.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class Deployments < Chain::Base + extend ::Gitlab::Utils::Override + include ::Gitlab::Ci::Pipeline::Chain::Helpers + + attr_reader :limit + private :limit + + def initialize(*) + super + + @limit = ::Gitlab::Ci::Pipeline::Quota::Deployments + .new(project.namespace, pipeline, command) + end + + override :perform! + def perform! + return unless limit.exceeded? + + limit.log_error!(project_id: project.id, plan: project.actual_plan_name) + error(limit.message, drop_reason: :deployments_limit_exceeded) + end + + override :break? + def break? + limit.exceeded? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index f9ae37aa273..654e24be8e1 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -10,12 +10,12 @@ module Gitlab PopulateError = Class.new(StandardError) def perform! - raise ArgumentError, 'missing stage seeds' unless @command.stage_seeds + raise ArgumentError, 'missing pipeline seed' unless @command.pipeline_seed ## # Populate pipeline with all stages, and stages with builds. # - pipeline.stages = @command.stage_seeds.map(&:to_resource) + pipeline.stages = @command.pipeline_seed.stages if stage_names.empty? return error('No stages / jobs for this pipeline.') diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index ba86b08d209..083f0bec1df 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -29,11 +29,11 @@ module Gitlab ## # Gather all runtime build/stage errors # - if stage_seeds_errors - return error(stage_seeds_errors.join("\n"), config_error: true) + if pipeline_seed.errors + return error(pipeline_seed.errors.join("\n"), config_error: true) end - @command.stage_seeds = stage_seeds + @command.pipeline_seed = pipeline_seed end def break? @@ -42,24 +42,12 @@ module Gitlab private - def stage_seeds_errors - stage_seeds.flat_map(&:errors).compact.presence - end - - def stage_seeds - strong_memoize(:stage_seeds) do - seeds = stages_attributes.inject([]) do |previous_stages, attributes| - seed = Gitlab::Ci::Pipeline::Seed::Stage.new(pipeline, attributes, previous_stages) - previous_stages + [seed] - end - - seeds.select(&:included?) + def pipeline_seed + strong_memoize(:pipeline_seed) do + stages_attributes = @command.yaml_processor_result.stages_attributes + Gitlab::Ci::Pipeline::Seed::Pipeline.new(pipeline, stages_attributes) end end - - def stages_attributes - @command.yaml_processor_result.stages_attributes - end end end end diff --git a/lib/gitlab/ci/pipeline/quota/deployments.rb b/lib/gitlab/ci/pipeline/quota/deployments.rb new file mode 100644 index 00000000000..ed32d0d3d49 --- /dev/null +++ b/lib/gitlab/ci/pipeline/quota/deployments.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Quota + class Deployments < ::Gitlab::Ci::Limit + include ::Gitlab::Utils::StrongMemoize + include ActionView::Helpers::TextHelper + + def initialize(namespace, pipeline, command) + @namespace = namespace + @pipeline = pipeline + @command = command + end + + def enabled? + limit > 0 + end + + def exceeded? + return false unless enabled? + + pipeline_deployment_count > limit + end + + def message + return unless exceeded? + + "Pipeline has too many deployments! Requested #{pipeline_deployment_count}, but the limit is #{limit}." + end + + private + + def pipeline_deployment_count + strong_memoize(:pipeline_deployment_count) do + @command.pipeline_seed.deployments_count + end + end + + def limit + strong_memoize(:limit) do + @namespace.actual_limits.ci_pipeline_deployments + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 91dbcc616ea..2271915a72b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -60,6 +60,7 @@ module Gitlab @seed_attributes .deep_merge(pipeline_attributes) .deep_merge(rules_attributes) + .deep_merge(allow_failure_criteria_attributes) .deep_merge(cache_attributes) end @@ -154,9 +155,15 @@ module Gitlab end def rules_attributes - return {} unless @using_rules + strong_memoize(:rules_attributes) do + next {} unless @using_rules - rules_result.build_attributes + if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) + rules_result.build_attributes(@seed_attributes) + else + rules_result.build_attributes + end + end end def rules_result @@ -176,6 +183,17 @@ module Gitlab @cache.build_attributes end end + + # If a job uses `allow_failure:exit_codes` and `rules:allow_failure` + # we need to prevent the exit codes from being persisted because they + # would break the behavior defined by `rules:allow_failure`. + def allow_failure_criteria_attributes + return {} unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? + return {} if rules_attributes[:allow_failure].nil? + return {} unless @seed_attributes.dig(:options, :allow_failure_criteria) + + { options: { allow_failure_criteria: nil } } + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index b20dc383419..5dff0788ec9 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -24,9 +24,7 @@ module Gitlab end def auto_stop_in - if Feature.enabled?(:environment_auto_stop_start_on_create) - job.environment_auto_stop_in - end + job.environment_auto_stop_in end def expanded_environment_name diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb new file mode 100644 index 00000000000..da9d853cf68 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Pipeline + include Gitlab::Utils::StrongMemoize + + def initialize(pipeline, stages_attributes) + @pipeline = pipeline + @stages_attributes = stages_attributes + end + + def errors + stage_seeds.flat_map(&:errors).compact.presence + end + + def stages + stage_seeds.map(&:to_resource) + end + + def size + stage_seeds.sum(&:size) + end + + def deployments_count + stage_seeds.sum do |stage_seed| + stage_seed.seeds.count do |build_seed| + build_seed.attributes[:environment].present? + end + end + end + + private + + def stage_seeds + strong_memoize(:stage_seeds) do + seeds = @stages_attributes.inject([]) do |previous_stages, attributes| + seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@pipeline, attributes, previous_stages) + previous_stages + [seed] + end + + seeds.select(&:included?) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb index 210eb17f2d3..ab048672b22 100644 --- a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb +++ b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb @@ -3,52 +3,43 @@ module Gitlab module Ci module Reports - class AccessibilityReportsComparer - include Gitlab::Utils::StrongMemoize - - STATUS_SUCCESS = 'success' - STATUS_FAILED = 'failed' - - attr_reader :base_reports, :head_reports - - def initialize(base_reports, head_reports) - @base_reports = base_reports || AccessibilityReports.new - @head_reports = head_reports + class AccessibilityReportsComparer < ReportsComparer + def initialize(base_report, head_report) + @base_report = base_report || AccessibilityReports.new + @head_report = head_report end - def status - head_reports.errors_count > 0 ? STATUS_FAILED : STATUS_SUCCESS + def success? + head_report.errors_count == 0 end def existing_errors strong_memoize(:existing_errors) do - base_reports.all_errors + base_report.all_errors & head_report.all_errors end end def new_errors strong_memoize(:new_errors) do - head_reports.all_errors - base_reports.all_errors + head_report.all_errors - base_report.all_errors end end def resolved_errors strong_memoize(:resolved_errors) do - base_reports.all_errors - head_reports.all_errors + base_report.all_errors - head_report.all_errors end end - def errors_count - head_reports.errors_count - end - def resolved_count resolved_errors.size end def total_count - existing_errors.size + new_errors.size + head_report.errors_count end + + alias_method :errors_count, :total_count end end end diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb new file mode 100644 index 00000000000..060a1e2399b --- /dev/null +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CodequalityReports + attr_reader :degradations, :error_message + + CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s + + def initialize + @degradations = {}.with_indifferent_access + @error_message = nil + end + + def add_degradation(degradation) + valid_degradation?(degradation) && @degradations[degradation.dig('fingerprint')] = degradation + end + + def set_error_message(error) + @error_message = error + end + + def degradations_count + @degradations.size + end + + def all_degradations + @degradations.values + end + + private + + def valid_degradation?(degradation) + JSON::Validator.validate!(CODECLIMATE_SCHEMA_PATH, degradation) + rescue JSON::Schema::ValidationError => e + set_error_message("Invalid degradation format: #{e.message}") + false + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb new file mode 100644 index 00000000000..88e02cd9004 --- /dev/null +++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CodequalityReportsComparer < ReportsComparer + def initialize(base_report, head_report) + @base_report = base_report || CodequalityReports.new + @head_report = head_report + end + + def success? + head_report.degradations_count == 0 + end + + def existing_errors + strong_memoize(:existing_errors) do + base_report.all_degradations & head_report.all_degradations + end + end + + def new_errors + strong_memoize(:new_errors) do + fingerprints = head_report.degradations.keys - base_report.degradations.keys + head_report.degradations.fetch_values(*fingerprints) + end + end + + def resolved_errors + strong_memoize(:resolved_errors) do + fingerprints = base_report.degradations.keys - head_report.degradations.keys + base_report.degradations.fetch_values(*fingerprints) + end + end + + def resolved_count + resolved_errors.size + end + + def total_count + head_report.degradations_count + end + + alias_method :errors_count, :total_count + end + end + end +end diff --git a/lib/gitlab/ci/reports/reports_comparer.rb b/lib/gitlab/ci/reports/reports_comparer.rb new file mode 100644 index 00000000000..d413d3a74f6 --- /dev/null +++ b/lib/gitlab/ci/reports/reports_comparer.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class ReportsComparer + include Gitlab::Utils::StrongMemoize + + STATUS_SUCCESS = 'success' + STATUS_FAILED = 'failed' + + attr_reader :base_report, :head_report + + def initialize(base_report, head_report) + @base_report = base_report + @head_report = head_report + end + + def status + success? ? STATUS_SUCCESS : STATUS_FAILED + end + + def success? + raise NotImplementedError + end + + def existing_errors + raise NotImplementedError + end + + def new_errors + raise NotImplementedError + end + + def resolved_errors + raise NotImplementedError + end + + def errors_count + raise NotImplementedError + end + + def resolved_count + resolved_errors.size + end + + def total_count + existing_errors.size + new_errors.size + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index fe23641802b..2ae9730ec1a 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18-gitlab.1" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 385959389de..e5b40e5f49a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml index 3b87d53f165..895e6e8ea6d 100644 --- a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml @@ -2,6 +2,8 @@ test: variables: POSTGRES_VERSION: 9.6.16 POSTGRES_DB: test + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password services: - "postgres:${POSTGRES_VERSION}" stage: test diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 3f62d92ad13..23dfeda31cc 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.34.1" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.36.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 65abee1f5eb..3faf07546de 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ayufan/openshift-cli +image: openshift/origin-cli stages: - build # dummy stage to follow the template guidelines diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 0ae8fd833c4..135f0df99fe 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -15,7 +15,8 @@ variables: FUZZAPI_VERSION: latest FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml FUZZAPI_TIMEOUT: 30 - FUZZAPI_REPORT: gl-api-fuzzing-report.xml + FUZZAPI_REPORT: gl-api-fuzzing-report.json + FUZZAPI_REPORT_ASSET_PATH: assets # FUZZAPI_D_NETWORK: testing-net # @@ -45,6 +46,7 @@ apifuzzer_fuzz: variables: FUZZAPI_PROJECT: $CI_PROJECT_PATH FUZZAPI_API: http://apifuzzer:80 + FUZZAPI_NEW_REPORT: 1 TZ: America/Los_Angeles services: - name: $FUZZAPI_IMAGE @@ -61,7 +63,7 @@ apifuzzer_fuzz: - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ script: # # Validate options @@ -75,6 +77,9 @@ apifuzzer_fuzz: # Run user provided pre-script - sh -c "$FUZZAPI_PRE_SCRIPT" # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # # Start scanning - worker-entry # @@ -82,8 +87,12 @@ apifuzzer_fuzz: - sh -c "$FUZZAPI_POST_SCRIPT" # artifacts: + when: always + paths: + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT reports: - junit: $FUZZAPI_REPORT + api_fuzzing: $FUZZAPI_REPORT apifuzzer_fuzz_dnd: stage: fuzz @@ -102,7 +111,7 @@ apifuzzer_fuzz_dnd: - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ services: - docker:19.03.12-dind script: @@ -115,6 +124,9 @@ apifuzzer_fuzz_dnd: # Run user provided pre-script - sh -c "$FUZZAPI_PRE_SCRIPT" # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # # Start peach testing engine container - | docker run -d \ @@ -155,6 +167,8 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ -e FUZZAPI_HAR \ -e FUZZAPI_OPENAPI \ -e FUZZAPI_POSTMAN_COLLECTION \ @@ -168,6 +182,8 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_SERVICE_START_TIMEOUT \ -e FUZZAPI_HTTP_USERNAME \ -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ $FUZZAPI_D_WORKER_ENV \ $FUZZAPI_D_WORKER_PORTS \ @@ -193,6 +209,8 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ -e FUZZAPI_HAR \ -e FUZZAPI_OPENAPI \ -e FUZZAPI_POSTMAN_COLLECTION \ @@ -206,7 +224,10 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_SERVICE_START_TIMEOUT \ -e FUZZAPI_HTTP_USERNAME \ -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ -v $CI_PROJECT_DIR:/app \ + -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ -p 81:80 \ -p 8001:8000 \ -p 515:514 \ @@ -239,7 +260,9 @@ apifuzzer_fuzz_dnd: paths: - ./gl-api_fuzzing*.log - ./gl-api_fuzzing*.zip + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT reports: - junit: $FUZZAPI_REPORT + api_fuzzing: $FUZZAPI_REPORT # end diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 3cbde9d30c8..5ea2363a0c5 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: container_scanning: stage: test - image: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION + image: "$CS_ANALYZER_IMAGE" variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes @@ -18,6 +18,7 @@ container_scanning: # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none + CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION allow_failure: true services: - name: $CLAIR_DB_IMAGE diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index a1b6dc2cc1b..9d47537c0f0 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -12,7 +12,7 @@ variables: coverage_fuzzing_unlicensed: - stage: test + stage: .pre allow_failure: true rules: - if: $GITLAB_FEATURES !~ /\bcoverage_fuzzing\b/ && $COVFUZZ_DISABLED == null diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml new file mode 100644 index 00000000000..a0564a16c07 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -0,0 +1,24 @@ +stages: + - build + - test + - deploy + - dast + +variables: + DAST_VERSION: 1 + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + +dast: + stage: dast + image: + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + variables: + GIT_STRATEGY: none + allow_failure: true + script: + - /analyze + artifacts: + reports: + dast: gl-dast-report.json diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 3789f0edc1c..b534dad9593 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -28,11 +28,8 @@ dependency_scanning: .ds-analyzer: extends: dependency_scanning allow_failure: true - rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 script: - /analyzer run diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a51cb61da6d..f4ee8ebd47e 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -30,10 +30,8 @@ sast: .sast-analyzer: extends: sast allow_failure: true - rules: - - if: $SAST_DISABLED - when: never - - if: $CI_COMMIT_BRANCH + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 script: - /analyzer run @@ -175,7 +173,7 @@ nodejs-scan-sast: - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ exists: - - 'package.json' + - '**/package.json' phpcs-security-audit-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index 6ebff102ccb..8ca1d2e08ba 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -14,6 +14,9 @@ variables: stage: test image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" services: [] + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 artifacts: reports: secret_detection: gl-secret-detection-report.json diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index e455bfac9de..910e711f046 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -56,5 +56,6 @@ cache: .destroy: &destroy stage: cleanup script: + - cd ${TF_ROOT} - gitlab-terraform destroy when: manual diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 0a739cf122d..035ba52da84 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -55,5 +55,5 @@ publish_package: npm publish && echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" } || { - echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry."; exit 1 + echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry." } diff --git a/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml new file mode 100644 index 00000000000..536cf9bd8d8 --- /dev/null +++ b/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml @@ -0,0 +1,41 @@ +publish: + image: node:latest + stage: deploy + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^v\d+\.\d+\.\d+.*$/ + changes: + - package.json + script: + # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry + - | + if [[ ! -f .npmrc ]]; then + echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' + { + echo "@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" + echo "${CI_API_V4_URL#http*:}/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=\${CI_JOB_TOKEN}" + } >> .npmrc + fi + - echo "Created the following .npmrc:"; cat .npmrc + + # Extract a few values from package.json + - NPM_PACKAGE_NAME=$(node -p "require('./package.json').name") + - NPM_PACKAGE_VERSION=$(node -p "require('./package.json').version") + + # Validate that the package name is properly scoped to the project's root namespace. + # For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention + - | + if [[ ! $NPM_PACKAGE_NAME =~ ^@$CI_PROJECT_ROOT_NAMESPACE/ ]]; then + echo "Invalid package scope! Packages must be scoped in the root namespace of the project, e.g. \"@${CI_PROJECT_ROOT_NAMESPACE}/${CI_PROJECT_NAME}\"" + echo 'For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention' + exit 1 + fi + + # Compare the version in package.json to all published versions. + # If the package.json version has not yet been published, run `npm publish`. + - | + if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then + npm publish + echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" + else + echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." + fi diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb index 62532ef1cd2..7cdb6a6c03c 100644 --- a/lib/gitlab/ci/trace/checksum.rb +++ b/lib/gitlab/ci/trace/checksum.rb @@ -64,10 +64,33 @@ module Gitlab end end + def state_bytesize + strong_memoize(:state_bytesize) do + build.pending_state&.trace_bytesize + end + end + + def trace_size + strong_memoize(:trace_size) do + trace_chunks.sum { |chunk| chunk_size(chunk) } + end + end + + def corrupted? + return false unless has_bytesize? + return false if valid? + + state_bytesize.to_i != trace_size.to_i + end + def chunks_count trace_chunks.to_a.size end + def has_bytesize? + state_bytesize.present? + end + private def chunk_size(chunk) diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index 097436d84ea..ce9efbda7ea 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -18,7 +18,8 @@ module Gitlab :conflict, # runner has sent unrecognized build state details :locked, # build trace has been locked by a different mechanism :stalled, # failed to migrate chunk due to a worker duplication - :invalid # malformed build trace has been detected using CRC32 + :invalid, # invalid build trace has been detected using CRC32 + :corrupted # malformed trace found after comparing CRC32 and size ].freeze def increment_trace_operation(operation: :unknown) diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 52a00e41214..cd7d781a574 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -77,6 +77,7 @@ module Gitlab options: { image: job[:image], services: job[:services], + allow_failure_criteria: job[:allow_failure_criteria], artifacts: job[:artifacts], dependencies: job[:dependencies], cross_dependencies: job.dig(:needs, :cross_dependency), |