diff options
Diffstat (limited to 'lib/gitlab/ci')
52 files changed, 536 insertions, 175 deletions
diff --git a/lib/gitlab/ci/ansi2json.rb b/lib/gitlab/ci/ansi2json.rb index 79114d35916..70b68c7b821 100644 --- a/lib/gitlab/ci/ansi2json.rb +++ b/lib/gitlab/ci/ansi2json.rb @@ -4,8 +4,8 @@ module Gitlab module Ci module Ansi2json - def self.convert(ansi, state = nil) - Converter.new.convert(ansi, state) + def self.convert(ansi, state = nil, verify_state: false) + Converter.new.convert(ansi, state, verify_state: verify_state) end end end diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 78f6c5bf0aa..84541208a2f 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -4,9 +4,13 @@ module Gitlab module Ci module Ansi2json class Converter - def convert(stream, new_state) + def convert(stream, new_state, verify_state: false) @lines = [] - @state = State.new(new_state, stream.size) + @state = if verify_state + SignedState.new(new_state, stream.size) + else + State.new(new_state, stream.size) + end append = false truncated = false diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb index fdd49df1e24..1d26bceb7b1 100644 --- a/lib/gitlab/ci/ansi2json/parser.rb +++ b/lib/gitlab/ci/ansi2json/parser.rb @@ -9,14 +9,14 @@ module Gitlab class Parser # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) COLOR = { - 0 => 'black', # not that this is gray in the intense color table + 0 => 'black', # Note: This is gray in the intense color table. 1 => 'red', 2 => 'green', 3 => 'yellow', 4 => 'blue', 5 => 'magenta', 6 => 'cyan', - 7 => 'white' # not that this is gray in the dark (aka default) color table + 7 => 'white' # Note: This is gray in the dark (aka default) color table. }.freeze STYLE_SWITCHES = { diff --git a/lib/gitlab/ci/ansi2json/signed_state.rb b/lib/gitlab/ci/ansi2json/signed_state.rb new file mode 100644 index 00000000000..98e2419f0a8 --- /dev/null +++ b/lib/gitlab/ci/ansi2json/signed_state.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'openssl' + +module Gitlab + module Ci + module Ansi2json + class SignedState < ::Gitlab::Ci::Ansi2json::State + include Gitlab::Utils::StrongMemoize + + SIGNATURE_KEY_SALT = 'gitlab-ci-ansi2json-state' + SEPARATOR = '--' + + def encode + encoded = super + + encoded + SEPARATOR + sign(encoded) + end + + private + + def sign(message) + ::OpenSSL::HMAC.hexdigest( + signature_digest, + signature_key, + message + ) + end + + def verify(signed_message) + signature_length = signature_digest.digest_length * 2 # a byte is exactly two hexadecimals + message_length = signed_message.length - SEPARATOR.length - signature_length + return if message_length <= 0 + + signature = signed_message.last(signature_length) + message = signed_message.first(message_length) + return unless valid_signature?(message, signature) + + message + end + + def valid_signature?(message, signature) + expected_signature = sign(message) + expected_signature.bytesize == signature.bytesize && + ::OpenSSL.fixed_length_secure_compare(signature, expected_signature) + end + + def decode_state(data) + return if data.blank? + + encoded_state = verify(data) + if encoded_state.blank? + ::Gitlab::AppLogger.warn(message: "#{self.class}: signature missing or invalid", invalid_state: data) + return + end + + decoded_state = Base64.urlsafe_decode64(encoded_state) + return unless decoded_state.present? + + ::Gitlab::Json.parse(decoded_state) + end + + def signature_digest + ::OpenSSL::Digest.new('SHA256') + end + + def signature_key + ::Gitlab::Application.key_generator.generate_key(SIGNATURE_KEY_SALT, signature_digest.block_length) + end + strong_memoize_attr :signature_key + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index b2b6ce649ed..279e1929b22 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -18,12 +18,13 @@ module Gitlab end def encode - state = { + json = { offset: @last_line_offset, style: @current_line.style.to_h, open_sections: @open_sections - } - Base64.urlsafe_encode64(state.to_json) + }.to_json + + Base64.urlsafe_encode64(json, padding: false) end def open_section(section, timestamp, options) @@ -91,7 +92,20 @@ module Gitlab decoded_state = Base64.urlsafe_decode64(state) return unless decoded_state.present? - Gitlab::Json.parse(decoded_state) + ::Gitlab::Json.parse(decoded_state) + rescue ArgumentError, JSON::ParserError => error + # This rescue is so that we don't break during the rollout or rollback + # of `sign_and_verify_ansi2json_state`, because we may receive a + # signed state even when the flag is disabled, and this would result + # in invalid Base64 (ArgumentError) or invalid JSON in case the signed + # state happens to decode as valid Base64 (JSON::ParserError). + # + # Once the flag has been fully rolled out this should not + # be possible (it would imply a backend bug) and we not rescue from + # this. + ::Gitlab::AppLogger.warn(message: "#{self.class}: decode error", invalid_state: state, error: error) + + nil end end end diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb index 1cddc9fcc98..3432ecdb250 100644 --- a/lib/gitlab/ci/build/cache.rb +++ b/lib/gitlab/ci/build/cache.rb @@ -9,8 +9,15 @@ module Gitlab def initialize(cache, pipeline) cache = Array.wrap(cache) @cache = cache.map.with_index do |cache, index| - Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache, index) + if Feature.enabled?(:ci_fix_for_runner_cache_prefix) + prefix = cache_prefix(cache, index) + + Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache, prefix) + else + Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache, index) + end end end @@ -23,6 +30,18 @@ module Gitlab end end end + + private + + def cache_prefix(cache, index) + files = cache.dig(:key, :files) if cache.is_a?(Hash) && cache[:key].is_a?(Hash) + + return index if files.blank? + + filenames = files.map { |file| file.split('.').first }.join('_') + + "#{index}_#{filenames}" + end end end end diff --git a/lib/gitlab/ci/components/header.rb b/lib/gitlab/ci/components/header.rb deleted file mode 100644 index 732874d7a88..00000000000 --- a/lib/gitlab/ci/components/header.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Components - ## - # Components::Header class represents full component specification that is being prepended as first YAML document - # in the CI Component file. - # - class Header - attr_reader :errors - - def initialize(header) - @header = header - @errors = [] - end - - def empty? - inputs_spec.to_h.empty? - end - - def inputs(args) - @input ||= Ci::Input::Inputs.new(inputs_spec, args) - end - - def context(args) - inputs(args).then do |input| - raise ArgumentError unless input.valid? - - Ci::Interpolation::Context.new({ inputs: input.to_hash }) - end - end - - private - - def inputs_spec - @header.dig(:spec, :inputs) - end - end - end - end -end diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index 010ce57d2a0..27a7611ffdd 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -6,6 +6,8 @@ module Gitlab class InstancePath include Gitlab::Utils::StrongMemoize + LATEST_VERSION_KEYWORD = '~latest' + def self.match?(address) address.include?('@') && address.start_with?(Settings.gitlab_ci['component_fqdn']) end @@ -39,9 +41,9 @@ module Gitlab File.join(component_dir, @content_filename).delete_prefix('/') end - # TODO: Add support when version is a released tag and "~latest" moving target def sha return unless project + return latest_version_sha if version == LATEST_VERSION_KEYWORD project.commit(version)&.id end @@ -69,6 +71,12 @@ module Gitlab ::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord end + + def latest_version_sha + return unless catalog_resource = project&.catalog_resource + + catalog_resource.latest_version&.sha + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 534b84afc23..0c293c3f0ef 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize ConfigError = Class.new(StandardError) - TIMEOUT_SECONDS = 30.seconds + TIMEOUT_SECONDS = ENV.fetch('GITLAB_CI_CONFIG_FETCH_TIMEOUT_SECONDS', 30).to_i.clamp(0, 60).seconds TIMEOUT_MESSAGE = 'Request timed out when fetching configuration files.' RESCUE_ERRORS = [ diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 2390ba05916..d31d1b366c3 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks environment coverage retry parallel interruptible timeout - release id_tokens].freeze + release id_tokens publish].freeze validations do validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS @@ -45,6 +45,8 @@ module Gitlab errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any? end end + + validates :publish, absence: { message: "can only be used within a `pages` job" }, unless: -> { pages_job? } end entry :before_script, Entry::Commands, @@ -125,10 +127,14 @@ module Gitlab inherit: false, metadata: { composable_class: ::Gitlab::Ci::Config::Entry::IdToken } + entry :publish, Entry::Publish, + description: 'Path to be published with Pages', + inherit: false + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, - :release, :allow_failure + :release, :allow_failure, :publish def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -169,7 +175,8 @@ module Gitlab allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, scheduling_type: needs_defined? ? :dag : :stage, - id_tokens: id_tokens_value + id_tokens: id_tokens_value, + publish: publish ).compact end @@ -177,6 +184,10 @@ module Gitlab allow_failure_defined? ? static_allow_failure : manual_action? end + def pages_job? + name == :pages + end + def self.allowed_keys ALLOWED_KEYS end diff --git a/lib/gitlab/ci/config/entry/publish.rb b/lib/gitlab/ci/config/entry/publish.rb new file mode 100644 index 00000000000..52a2487009e --- /dev/null +++ b/lib/gitlab/ci/config/entry/publish.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the path to be published with Pages. + # + class Publish < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String + end + + def self.default + 'public' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 0b90d240a15..273d78bd583 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -22,7 +22,7 @@ module Gitlab strong_memoize(:content) do Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) rescue Gitlab::Ci::ArtifactFileReader::Error => error - errors.push(error.message) + errors.push(error.message) # TODO this memoizes the error message as a content! end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 7060754a670..553f2a2d754 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -61,18 +61,6 @@ module Gitlab [params, context.project&.full_path, context.sha].hash end - def load_and_validate_expanded_hash! - context.logger.instrument(:config_file_fetch_content_hash) do - content_hash # calling the method loads then memoizes the result - end - - context.logger.instrument(:config_file_expand_content_includes) do - expanded_content_hash # calling the method expands then memoizes the result - end - - validate_hash! - end - # This method is overridden to load context into the memoized result # or to lazily load context via BatchLoader def preload_context @@ -94,32 +82,59 @@ module Gitlab end def validate_context! - raise NotImplementedError, 'subclass must implement validate_context' + raise NotImplementedError, 'subclass must implement `validate_context!`' end def validate_content! - if content.blank? - errors.push("Included file `#{masked_location}` is empty or does not exist!") + errors.push("Included file `#{masked_location}` is empty or does not exist!") if content.blank? + end + + def load_and_validate_expanded_hash! + context.logger.instrument(:config_file_fetch_content_hash) do + content_result # calling the method loads YAML then memoizes the content result + end + + context.logger.instrument(:config_file_interpolate_result) do + interpolator.interpolate! + end + + return validate_interpolation! unless interpolator.valid? + + context.logger.instrument(:config_file_expand_content_includes) do + expanded_content_hash # calling the method expands then memoizes the result end + + validate_hash! end protected def content_result - strong_memoize(:content_hash) do - ::Gitlab::Ci::Config::Yaml - .load_result!(content, project: context.project) - end + ::Gitlab::Ci::Config::Yaml + .load_result!(content, project: context.project) + end + strong_memoize_attr :content_result + + def content_inputs + params.to_h[:with] end + strong_memoize_attr :content_inputs def content_hash - return unless content_result.valid? + interpolator.interpolate! + + interpolator.to_hash + end + strong_memoize_attr :content_hash - content_result.content + def interpolator + External::Interpolator + .new(content_result, content_inputs, context) end + strong_memoize_attr :interpolator def expanded_content_hash - return unless content_hash + return if content_hash.blank? strong_memoize(:expanded_content_hash) do expand_includes(content_hash) @@ -132,6 +147,12 @@ module Gitlab end end + def validate_interpolation! + return if interpolator.valid? + + errors.push("`#{masked_location}`: #{interpolator.error_message}") + end + def expand_includes(hash) External::Processor.new(hash, context.mutate(expand_context_attrs)).perform end diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index 7ab7dc3d64e..9679d78a1aa 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -11,6 +11,7 @@ module Gitlab def initialize(params, context) @location = params[:component] + super end @@ -48,9 +49,7 @@ module Gitlab end def validate_content! - return if content.present? - - errors.push(component_result.message) + errors.push(component_result.message) unless content.present? end private diff --git a/lib/gitlab/ci/config/external/interpolator.rb b/lib/gitlab/ci/config/external/interpolator.rb new file mode 100644 index 00000000000..5629c4a9766 --- /dev/null +++ b/lib/gitlab/ci/config/external/interpolator.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + ## + # Config::External::Interpolation perform includable file interpolation, and surfaces all possible interpolation + # errors. It is designed to provide an external file's validation context too. + # + class Interpolator + include ::Gitlab::Utils::StrongMemoize + + attr_reader :config, :args, :ctx, :errors + + def initialize(config, args, ctx = nil) + @config = config + @args = args.to_h + @ctx = ctx + @errors = [] + + validate! + end + + def valid? + @errors.none? + end + + def ready? + ## + # Interpolation is ready when it has been either interrupted by an error or finished with a result. + # + @result || @errors.any? + end + + def interpolate? + enabled? && has_header? && valid? + end + + def has_header? + config.has_header? && config.header.present? + end + + def to_hash + @result.to_h + end + + def error_message + # Interpolator can have multiple error messages, like: ["interpolation interrupted by errors", "unknown + # interpolation key: `abc`"] ? + # + # We are joining them together into a single one, because only one error can be surfaced when an external + # file gets included and is invalid. The limit to three error messages combined is more than required. + # + @errors.first(3).join(', ') + end + + ## + # TODO Add `instrument.logger` instrumentation blocks: + # https://gitlab.com/gitlab-org/gitlab/-/issues/396722 + # + def interpolate! + return {} unless valid? + return @result ||= content.to_h unless interpolate? + + return @errors.concat(header.errors) unless header.valid? + return @errors.concat(inputs.errors) unless inputs.valid? + return @errors.concat(context.errors) unless context.valid? + return @errors.concat(template.errors) unless template.valid? + + @result ||= template.interpolated.to_h.deep_symbolize_keys + end + strong_memoize_attr :interpolate! + + private + + def validate! + return errors.push('content does not have a valid YAML syntax') unless config.valid? + + return unless has_header? && !enabled? + + errors.push('can not evaluate included file because interpolation is disabled') + end + + def enabled? + return false if ctx.nil? + + ::Feature.enabled?(:ci_includable_files_interpolation, ctx.project) + end + + def header + @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header| + header.key = 'header' + + header.compose! + end + end + + def content + @content ||= config.content + end + + def spec + @spec ||= header.inputs_value + end + + def inputs + @inputs ||= Ci::Input::Inputs.new(spec, args) + end + + def context + @context ||= Ci::Interpolation::Context.new({ inputs: inputs.to_hash }) + end + + def template + @template ||= ::Gitlab::Ci::Interpolation::Template + .new(content, context) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb index e59eaa6d324..5072d0971cf 100644 --- a/lib/gitlab/ci/config/external/mapper/matcher.rb +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -7,22 +7,13 @@ module Gitlab class Mapper # Matches the first file type that matches the given location class Matcher < Base - FILE_CLASSES = [ - External::File::Local, - External::File::Project, - External::File::Component, - External::File::Remote, - External::File::Template, - External::File::Artifact - ].freeze - - FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze + include Gitlab::Utils::StrongMemoize private def process_without_instrumentation(locations) locations.map do |location| - matching = FILE_CLASSES.map do |file_class| + matching = file_classes.map do |file_class| file_class.new(location, context) end.select(&:matching?) @@ -31,10 +22,10 @@ module Gitlab elsif matching.empty? raise Mapper::AmbigiousSpecificationError, "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ - "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + "Valid subkeys are: `#{file_subkeys.join('`, `')}`" else raise Mapper::AmbigiousSpecificationError, - "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + "Each include must use only one of: `#{file_subkeys.join('`, `')}`" end end end @@ -42,6 +33,28 @@ module Gitlab def masked_location(location) context.mask_variables_from(location) end + + def file_subkeys + file_classes.map { |f| f.name.demodulize.downcase }.freeze + end + strong_memoize_attr :file_subkeys + + def file_classes + classes = [ + External::File::Local, + External::File::Project, + External::File::Remote, + External::File::Template, + External::File::Artifact + ] + + if Feature.enabled?(:ci_include_components, context.project&.root_namespace) + classes << External::File::Component + end + + classes + end + strong_memoize_attr :file_classes end end end diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb index 525b009afe3..7f0edaaac4c 100644 --- a/lib/gitlab/ci/config/header/input.rb +++ b/lib/gitlab/ci/config/header/input.rb @@ -6,6 +6,7 @@ module Gitlab module Header ## # Input parameter used for interpolation with the CI configuration. + # class Input < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable diff --git a/lib/gitlab/ci/config/header/spec.rb b/lib/gitlab/ci/config/header/spec.rb index 98d6d0d5783..4753c1eb441 100644 --- a/lib/gitlab/ci/config/header/spec.rb +++ b/lib/gitlab/ci/config/header/spec.rb @@ -10,7 +10,7 @@ module Gitlab ALLOWED_KEYS = %i[inputs].freeze validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :config, allowed_keys: ALLOWED_KEYS end entry :inputs, ::Gitlab::Config::Entry::ComposableHash, diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index d1b1b8caa5c..729e7e3ac05 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -20,7 +20,8 @@ module Gitlab ::Gitlab::Config::Loader::MultiDocYaml.new( content, max_documents: MAX_DOCUMENTS, - additional_permitted_classes: AVAILABLE_TAGS + additional_permitted_classes: AVAILABLE_TAGS, + reject_empty: true ).load! else ::Gitlab::Config::Loader::Yaml diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb index 1a3ca53c161..33f9a454106 100644 --- a/lib/gitlab/ci/config/yaml/result.rb +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -17,7 +17,9 @@ module Gitlab end def has_header? - @config.size > 1 + return false unless @config.first.is_a?(Hash) + + @config.size > 1 && @config.first.key?(:spec) end def header @@ -27,7 +29,9 @@ module Gitlab end def content - @config.last + return @config.last if has_header? + + @config.first end end end diff --git a/lib/gitlab/ci/input/arguments/default.rb b/lib/gitlab/ci/input/arguments/default.rb index fd61c1ab786..c6762b04870 100644 --- a/lib/gitlab/ci/input/arguments/default.rb +++ b/lib/gitlab/ci/input/arguments/default.rb @@ -9,7 +9,9 @@ module Gitlab # class Default < Input::Arguments::Base def validate! - error('invalid specification') unless default.present? + return error('argument specification invalid') unless spec.key?(:default) + + error('invalid default value') unless default.is_a?(String) || default.nil? end ## @@ -35,6 +37,8 @@ module Gitlab end def self.matches?(spec) + return false unless spec.is_a?(Hash) + spec.count == 1 && spec.each_key.first == :default end end diff --git a/lib/gitlab/ci/input/arguments/options.rb b/lib/gitlab/ci/input/arguments/options.rb index debc89b10bd..855dab129be 100644 --- a/lib/gitlab/ci/input/arguments/options.rb +++ b/lib/gitlab/ci/input/arguments/options.rb @@ -25,7 +25,8 @@ module Gitlab # The configuration above will return an empty value. # def validate! - return error('argument specification invalid') if options.to_a.empty? + return error('argument specification invalid') unless options.is_a?(Array) + return error('options argument empty') if options.empty? if !value.nil? error("argument value #{value} not allowlisted") unless options.include?(value) @@ -43,6 +44,8 @@ module Gitlab end def self.matches?(spec) + return false unless spec.is_a?(Hash) + spec.count == 1 && spec.each_key.first == :options end end diff --git a/lib/gitlab/ci/input/arguments/required.rb b/lib/gitlab/ci/input/arguments/required.rb index b4e218ed29e..2e39f548731 100644 --- a/lib/gitlab/ci/input/arguments/required.rb +++ b/lib/gitlab/ci/input/arguments/required.rb @@ -28,7 +28,7 @@ module Gitlab # website: # ``` # - # An empty value, that has no specification is also considered as a "required" input, however we should + # An empty string value, that has no specification is also considered as a "required" input, however we should # never see that being used, because it will be rejected by Ci::Config::Header validation. # # ```yaml @@ -36,8 +36,17 @@ module Gitlab # inputs: # website: "" # ``` + # + # An empty hash value is also considered to be a required argument: + # + # ```yaml + # spec: + # inputs: + # website: {} + # ``` + # def self.matches?(spec) - spec.to_s.empty? + spec.blank? end end end diff --git a/lib/gitlab/ci/input/inputs.rb b/lib/gitlab/ci/input/inputs.rb index 743ae2ecf1e..1b544e63e7d 100644 --- a/lib/gitlab/ci/input/inputs.rb +++ b/lib/gitlab/ci/input/inputs.rb @@ -19,8 +19,8 @@ module Gitlab ].freeze def initialize(spec, args) - @spec = spec - @args = args + @spec = spec.to_h + @args = args.to_h @inputs = [] @errors = [] diff --git a/lib/gitlab/ci/interpolation/access.rb b/lib/gitlab/ci/interpolation/access.rb index 42598458902..f9bbd3e118d 100644 --- a/lib/gitlab/ci/interpolation/access.rb +++ b/lib/gitlab/ci/interpolation/access.rb @@ -45,7 +45,11 @@ module Gitlab raise ArgumentError, 'access path invalid' unless valid? @value ||= objects.inject(@ctx) do |memo, value| - memo.fetch(value.to_sym) + key = value.to_sym + + break @errors.push("unknown interpolation key: `#{key}`") unless memo.key?(key) + + memo.fetch(key) end rescue KeyError => e @errors.push(e) diff --git a/lib/gitlab/ci/interpolation/context.rb b/lib/gitlab/ci/interpolation/context.rb index ce7a86a3c9b..69c1fbb792c 100644 --- a/lib/gitlab/ci/interpolation/context.rb +++ b/lib/gitlab/ci/interpolation/context.rb @@ -38,6 +38,10 @@ module Gitlab @context.fetch(field) end + def key?(name) + @context.key?(name) + end + def to_h @context.to_h end @@ -53,7 +57,7 @@ module Gitlab end end - values.max + values.max.to_i end def self.fabricate(context) diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index cfefa79d9e0..fdff5035d37 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -20,11 +20,23 @@ module Gitlab attr_reader :aud def reserved_claims - super.merge( + super.merge({ iss: Settings.gitlab.base_url, sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", - aud: aud - ) + aud: aud, + user_identities: user_identities + }.compact) + end + + def user_identities + return unless user&.pass_user_identities_to_ci_jwt + + user.identities.map do |identity| + { + provider: identity.provider.to_s, + extern_uid: identity.extern_uid.to_s + } + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 484e18c6979..98f488d0f38 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -123,6 +123,7 @@ module Gitlab end @needs_attributes.flat_map do |need| + # We ignore the optional needed job in case it is excluded from the pipeline due to the job's rules. next if need[:optional] result = need_present?(need) diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 45e67528f12..bf48c7d0bb7 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -190,6 +190,10 @@ module Gitlab original_data['assets'] || [] end + def raw_source_code_extract + original_data['raw_source_code_extract'] + end + # Returns either the max priority signature hex # or the location fingerprint def location_fingerprint diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index a4434e2c144..54f6784b847 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -11,12 +11,12 @@ module Gitlab Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, - Status::Build::WaitingForResource, Status::Build::Preparing, Status::Build::Pending, Status::Build::Skipped, Status::Build::WaitingForApproval], - [Status::Build::Cancelable, + [Status::Build::WaitingForResource, + Status::Build::Cancelable, Status::Build::Retryable], [Status::Build::FailedUnmetPrerequisites, Status::Build::Failed], diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 002bd846ab1..1ba78b357e5 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -8,17 +8,18 @@ module Gitlab # This class accepts an array of arrays/hashes/or objects # `with_allow_failure` will be removed when deleting ci_remove_ensure_stage_service - def initialize(all_statuses, with_allow_failure: true, dag: false) - unless all_statuses.respond_to?(:pluck) - raise ArgumentError, "all_statuses needs to respond to `.pluck`" + def initialize(all_jobs, with_allow_failure: true, dag: false, project: nil) + unless all_jobs.respond_to?(:pluck) + raise ArgumentError, "all_jobs needs to respond to `.pluck`" end @status_set = Set.new @status_key = 0 @allow_failure_key = 1 if with_allow_failure @dag = dag + @project = project - consume_all_statuses(all_statuses) + consume_all_jobs(all_jobs) end # The status calculation is order dependent, @@ -28,11 +29,13 @@ module Gitlab # based on what statuses are no longer valid based on the # data set that we have # - # This method is used for two cases: - # 1. When it is called for a stage or a pipeline (with `all_statuses` from all jobs in a stage or a pipeline), + # This method is used for three cases: + # 1. When it is called for a stage or a pipeline (with `all_jobs` from all jobs in a stage or a pipeline), # then, the returned status is assigned to the stage or pipeline. - # 2. When it is called for a job (with `all_statuses` from all previous jobs or all needed jobs), + # 2. When it is called for a job (with `all_jobs` from all previous jobs or all needed jobs), # then, the returned status is used to determine if the job is processed or not. + # 3. When it is called for a group (of jobs that are related), + # then, the returned status is used to show the overall status of the group. # rubocop: disable Metrics/CyclomaticComplexity # rubocop: disable Metrics/PerceivedComplexity def status @@ -42,9 +45,6 @@ module Gitlab if @dag && any_skipped_or_ignored? # The DAG job is skipped if one of the needs does not run at all. 'skipped' - elsif @dag && !only_of?(:success, :failed, :canceled, :skipped, :success_with_warnings) - # DAG is blocked from executing if a dependent is not "complete" - 'pending' elsif only_of?(:skipped, :ignored) 'skipped' elsif only_of?(:success, :skipped, :success_with_warnings, :ignored) @@ -101,41 +101,41 @@ module Gitlab any_of?(:skipped) || any_of?(:ignored) end - def consume_all_statuses(all_statuses) + def consume_all_jobs(all_jobs) columns = [] columns[@status_key] = :status columns[@allow_failure_key] = :allow_failure if @allow_failure_key - all_statuses + all_jobs .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord - .each do |status_attrs| - consume_status(Array.wrap(status_attrs)) + .each do |job_attrs| + consume_job_status(Array.wrap(job_attrs)) end end - def consume_status(status_attrs) + def consume_job_status(job_attrs) status_result = - if success_with_warnings?(status_attrs) + if success_with_warnings?(job_attrs) :success_with_warnings - elsif ignored_status?(status_attrs) + elsif ignored_status?(job_attrs) :ignored else - status_attrs[@status_key].to_sym + job_attrs[@status_key].to_sym end @status_set.add(status_result) end - def success_with_warnings?(status) + def success_with_warnings?(job_attrs) @allow_failure_key && - status[@allow_failure_key] && - ::Ci::HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key]) + job_attrs[@allow_failure_key] && + ::Ci::HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(job_attrs[@status_key]) end - def ignored_status?(status) + def ignored_status?(job_attrs) @allow_failure_key && - status[@allow_failure_key] && - ::Ci::HasStatus::IGNORED_STATUSES.include?(status[@status_key]) + job_attrs[@allow_failure_key] && + ::Ci::HasStatus::IGNORED_STATUSES.include?(job_attrs[@status_key]) end end end diff --git a/lib/gitlab/ci/status/processable/waiting_for_resource.rb b/lib/gitlab/ci/status/processable/waiting_for_resource.rb index c9b1dd795d0..ac82c99b5f1 100644 --- a/lib/gitlab/ci/status/processable/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/processable/waiting_for_resource.rb @@ -17,9 +17,39 @@ module Gitlab } end + def has_action? + current_processable.present? + end + + def action_icon + nil + end + + def action_title + nil + end + + def action_button_title + _('View job currently using resource') + end + + def action_path + project_job_path(subject.project, current_processable) + end + + def action_method + :get + end + def self.matches?(processable, _) processable.waiting_for_resource? end + + private + + def current_processable + @current_processable ||= subject.resource_group.current_processable + end end end end diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 2f7c16f0904..aeadc89095b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.30.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.31.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 2f7c16f0904..aeadc89095b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.30.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.31.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 47b79302828..b2ab6704e35 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE_TAG: "0.89.0" + CODE_QUALITY_IMAGE_TAG: "0.94.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml index 7f8e2150c71..8063f3d1e69 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml @@ -40,7 +40,7 @@ container_scanning: reports: container_scanning: gl-container-scanning-report.json dependency_scanning: gl-dependency-scanning-report.json - paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json, "**/gl-sbom-*.cdx.json"] dependencies: [] script: - gtcs scan diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml index 15688da71ab..24c23ce89f3 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml @@ -40,12 +40,12 @@ container_scanning: reports: container_scanning: gl-container-scanning-report.json dependency_scanning: gl-dependency-scanning-report.json - paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json, "**/gl-sbom-*.cdx.json"] dependencies: [] script: - gtcs scan rules: - - if: $CONTAINER_SCANNING_DISABLED + - if: $CONTAINER_SCANNING_DISABLED == 'true' || $CONTAINER_SCANNING_DISABLED == '1' when: never # Add the job to merge request pipelines if there's an open merge request. diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 61c2b468899..e336f69a7f6 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.47.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 31d19779434..2196630296b 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -63,6 +63,7 @@ dependency_scanning: - '**/npm-shrinkwrap.json' - '**/package-lock.json' - '**/yarn.lock' + - '**/pnpm-lock.yaml' - '**/packages.lock.json' - '**/conan.lock' diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml index 9ab17997c27..46161dce74c 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -63,6 +63,7 @@ dependency_scanning: - '**/npm-shrinkwrap.json' - '**/package-lock.json' - '**/yarn.lock' + - '**/pnpm-lock.yaml' - '**/packages.lock.json' - '**/conan.lock' @@ -74,7 +75,7 @@ gemnasium-dependency_scanning: DS_ANALYZER_NAME: "gemnasium" GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ when: never @@ -121,7 +122,7 @@ gemnasium-maven-dependency_scanning: variables: DS_ANALYZER_NAME: "gemnasium-maven" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ when: never @@ -169,7 +170,7 @@ gemnasium-python-dependency_scanning: variables: DS_ANALYZER_NAME: "gemnasium-python" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 9bac82b660f..ea6216a9210 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.47.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index ec43217792f..34560600c10 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.47.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml index e47f669c2e2..8e1b0159cb0 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml @@ -32,7 +32,7 @@ license_scanning: license_scanning: gl-license-scanning-report.json dependencies: [] rules: - - if: $LICENSE_MANAGEMENT_DISABLED + - if: $LICENSE_MANAGEMENT_DISABLED == 'true' || $LICENSE_MANAGEMENT_DISABLED == '1' when: never # Add the job to merge request pipelines if there's an open merge request. diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 8b49d2de8cf..7b2e9e1222a 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ # # Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). -# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables +# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-cicd-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index febbb36d834..5797bcbaca9 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -32,7 +32,7 @@ test: script: - python setup.py test - pip install tox flake8 # you can also use tox - - tox -e py36,flake8 + - tox -e py,flake8 run: script: diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 27bcc14bcf5..de8a21819cc 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -44,13 +44,19 @@ dast: reports: dast: gl-dast-report.json rules: - - if: $DAST_DISABLED + - if: $DAST_DISABLED == 'true' || $DAST_DISABLED == '1' when: never - - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == 'true' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == '1' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED == 'true' + when: never - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED + $REVIEW_DISABLED == '1' when: never # Add the job to merge request pipelines if there's an open merge request. diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index bc23a7c2a95..3249bd2bcac 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -52,6 +52,9 @@ cache: - gitlab-terraform plan-json resource_group: ${TF_STATE_NAME} artifacts: + # The next line, which disables public access to pipeline artifacts, may not be available everywhere. + # See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic + public: false paths: - ${TF_ROOT}/plan.cache reports: diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 32f64948635..a3f1b472710 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -166,13 +166,6 @@ module Gitlab end def destroy! - # TODO: Remove this logging once we confirmed new live trace architecture is functional. - # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. - unless build.has_archived_trace? - Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.', - job_id: build.id) - end - trace_chunks.fast_destroy_all @tell = @size = 0 ensure diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 89d681c418d..86e54fdfcdf 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -140,11 +140,13 @@ module Gitlab # Set environment name here so we can access it when evaluating the job's rules variables.append(key: 'CI_ENVIRONMENT_NAME', value: job.environment) if job.environment - # legacy variables - variables.append(key: 'CI_BUILD_NAME', value: job.name) - variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) - variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request - variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? + if Feature.disabled?(:ci_remove_legacy_predefined_variables, project) + # legacy variables + variables.append(key: 'CI_BUILD_NAME', value: job.name) + variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) + variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request + variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? + end end end diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb index 96d6f1673b9..1e7a18d70b0 100644 --- a/lib/gitlab/ci/variables/builder/pipeline.rb +++ b/lib/gitlab/ci/variables/builder/pipeline.rb @@ -40,7 +40,7 @@ module Gitlab attr_reader :pipeline - def predefined_commit_variables + def predefined_commit_variables # rubocop:disable Metrics/AbcSize - Remove this rubocop:disable when FF `ci_remove_legacy_predefined_variables` is removed. Gitlab::Ci::Variables::Collection.new.tap do |variables| next variables unless pipeline.sha.present? @@ -57,7 +57,9 @@ module Gitlab variables.append(key: 'CI_COMMIT_TIMESTAMP', value: pipeline.git_commit_timestamp.to_s) variables.append(key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text.to_s) - variables.concat(legacy_predefined_commit_variables) + if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project) + variables.concat(legacy_predefined_commit_variables) + end end end strong_memoize_attr :predefined_commit_variables @@ -81,7 +83,9 @@ module Gitlab variables.append(key: 'CI_COMMIT_TAG', value: pipeline.ref) variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message) - variables.concat(legacy_predefined_commit_tag_variables) + if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project) + variables.concat(legacy_predefined_commit_tag_variables) + end end end strong_memoize_attr :predefined_commit_tag_variables diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 59acfa80258..0f9e7daf4b8 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -99,7 +99,7 @@ module Gitlab validate_duplicate_needs!(name, needs) needs.each do |need| - validate_job_dependency!(name, need[:name], 'need') + validate_job_dependency!(name, need[:name], 'need', optional: need[:optional]) end end @@ -109,8 +109,13 @@ module Gitlab end end - def validate_job_dependency!(name, dependency, dependency_type = 'dependency') + def validate_job_dependency!(name, dependency, dependency_type = 'dependency', optional: false) unless @jobs[dependency.to_sym] + # Here, we ignore the optional needed job if it is not in the result YAML due to the `include` + # rules. In `lib/gitlab/ci/pipeline/seed/build.rb`, we use `optional` again to ignore the + # optional needed job in case it is excluded from the pipeline due to the job's rules. + return if optional + error!("#{name} job: undefined #{dependency_type}: #{dependency}") end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index d867439b10b..6207b595fc6 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -123,7 +123,8 @@ module Gitlab start_in: job[:start_in], trigger: job[:trigger], bridge_needs: job.dig(:needs, :bridge)&.first, - release: job[:release] + release: job[:release], + publish: job[:publish] }.compact }.compact end |