diff options
Diffstat (limited to 'lib/gitlab/ci')
26 files changed, 117 insertions, 249 deletions
diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index a5481071fc5..a635f409109 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -9,7 +9,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_KEYS = %i[key untracked paths when policy unprotect].freeze ALLOWED_POLICY = %w[pull-push push pull].freeze DEFAULT_POLICY = 'pull-push' ALLOWED_WHEN = %w[on_success on_failure always].freeze @@ -33,18 +33,22 @@ module Gitlab entry :key, Entry::Key, description: 'Cache key used to define a cache affinity.' + entry :unprotect, ::Gitlab::Config::Entry::Boolean, + description: 'Unprotect the cache from a protected ref.' + entry :untracked, ::Gitlab::Config::Entry::Boolean, description: 'Cache all untracked files.' entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' - attributes :policy, :when + attributes :policy, :when, :unprotect def value result = super result[:key] = key_value + result[:unprotect] = unprotect || false result[:policy] = policy || DEFAULT_POLICY # Use self.when to avoid conflict with reserved word result[:when] = self.when || DEFAULT_WHEN diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index e0a052ffdfd..e0f0903174c 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -27,9 +27,9 @@ module Gitlab validates :config, disallowed_keys: { in: %i[only except start_in], - message: 'key may not be used with `rules`' - }, - if: :has_rules? + message: 'key may not be used with `rules`', + ignore_nil: true + }, if: :has_rules_value? with_options allow_nil: true do validates :extends, array_of_strings_or_string: true diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb index 5c78a8f68c7..e91714e3f5c 100644 --- a/lib/gitlab/ci/config/entry/product/parallel.rb +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -12,7 +12,7 @@ module Gitlab strategy :ParallelBuilds, if: -> (config) { config.is_a?(Numeric) } strategy :MatrixBuilds, if: -> (config) { config.is_a?(Hash) } - PARALLEL_LIMIT = 50 + PARALLEL_LIMIT = 200 class ParallelBuilds < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 16844fa88db..6408f412e6f 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -54,7 +54,7 @@ module Gitlab end def value - @config.transform_values do |value| + @config.compact.transform_values do |value| if value.is_a?(Hash) value else diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index decb568ffc9..a5c6aaa1e3a 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -54,9 +54,7 @@ module Gitlab validates :key, alphanumeric: true validates :config_value, alphanumeric: true, allow_nil: true validates :config_description, alphanumeric: true, allow_nil: true - validates :config_expand, boolean: true, allow_nil: true, if: -> { - ci_raw_variables_in_yaml_config_enabled? - } + validates :config_expand, boolean: true, allow_nil: true validates :config_options, array_of_strings: true, allow_nil: true validate do @@ -82,16 +80,10 @@ module Gitlab end def value_with_data - if ci_raw_variables_in_yaml_config_enabled? - { - value: config_value.to_s, - raw: (!config_expand if has_config_expand?) - }.compact - else - { - value: config_value.to_s - }.compact - end + { + value: config_value.to_s, + raw: (!config_expand if has_config_expand?) + }.compact end def value_with_prefill_data @@ -100,10 +92,6 @@ module Gitlab options: config_options ).compact end - - def ci_raw_variables_in_yaml_config_enabled? - YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) - end end class UnknownStrategy < ::Gitlab::Config::Entry::Node diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 21a57640aee..140cbfac5c1 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -38,10 +38,6 @@ module Gitlab private - def project - context&.parent_pipeline&.project - end - def validate_context! context.logger.instrument(:config_file_artifact_validate_context) do if !creating_child_pipeline? diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 65caf4ac47d..7899fe0ff73 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,7 +47,6 @@ module Gitlab end def validate! - context.check_execution_time! if ::Feature.disabled?(:ci_refactoring_external_mapper, context.project) validate_location! validate_context! if valid? fetch_and_validate_content! if valid? diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index a41bc2b39f2..61b4d1ada10 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,18 +7,6 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize - # Will be removed with FF ci_refactoring_external_mapper - FILE_CLASSES = [ - External::File::Local, - External::File::Project, - External::File::Remote, - External::File::Template, - External::File::Artifact - ].freeze - - # Will be removed with FF ci_refactoring_external_mapper - FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze - Error = Class.new(StandardError) AmbigiousSpecificationError = Class.new(Error) TooManyIncludesError = Class.new(Error) @@ -32,11 +20,7 @@ module Gitlab return [] if @locations.empty? context.logger.instrument(:config_mapper_process) do - if ::Feature.enabled?(:ci_refactoring_external_mapper, context.project) - process_without_instrumentation - else - legacy_process_without_instrumentation - end + process_without_instrumentation end end @@ -57,138 +41,6 @@ module Gitlab files end - - # This and the following methods will be removed with FF ci_refactoring_external_mapper - def legacy_process_without_instrumentation - @locations - .map(&method(:normalize_location)) - .filter_map(&method(:verify_rules)) - .flat_map(&method(:expand_project_files)) - .flat_map(&method(:expand_wildcard_paths)) - .map(&method(:expand_variables)) - .map(&method(:select_first_matching)) - .each(&method(:verify!)) - end - - # convert location if String to canonical form - def normalize_location(location) - if location.is_a?(String) - expanded_location = expand_variables(location) - normalize_location_string(expanded_location) - else - location.deep_symbolize_keys - end - end - - def verify_rules(location) - logger.instrument(:config_mapper_rules) do - verify_rules_without_instrumentation(location) - end - end - - def verify_rules_without_instrumentation(location) - return unless Rules.new(location[:rules]).evaluate(context).pass? - - location - end - - def expand_project_files(location) - return location unless location[:project] - - Array.wrap(location[:file]).map do |file| - location.merge(file: file) - end - end - - def expand_wildcard_paths(location) - logger.instrument(:config_mapper_wildcards) do - expand_wildcard_paths_without_instrumentation(location) - end - end - - def expand_wildcard_paths_without_instrumentation(location) - # We only support local files for wildcard paths - return location unless location[:local] && location[:local].include?('*') - - context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| - { local: path } - end - end - - def normalize_location_string(location) - if ::Gitlab::UrlSanitizer.valid?(location) - { remote: location } - else - { local: location } - end - end - - def select_first_matching(location) - logger.instrument(:config_mapper_select) do - select_first_matching_without_instrumentation(location) - end - end - - def select_first_matching_without_instrumentation(location) - matching = FILE_CLASSES.map do |file_class| - file_class.new(location, context) - end.select(&:matching?) - - if matching.one? - matching.first - elsif matching.empty? - raise AmbigiousSpecificationError, "`#{masked_location(location.to_json)}` does not have a valid subkey for include. Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" - else - raise AmbigiousSpecificationError, "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" - end - end - - def verify!(location_object) - verify_max_includes! - location_object.validate! - expandset.add(location_object) - end - - def verify_max_includes! - if expandset.count >= context.max_includes - raise TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" - end - end - - def expand_variables(data) - logger.instrument(:config_mapper_variables) do - expand_variables_without_instrumentation(data) - end - end - - def expand_variables_without_instrumentation(data) - if data.is_a?(String) - expand(data) - else - transform(data) - end - end - - def transform(data) - data.transform_values do |values| - case values - when Array - values.map { |value| expand(value.to_s) } - when String - expand(values) - else - values - end - end - end - - def expand(data) - ExpandVariables.expand(data, -> { context.variables_hash }) - end - - def masked_location(location) - context.mask_variables_from(location) - end end end end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index ab5203252a2..e6a2e5c3b33 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -17,7 +17,7 @@ module Gitlab secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4] }.freeze - VERSIONS_TO_REMOVE_IN_16_0 = [].freeze + VERSIONS_TO_REMOVE_IN_16_0 = %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3].freeze DEPRECATED_VERSIONS = { cluster_image_scanning: VERSIONS_TO_REMOVE_IN_16_0, @@ -30,6 +30,8 @@ module Gitlab secret_detection: VERSIONS_TO_REMOVE_IN_16_0 }.freeze + CURRENT_VERSIONS = SUPPORTED_VERSIONS.to_h { |k, v| [k, v - DEPRECATED_VERSIONS[k]] } + class Schema def root_path File.join(__dir__, 'schemas') @@ -129,6 +131,11 @@ module Gitlab end def report_uses_deprecated_schema_version? + # Avoid deprecation warnings for GitLab security scanners + # To be removed via https://gitlab.com/gitlab-org/gitlab/-/issues/386798 + return if report_data.dig('scan', 'scanner', 'vendor', 'name')&.downcase == 'gitlab' + return if report_data.dig('scan', 'analyzer', 'vendor', 'name')&.downcase == 'gitlab' + DEPRECATED_VERSIONS[report_type].include?(report_version) end @@ -182,11 +189,15 @@ module Gitlab def add_deprecated_report_version_message log_warnings(problem_type: 'using_deprecated_schema_version') - template = _("Version %{report_version} for report type %{report_type} has been deprecated,"\ - " supported versions for this report type are: %{supported_schema_versions}."\ - " GitLab will attempt to parse and ingest this report if valid.") + template = _("version %{report_version} for report type %{report_type} is deprecated. "\ + "However, GitLab will still attempt to parse and ingest this report. "\ + "Upgrade the security report to one of the following versions: %{current_schema_versions}.") - message = format(template, report_version: report_version, report_type: report_type, supported_schema_versions: supported_schema_versions) + message = format( + template, + report_version: report_version, + report_type: report_type, + current_schema_versions: current_schema_versions) add_message_as(level: :deprecation_warning, message: message) end @@ -207,6 +218,10 @@ module Gitlab ) end + def current_schema_versions + CURRENT_VERSIONS[report_type].join(", ") + end + def supported_schema_versions SUPPORTED_VERSIONS[report_type].join(", ") end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 31b130b5ab7..d2dc712e366 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -127,6 +127,10 @@ module Gitlab .observe({ plan: project.actual_plan_name }, jobs_count) end + def observe_pipeline_includes_count(pipeline) + logger.observe(:pipeline_includes_count, pipeline.config_metadata&.[](:includes)&.count, once: true) + end + def increment_pipeline_failure_reason_counter(reason) metrics.pipeline_failure_reason_counter .increment(reason: (reason || :unknown_failure).to_s) diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb index a8276d84b87..99e438ddbae 100644 --- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -6,7 +6,7 @@ module Gitlab module Chain class CreateDeployments < Chain::Base def perform! - create_deployments! + create_deployments! if Feature.disabled?(:move_create_deployments_to_worker, pipeline.project) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 654e24be8e1..c59ef2ba6a4 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -18,7 +18,8 @@ module Gitlab pipeline.stages = @command.pipeline_seed.stages if stage_names.empty? - return error('No stages / jobs for this pipeline.') + return error('Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.') end if pipeline.invalid? diff --git a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb index 89befb2a65b..e7a9009f8f4 100644 --- a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb +++ b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb @@ -22,8 +22,7 @@ module Gitlab private def set_pipeline_name - return if Feature.disabled?(:pipeline_name, pipeline.project) || - @command.yaml_processor_result.workflow_name.blank? + return if @command.yaml_processor_result.workflow_name.blank? name = @command.yaml_processor_result.workflow_name name = ExpandVariables.expand(name, -> { global_context.variables.sort_and_expand_all }) diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index de147914850..dd097187955 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -30,6 +30,7 @@ module Gitlab @command.observe_creation_duration(current_monotonic_time - @start) @command.observe_pipeline_size(@pipeline) @command.observe_jobs_count_in_alive_pipelines + @command.observe_pipeline_includes_count(@pipeline) @pipeline end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index f393406b549..8286dfc6560 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -121,7 +121,7 @@ module Gitlab def enabled? ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) end - strong_memoize_attr :enabled?, :enabled + strong_memoize_attr :enabled? def observations @observations ||= {} diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index b0b79b994c1..684b58474ad 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -53,7 +53,7 @@ module Gitlab end end end - strong_memoize_attr :included?, :inclusion + strong_memoize_attr :included? def errors logger.instrument(:pipeline_seed_build_errors) do @@ -261,7 +261,7 @@ module Gitlab def reuse_build_in_seed_context? Feature.enabled?(:ci_reuse_build_in_seed_context, @pipeline.project) end - strong_memoize_attr :reuse_build_in_seed_context?, :reuse_build_in_seed_context + strong_memoize_attr :reuse_build_in_seed_context? end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 781065a63db..409b6658cc0 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -14,6 +14,7 @@ module Gitlab @policy = local_cache.delete(:policy) @untracked = local_cache.delete(:untracked) @when = local_cache.delete(:when) + @unprotect = local_cache.delete(:unprotect) @custom_key_prefix = custom_key_prefix raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? @@ -25,7 +26,8 @@ module Gitlab paths: @paths, policy: @policy, untracked: @untracked, - when: @when + when: @when, + unprotect: @unprotect }.compact end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index 0074f3675e0..5e77db3d336 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -22,14 +22,26 @@ module Gitlab def illustration_content if can?(user, :update_build, subject) - _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') + manual_job_action_message else generic_permission_failure_message end end + def manual_job_action_message + if subject.retryable? + _("You can modify this job's CI/CD variables before running it again.") + else + _('This job does not start automatically and must be started manually. You can add CI/CD variables below for last-minute configuration changes before starting the job.') + end + end + def generic_permission_failure_message - _("This job does not run automatically and must be started manually, but you do not have access to it.") + if subject.outdated_deployment? + _("This deployment job does not run automatically and must be started manually, but it's older than the latest deployment, and therefore can't run.") + else + _("This job does not run automatically and must be started manually, but you do not have access to it.") + end end end end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b4beeb60dfd..47b79302828 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE_TAG: "0.87.3" + CODE_QUALITY_IMAGE_TAG: "0.89.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7a208584c4c..6884a9556b4 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 292b0a0036d..dc7e5f445d2 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index ba03ad6304f..9e15b07f5d1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 2c5027cdb43..8b49d2de8cf 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -255,7 +255,7 @@ sobelow-sast: when: never - if: $CI_COMMIT_BRANCH exists: - - 'mix.exs' + - '**/mix.exs' spotbugs-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index 58709d3ab62..1c4dbe6cd0f 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -332,12 +332,12 @@ sobelow-sast: when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - - 'mix.exs' + - '**/mix.exs' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. exists: - - 'mix.exs' + - '**/mix.exs' spotbugs-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index e9766061072..9960a6fbdf5 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -72,8 +72,36 @@ module Gitlab Collection.new(@variables.reject(&block)) end - # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. - def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) + def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true) + sorted = Sort.new(self) + return self.class.new(self, sorted.errors) unless sorted.valid? + + new_collection = self.class.new + + sorted.tsort.each do |item| + unless item.depends_on + new_collection.append(item) + next + end + + # expand variables as they are added + variable = item.to_runner_variable + variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, + expand_file_refs: expand_file_refs, + expand_raw_refs: expand_raw_refs) + new_collection.append(variable) + end + + new_collection + end + + def to_s + "#{@variables_by_key.keys}, @errors='#{@errors}'" + end + + protected + + def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true) value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%) full_match = match[0] @@ -88,19 +116,20 @@ module Gitlab if variable # VARIABLE_NAME is an existing variable if variable.file? - # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 - if project - # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` - # when the variables are sent to Runner. - Gitlab::AppJsonLogger.info(event: 'file_variable_is_referenced_in_another_variable', - project_id: project.id, - variable: variable_name) - end - expand_file_refs ? variable.value : full_match elsif variable.raw? - # With `full_match`, we defer the expansion of raw variables to the runner. If we expand them here, - # the runner will not know the expanded value is a raw variable and it tries to expand it again. + # Normally, it's okay to expand a raw variable if it's referenced in another variable because + # its rawness is not broken. However, the runner also tries to expand variables. + # Here, with `full_match`, we defer the expansion of raw variables to the runner. + # If we expand them here, the runner will not know that the expanded value is a raw variable + # and it tries to expand it again. + # Example: `A` is a normal variable with value `normal`. + # `B` is a raw variable with value `raw-$A`. + # `C` is a normal variable with value `$B`. + # If we expanded `C` here, the runner would receive `C` as `raw-$A`. And since `A` is a normal + # variable, the runner would expand it. So, the result would be `raw-normal`. + # With `full_match`, the runner receives `C` as `$B`. And since `B` is a raw variable, the + # runner expanded it as `raw-$A`, which is what we want. # Discussion: https://gitlab.com/gitlab-org/gitlab/-/issues/353991#note_1103274951 expand_raw_refs ? variable.value : full_match else @@ -115,36 +144,7 @@ module Gitlab end end - # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. - def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) - sorted = Sort.new(self) - return self.class.new(self, sorted.errors) unless sorted.valid? - - new_collection = self.class.new - - sorted.tsort.each do |item| - unless item.depends_on - new_collection.append(item) - next - end - - # expand variables as they are added - variable = item.to_runner_variable - variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, - expand_file_refs: expand_file_refs, - expand_raw_refs: expand_raw_refs, - project: project) - new_collection.append(variable) - end - - new_collection - end - - def to_s - "#{@variables_by_key.keys}, @errors='#{@errors}'" - end - - protected + private attr_reader :variables end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f2c1ad0575d..d867439b10b 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -64,12 +64,7 @@ module Gitlab private def assign_valid_attributes - @root_variables = if YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) - transform_to_array(@ci_config.variables_with_data) - else - transform_to_array(@ci_config.variables) - end - + @root_variables = transform_to_array(@ci_config.variables_with_data) @root_variables_with_prefill_data = @ci_config.variables_with_prefill_data @stages = @ci_config.stages |