diff options
Diffstat (limited to 'lib/gitlab/ci/config/external')
| -rw-r--r-- | lib/gitlab/ci/config/external/context.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/file/base.rb | 7 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/file/component.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper.rb | 1 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/matcher.rb | 11 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/verifier.rb | 15 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/rules.rb | 20 |
7 files changed, 51 insertions, 17 deletions
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index b8e012ec851..c57391d355c 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -14,7 +14,9 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config - attr_reader :expandset, :execution_deadline, :logger, :max_includes + attr_reader :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes + + attr_accessor :total_file_size_in_bytes delegate :instrument, to: :logger @@ -32,6 +34,9 @@ module Gitlab @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) @max_includes = Gitlab::CurrentSettings.current_application_settings.ci_max_includes + @max_total_yaml_size_bytes = + Gitlab::CurrentSettings.current_application_settings.ci_max_total_yaml_size_bytes + @total_file_size_in_bytes = 0 yield self if block_given? end @@ -59,6 +64,7 @@ module Gitlab ctx.execution_deadline = execution_deadline ctx.logger = logger ctx.max_includes = max_includes + ctx.max_total_yaml_size_bytes = max_total_yaml_size_bytes end end @@ -100,7 +106,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline, :logger, :max_includes + attr_writer :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes private diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 8bcb2a389d2..efba81c7420 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -92,6 +92,11 @@ module Gitlab def load_and_validate_expanded_hash! return errors.push("`#{masked_location}`: #{content_result.error}") unless content_result.valid? + if content_result.interpolated? && context.user.present? + ::Gitlab::UsageDataCounters::HLLRedisCounter + .track_event('ci_interpolation_users', values: context.user.id) + end + context.logger.instrument(:config_file_expand_content_includes) do expanded_content_hash # calling the method expands then memoizes the result end @@ -109,7 +114,7 @@ module Gitlab def content_result context.logger.instrument(:config_file_fetch_content_hash) do - ::Gitlab::Ci::Config::Yaml::Loader.new(content, inputs: content_inputs, current_user: context.user).load + ::Gitlab::Ci::Config::Yaml::Loader.new(content, inputs: content_inputs).load end end strong_memoize_attr :content_result diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index 9679d78a1aa..15cc0783b86 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -15,10 +15,6 @@ module Gitlab super end - def matching? - super && ::Feature.enabled?(:ci_include_components, context.project&.root_namespace) - end - def content return unless component_result.success? diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 61b4d1ada10..cff7954235f 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -10,6 +10,7 @@ module Gitlab Error = Class.new(StandardError) AmbigiousSpecificationError = Class.new(Error) TooManyIncludesError = Class.new(Error) + TooMuchDataInPipelineTreeError = Class.new(Error) def initialize(values, context) @locations = Array.wrap(values.fetch(:include, [])).compact diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb index 5072d0971cf..f9d5b0ebd01 100644 --- a/lib/gitlab/ci/config/external/mapper/matcher.rb +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -40,19 +40,14 @@ module Gitlab strong_memoize_attr :file_subkeys def file_classes - classes = [ + [ External::File::Local, External::File::Project, External::File::Remote, External::File::Template, - External::File::Artifact + External::File::Artifact, + External::File::Component ] - - if Feature.enabled?(:ci_include_components, context.project&.root_namespace) - classes << External::File::Component - end - - classes end strong_memoize_attr :file_classes end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 95975e4661b..580cae8a207 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'objspace' + module Gitlab module Ci class Config @@ -37,6 +39,13 @@ module Gitlab file.validate_content! if file.valid? file.load_and_validate_expanded_hash! if file.valid? + + next unless Feature.enabled?(:introduce_ci_max_total_yaml_size_bytes, context.project) && file.valid? + + # We are checking the file.content.to_s because that is returning the actual content of the file, + # whereas file.content would return the BatchLoader. + context.total_file_size_in_bytes += ObjectSpace.memsize_of(file.content.to_s) + verify_max_total_pipeline_size! end end # rubocop: enable Metrics/CyclomaticComplexity @@ -50,6 +59,12 @@ module Gitlab def verify_execution_time! context.check_execution_time! end + + def verify_max_total_pipeline_size! + return if context.total_file_size_in_bytes <= context.max_total_yaml_size_bytes + + raise Mapper::TooMuchDataInPipelineTreeError, "Total size of combined CI/CD configuration is too big" + end end end end diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 59e666b8bb5..0e6209460e0 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -5,15 +5,29 @@ module Gitlab class Config module External class Rules + # Remove these two constants when FF `ci_refactor_external_rules` is removed ALLOWED_KEYS = Entry::Include::Rules::Rule::ALLOWED_KEYS ALLOWED_WHEN = Entry::Include::Rules::Rule::ALLOWED_WHEN InvalidIncludeRulesError = Class.new(Mapper::Error) def initialize(rule_hashes) - validate(rule_hashes) + if Feature.enabled?(:ci_refactor_external_rules) + return unless rule_hashes - @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) + # We must compose the include rules entry here because included + # files are expanded before `@root.compose!` runs in Ci::Config. + rules_entry = Entry::Include::Rules.new(rule_hashes) + rules_entry.compose! + + raise InvalidIncludeRulesError, "include:#{rules_entry.errors.first}" unless rules_entry.valid? + + @rule_list = Build::Rules::Rule.fabricate_list(rules_entry.value) + else + validate(rule_hashes) + + @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) + end end def evaluate(context) @@ -32,6 +46,7 @@ module Gitlab @rule_list.find { |rule| rule.matches?(nil, context) } end + # Remove this method when FF `ci_refactor_external_rules` is removed def validate(rule_hashes) return unless rule_hashes.is_a?(Array) @@ -42,6 +57,7 @@ module Gitlab end end + # Remove this method when FF `ci_refactor_external_rules` is removed def valid_when?(rule_hash) rule_hash[:when].nil? || rule_hash[:when].in?(ALLOWED_WHEN) end |