diff options
Diffstat (limited to 'lib/gitlab/ci/config/external')
| -rw-r--r-- | lib/gitlab/ci/config/external/file/base.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/file/remote.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper.rb | 40 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/base.rb | 36 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/filter.rb | 22 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/location_expander.rb | 42 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/matcher.rb | 49 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/normalizer.rb | 46 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/variables_expander.rb | 49 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/mapper/verifier.rb | 37 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/processor.rb | 4 |
11 files changed, 310 insertions, 23 deletions
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 57ff606c9ee..65caf4ac47d 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,7 +47,7 @@ module Gitlab end def validate! - validate_execution_time! + context.check_execution_time! if ::Feature.disabled?(:ci_refactoring_external_mapper, context.project) validate_location! validate_context! if valid? fetch_and_validate_content! if valid? @@ -87,10 +87,6 @@ module Gitlab nil end - def validate_execution_time! - context.check_execution_time! - end - def validate_location! if invalid_location_type? errors.push("Included file `#{masked_location}` needs to be a string") diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index b0c540685d4..ed37357dc53 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -53,7 +53,7 @@ module Gitlab errors.push("Remote file `#{masked_location}` could not be fetched because of a timeout error!") rescue Gitlab::HTTP::Error errors.push("Remote file `#{masked_location}` could not be fetched because of HTTP error!") - rescue Gitlab::HTTP::BlockedUrlError => e + rescue Errno::ECONNREFUSED, Gitlab::HTTP::BlockedUrlError => e errors.push("Remote file could not be fetched because #{e}!") end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index fc03ac125fd..a41bc2b39f2 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,6 +7,7 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize + # Will be removed with FF ci_refactoring_external_mapper FILE_CLASSES = [ External::File::Local, External::File::Project, @@ -15,6 +16,7 @@ module Gitlab External::File::Artifact ].freeze + # Will be removed with FF ci_refactoring_external_mapper FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze Error = Class.new(StandardError) @@ -22,27 +24,43 @@ module Gitlab TooManyIncludesError = Class.new(Error) def initialize(values, context) - @locations = Array.wrap(values.fetch(:include, [])) + @locations = Array.wrap(values.fetch(:include, [])).compact @context = context end def process - return [] if locations.empty? + return [] if @locations.empty? - logger.instrument(:config_mapper_process) do - process_without_instrumentation + context.logger.instrument(:config_mapper_process) do + if ::Feature.enabled?(:ci_refactoring_external_mapper, context.project) + process_without_instrumentation + else + legacy_process_without_instrumentation + end end end private - attr_reader :locations, :context + attr_reader :context delegate :expandset, :logger, to: :context def process_without_instrumentation - locations - .compact + locations = Normalizer.new(context).process(@locations) + locations = Filter.new(context).process(locations) + locations = LocationExpander.new(context).process(locations) + locations = VariablesExpander.new(context).process(locations) + + files = Matcher.new(context).process(locations) + Verifier.new(context).process(files) + + files + end + + # This and the following methods will be removed with FF ci_refactoring_external_mapper + def legacy_process_without_instrumentation + @locations .map(&method(:normalize_location)) .filter_map(&method(:verify_rules)) .flat_map(&method(:expand_project_files)) @@ -52,14 +70,8 @@ module Gitlab .each(&method(:verify!)) end - def normalize_location(location) - logger.instrument(:config_mapper_normalize) do - normalize_location_without_instrumentation(location) - end - end - # convert location if String to canonical form - def normalize_location_without_instrumentation(location) + def normalize_location(location) if location.is_a?(String) expanded_location = expand_variables(location) normalize_location_string(expanded_location) diff --git a/lib/gitlab/ci/config/external/mapper/base.rb b/lib/gitlab/ci/config/external/mapper/base.rb new file mode 100644 index 00000000000..d2f56d0b8f6 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/base.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Base class for mapper classes + class Base + def initialize(context) + @context = context + end + + def process(*args) + context.logger.instrument(mapper_instrumentation_key) do + process_without_instrumentation(*args) + end + end + + private + + attr_reader :context + + def process_without_instrumentation + raise NotImplementedError + end + + def mapper_instrumentation_key + "config_mapper_#{self.class.name.demodulize.downcase}".to_sym + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/filter.rb b/lib/gitlab/ci/config/external/mapper/filter.rb new file mode 100644 index 00000000000..4d2b26c7d98 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/filter.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Filters locations according to rules + class Filter < Base + private + + def process_without_instrumentation(locations) + locations.select do |location| + Rules.new(location[:rules]).evaluate(context).pass? + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/location_expander.rb b/lib/gitlab/ci/config/external/mapper/location_expander.rb new file mode 100644 index 00000000000..a4ca058f0d9 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/location_expander.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Expands locations to include all files matching the pattern + class LocationExpander < Base + private + + def process_without_instrumentation(locations) + locations.flat_map do |location| + if location[:project] + expand_project_files(location) + elsif location[:local] + expand_wildcard_paths(location) + else + location + end + end + end + + def expand_project_files(location) + Array.wrap(location[:file]).map do |file| + location.merge(file: file) + end + end + + def expand_wildcard_paths(location) + return location unless location[:local].include?('*') + + context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| + { local: path } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb new file mode 100644 index 00000000000..85e19ff1ced --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Matches the first file type that matches the given location + class Matcher < Base + FILE_CLASSES = [ + External::File::Local, + External::File::Project, + External::File::Remote, + External::File::Template, + External::File::Artifact + ].freeze + + FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze + + private + + def process_without_instrumentation(locations) + locations.map do |location| + matching = FILE_CLASSES.map do |file_class| + file_class.new(location, context) + end.select(&:matching?) + + if matching.one? + matching.first + elsif matching.empty? + raise Mapper::AmbigiousSpecificationError, + "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ + "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + else + raise Mapper::AmbigiousSpecificationError, + "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + end + end + end + + def masked_location(location) + context.mask_variables_from(location) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/normalizer.rb b/lib/gitlab/ci/config/external/mapper/normalizer.rb new file mode 100644 index 00000000000..8fc798e78a0 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/normalizer.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Converts locations to canonical form (local:/remote:) if String + class Normalizer < Base + def initialize(context) + super + + @variables_expander = VariablesExpander.new(context) + end + + private + + attr_reader :variables_expander + + def process_without_instrumentation(locations) + locations.map do |location| + if location.is_a?(String) + # We need to expand before normalizing because the information of + # whether if it's a remote or local path may be hidden inside the variable. + location = variables_expander.expand(location) + + normalize_location_string(location) + else + location.deep_symbolize_keys + end + end + end + + def normalize_location_string(location) + if ::Gitlab::UrlSanitizer.valid?(location) + { remote: location } + else + { local: location } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/variables_expander.rb b/lib/gitlab/ci/config/external/mapper/variables_expander.rb new file mode 100644 index 00000000000..fddf04984d8 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/variables_expander.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Handles variable expansion + class VariablesExpander < Base + def expand(data) + if data.is_a?(String) + expand_variable(data) + else + transform_and_expand_variable(data) + end + end + + private + + def process_without_instrumentation(locations) + locations.map { |location| expand(location) } + end + + def transform_and_expand_variable(data) + data.transform_values do |values| + case values + when Array + values.map { |value| expand_variable(value.to_s) } + when String + expand_variable(values) + else + values + end + end + end + + def expand_variable(data) + ExpandVariables.expand(data, -> { variables }) + end + + def variables + @variables ||= context.variables_hash + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb new file mode 100644 index 00000000000..6d6f227b940 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + # Fetches file contents and verifies them + class Verifier < Base + private + + def process_without_instrumentation(files) + files.select do |file| + verify_max_includes! + verify_execution_time! + + file.validate! + + context.expandset.add(file) + end + end + + def verify_max_includes! + return if context.expandset.count < context.max_includes + + raise Mapper::TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" + end + + def verify_execution_time! + context.check_execution_time! + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index 6a4aee26d80..e15b51fbff4 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -32,9 +32,7 @@ module Gitlab def validate_external_files! @external_files.each do |file| - logger.instrument(:config_external_verify) do - raise IncludeError, file.error_message unless file.valid? - end + raise IncludeError, file.error_message unless file.valid? end end |