diff options
Diffstat (limited to 'lib/gitlab/ci/config')
| -rw-r--r-- | lib/gitlab/ci/config/entry/artifacts.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/entry/bridge.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/entry/job.rb | 54 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/entry/processable.rb | 20 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/entry/product/matrix.rb | 61 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/entry/product/parallel.rb | 57 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/entry/product/variables.rb | 36 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/external/context.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/normalizer.rb | 20 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/normalizer/factory.rb | 38 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/normalizer/matrix_strategy.rb | 68 | ||||
| -rw-r--r-- | lib/gitlab/ci/config/normalizer/number_strategy.rb | 47 |
12 files changed, 375 insertions, 32 deletions
diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index a9a9636637f..206dbaea272 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -42,7 +42,7 @@ module Gitlab inclusion: { in: %w[on_success on_failure always], message: 'should be on_success, on_failure ' \ 'or always' } - validates :expire_in, duration: true + validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::Artifacts::ExpireInParser } end end diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index f4362d3b0ce..a8b67a1db4f 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -11,7 +11,7 @@ module Gitlab class Bridge < ::Gitlab::Config::Entry::Node include ::Gitlab::Ci::Config::Entry::Processable - ALLOWED_KEYS = %i[trigger allow_failure when needs].freeze + ALLOWED_KEYS = %i[trigger].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a615cab1a80..f960cec1f26 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,9 +11,8 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Processable ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze - ALLOWED_KEYS = %i[tags script type image services - allow_failure type when start_in artifacts cache - dependencies before_script needs after_script + ALLOWED_KEYS = %i[tags script type image services start_in artifacts + cache dependencies before_script after_script environment coverage retry parallel interruptible timeout resource_group release secrets].freeze @@ -23,18 +22,9 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :script, presence: true - validates :config, - disallowed_keys: { - in: %i[release], - message: 'release features are not enabled' - }, - unless: -> { Gitlab::Ci::Features.release_generation_enabled? } with_options allow_nil: true do validates :allow_failure, boolean: true - validates :parallel, numericality: { only_integer: true, - greater_than_or_equal_to: 2, - less_than_or_equal_to: 50 } validates :when, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" @@ -124,13 +114,47 @@ module Gitlab description: 'This job will produce a release.', inherit: false + entry :parallel, Entry::Product::Parallel, + description: 'Parallel configuration for this job.', + inherit: false + attributes :script, :tags, :allow_failure, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release + Matcher = Struct.new(:name, :config) do + def applies? + job_is_not_hidden? && + config_is_a_hash? && + has_job_keys? + end + + private + + def job_is_not_hidden? + !name.to_s.start_with?('.') + end + + def config_is_a_hash? + config.is_a?(Hash) + end + + def has_job_keys? + if name == :default + config.key?(:script) + else + (ALLOWED_KEYS & config.keys).any? + end + end + end + def self.matching?(name, config) - !name.to_s.start_with?('.') && - config.is_a?(Hash) && config.key?(:script) + if Gitlab::Ci::Features.job_entry_matches_all_keys? + Matcher.new(name, config).applies? + else + !name.to_s.start_with?('.') && + config.is_a?(Hash) && config.key?(:script) + end end def self.visible? @@ -174,7 +198,7 @@ module Gitlab environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, - parallel: has_parallel? ? parallel.to_i : nil, + parallel: has_parallel? ? parallel_value : nil, interruptible: interruptible_defined? ? interruptible_value : nil, timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, artifacts: artifacts_value, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index b4539475d88..f10c509d0cc 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -14,7 +14,8 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Inheritable - PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit].freeze + PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables + inherit allow_failure when needs].freeze included do validations do @@ -82,8 +83,8 @@ module Gitlab @entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables end - if has_rules? && !has_workflow_rules && Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? - add_warning('uses `rules` without defining `workflow:rules`') + unless has_workflow_rules + validate_against_warnings end # inherit root variables @@ -93,6 +94,19 @@ module Gitlab end end + def validate_against_warnings + # If rules are valid format and workflow rules are not specified + return unless rules_value + return unless Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? + + last_rule = rules_value.last + + if last_rule&.keys == [:when] && last_rule[:when] != 'never' + docs_url = 'read more: https://docs.gitlab.com/ee/ci/troubleshooting.html#pipeline-warnings' + add_warning("may allow multiple pipelines to run for a single action due to `rules:when` clause with no `workflow:rules` - #{docs_url}") + end + end + def name metadata[:name] end diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb new file mode 100644 index 00000000000..6af809d46c1 --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents matrix style parallel builds. + # + module Product + class Matrix < ::Gitlab::Config::Entry::Node + include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + validations do + validates :config, array_of_hashes: true + + validate on: :composed do + limit = Entry::Product::Parallel::PARALLEL_LIMIT + + if number_of_generated_jobs > limit + errors.add(:config, "generates too many jobs (maximum is #{limit})") + end + end + end + + def compose!(deps = nil) + super(deps) do + @config.each_with_index do |variables, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Product::Variables) + .value(variables) + .with(parent: self, description: 'matrix variables definition.') # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + + def value + strong_memoize(:value) do + @entries.values.map(&:value) + end + end + + # rubocop:disable CodeReuse/ActiveRecord + def number_of_generated_jobs + value.sum do |config| + config.values.reduce(1) { |acc, values| acc * values.size } + end + end + # rubocop:enable CodeReuse/ActiveRecord + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb new file mode 100644 index 00000000000..cd9eabbbc66 --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a parallel job config. + # + module Product + class Parallel < ::Gitlab::Config::Entry::Simplifiable + strategy :ParallelBuilds, if: -> (config) { config.is_a?(Numeric) } + strategy :MatrixBuilds, if: -> (config) { config.is_a?(Hash) } + + PARALLEL_LIMIT = 50 + + class ParallelBuilds < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, numericality: { only_integer: true, + greater_than_or_equal_to: 2, + less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT }, + allow_nil: true + end + + def value + { number: super.to_i } + end + end + + class MatrixBuilds < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable + + PERMITTED_KEYS = %i[matrix].freeze + + validations do + validates :config, allowed_keys: PERMITTED_KEYS + validates :config, required_keys: PERMITTED_KEYS + end + + entry :matrix, Entry::Product::Matrix, + description: 'Variables definition for matrix builds' + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be an integer or a hash"] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb new file mode 100644 index 00000000000..ac4f70fb69e --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents variables for parallel matrix builds. + # + module Product + class Variables < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, variables: { array_values: true } + validates :config, length: { + minimum: 2, + too_short: 'requires at least %{count} items' + } + end + + def self.default(**) + {} + end + + def value + @config + .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } + .to_h + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 814dcc66362..cf6c2961ee7 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -54,7 +54,7 @@ module Gitlab end def execution_expired? - return false if execution_deadline.zero? + return false if execution_deadline == 0 current_monotonic_time > execution_deadline end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 1139efee9e8..451ba14bb89 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -32,7 +32,7 @@ module Gitlab return unless job_names job_names.flat_map do |job_name| - parallelized_jobs[job_name.to_sym] || job_name + parallelized_jobs[job_name.to_sym]&.map(&:name) || job_name end end @@ -42,10 +42,8 @@ module Gitlab job_needs.flat_map do |job_need| job_need_name = job_need[:name].to_sym - if all_job_names = parallelized_jobs[job_need_name] - all_job_names.map do |job_name| - job_need.merge(name: job_name) - end + if all_jobs = parallelized_jobs[job_need_name] + all_jobs.map { |job| job_need.merge(name: job.name) } else job_need end @@ -57,7 +55,7 @@ module Gitlab @jobs_config.each_with_object({}) do |(job_name, config), hash| next unless config[:parallel] - hash[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + hash[job_name] = parallelize_job_config(job_name, config[:parallel]) end end end @@ -65,9 +63,9 @@ module Gitlab def expand_parallelize_jobs @jobs_config.each_with_object({}) do |(job_name, config), hash| if parallelized_jobs.key?(job_name) - parallelized_jobs[job_name].each_with_index do |name, index| - hash[name.to_sym] = - yield(name, config.merge(name: name, instance: index + 1)) + parallelized_jobs[job_name].each do |job| + hash[job.name.to_sym] = + yield(job.name, config.deep_merge(job.attributes)) end else hash[job_name] = yield(job_name, config) @@ -75,8 +73,8 @@ module Gitlab end end - def self.parallelize_job_names(name, total) - Array.new(total) { |index| "#{name} #{index + 1}/#{total}" } + def parallelize_job_config(name, config) + Normalizer::Factory.new(name, config).create end end end diff --git a/lib/gitlab/ci/config/normalizer/factory.rb b/lib/gitlab/ci/config/normalizer/factory.rb new file mode 100644 index 00000000000..bf813f8e878 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/factory.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class Factory + include Gitlab::Utils::StrongMemoize + + def initialize(name, config) + @name = name + @config = config + end + + def create + return [] unless strategy + + strategy.build_from(@name, @config) + end + + private + + def strategy + strong_memoize(:strategy) do + strategies.find do |strategy| + strategy.applies_to?(@config) + end + end + end + + def strategies + [NumberStrategy, MatrixStrategy] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb new file mode 100644 index 00000000000..db21274a9ed --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class MatrixStrategy + class << self + def applies_to?(config) + config.is_a?(Hash) && config.key?(:matrix) + end + + def build_from(job_name, initial_config) + config = expand(initial_config[:matrix]) + total = config.size + + config.map.with_index do |vars, index| + new(job_name, index.next, vars, total) + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def expand(config) + config.flat_map do |config| + values = config.values + + values[0] + .product(*values.from(1)) + .map { |vals| config.keys.zip(vals).to_h } + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + + def initialize(job_name, instance, variables, total) + @job_name = job_name + @instance = instance + @variables = variables.to_h + @total = total + end + + def attributes + { + name: name, + instance: instance, + variables: variables, + parallel: { total: total } + } + end + + def name_with_details + vars = variables.map { |key, value| "#{key}=#{value}"}.join('; ') + + "#{job_name} (#{vars})" + end + + def name + "#{job_name} #{instance}/#{total}" + end + + private + + attr_reader :job_name, :instance, :variables, :total + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/number_strategy.rb b/lib/gitlab/ci/config/normalizer/number_strategy.rb new file mode 100644 index 00000000000..4754e7b46d4 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/number_strategy.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class NumberStrategy + class << self + def applies_to?(config) + config.is_a?(Integer) || config.is_a?(Hash) && config.key?(:number) + end + + def build_from(job_name, config) + total = config.is_a?(Hash) ? config[:number] : config + + Array.new(total) do |index| + new(job_name, index.next, total) + end + end + end + + def initialize(job_name, instance, total) + @job_name = job_name + @instance = instance + @total = total + end + + def attributes + { + name: name, + instance: instance, + parallel: { total: total } + } + end + + def name + "#{job_name} #{instance}/#{total}" + end + + private + + attr_reader :job_name, :instance, :total + end + end + end + end +end |