Introducing new Syntax for Ci::Build inclusion rules

- Added Gitlab::Ci::Config::Entry::Rules and Gitlab::Ci::Config::Entry::Rules:Rule
  to handle lists of Rule objects to be evalauted for job inclusion
- Added `if:` and `changes:` as available Rules::Rule::Clause classes
- Added Rules handling logic to Seed::Build#included? with extra specs
- Use DisallowedKeysValidator to mutually exclude rules: from only:/except: on job config

11 files changed, 317 insertions, 24 deletions

diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb
index 0698136166a..e9c8864123f 100644
--- a/lib/gitlab/ci/build/policy/variables.rb
+++ b/lib/gitlab/ci/build/policy/variables.rb
@@ -10,7 +10,7 @@ module Gitlab
           end
 
           def satisfied_by?(pipeline, seed)
-            variables = seed.to_resource.scoped_variables_hash
+            variables = seed.scoped_variables_hash
 
             statements = @expressions.map do |statement|
               ::Gitlab::Ci::Pipeline::Expression::Statement
diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb
new file mode 100644
index 00000000000..89623a809c9
--- /dev/null
+++ b/lib/gitlab/ci/build/rules.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules
+        include ::Gitlab::Utils::StrongMemoize
+
+        Result = Struct.new(:when, :start_in)
+
+        def initialize(rule_hashes, default_when = 'on_success')
+          @rule_list    = Rule.fabricate_list(rule_hashes)
+          @default_when = default_when
+        end
+
+        def evaluate(pipeline, build)
+          if @rule_list.nil?
+            Result.new(@default_when)
+          elsif matched_rule = match_rule(pipeline, build)
+            Result.new(
+              matched_rule.attributes[:when] || @default_when,
+              matched_rule.attributes[:start_in]
+            )
+          else
+            Result.new('never')
+          end
+        end
+
+        private
+
+        def match_rule(pipeline, build)
+          @rule_list.find { |rule| rule.matches?(pipeline, build) }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/rules/rule.rb b/lib/gitlab/ci/build/rules/rule.rb
new file mode 100644
index 00000000000..8d52158c8d2
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule
+        attr_accessor :attributes
+
+        def self.fabricate_list(list)
+          list.map(&method(:new)) if list
+        end
+
+        def initialize(spec)
+          @clauses    = []
+          @attributes = {}
+
+          spec.each do |type, value|
+            if clause = Clause.fabricate(type, value)
+              @clauses << clause
+            else
+              @attributes.merge!(type => value)
+            end
+          end
+        end
+
+        def matches?(pipeline, build)
+          @clauses.all? { |clause| clause.satisfied_by?(pipeline, build) }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb
new file mode 100644
index 00000000000..ff0baf3348c
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule/clause.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause
+        ##
+        # Abstract class that defines an interface of a single
+        # job rule specification.
+        #
+        # Used for job's inclusion rules configuration.
+        #
+        UnknownClauseError = Class.new(StandardError)
+
+        def self.fabricate(type, value)
+          type = type.to_s.camelize
+
+          self.const_get(type).new(value) if self.const_defined?(type)
+        end
+
+        def initialize(spec)
+          @spec = spec
+        end
+
+        def satisfied_by?(pipeline, seed = nil)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb
new file mode 100644
index 00000000000..81d2ee6c24c
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause::Changes < Rules::Rule::Clause
+        def initialize(globs)
+          @globs = Array(globs)
+        end
+
+        def satisfied_by?(pipeline, seed)
+          return true if pipeline.modified_paths.nil?
+
+          pipeline.modified_paths.any? do |path|
+            @globs.any? do |glob|
+              File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb
new file mode 100644
index 00000000000..18c3b450f95
--- /dev/null
+++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause::If < Rules::Rule::Clause
+        def initialize(expression)
+          @expression = expression
+        end
+
+        def satisfied_by?(pipeline, seed)
+          variables = seed.scoped_variables_hash
+
+          ::Gitlab::Ci::Pipeline::Expression::Statement.new(@expression, variables).truthful?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 29a52b9da17..6e11c582750 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -11,7 +11,8 @@ module Gitlab
           include ::Gitlab::Config::Entry::Configurable
           include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[tags script only except type image services
+          ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze
+          ALLOWED_KEYS = %i[tags script only except rules type image services
                             allow_failure type stage when start_in artifacts cache
                             dependencies needs before_script after_script variables
                             environment coverage retry parallel extends].freeze
@@ -19,12 +20,19 @@ module Gitlab
           REQUIRED_BY_NEEDS = %i[stage].freeze
 
           validations do
+            validates :config, type: Hash
             validates :config, allowed_keys: ALLOWED_KEYS
             validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs?
             validates :config, presence: true
             validates :script, presence: true
             validates :name, presence: true
             validates :name, type: Symbol
+            validates :config,
+              disallowed_keys: {
+                in: %i[only except when start_in],
+                message: 'key may not be used with `rules`'
+              },
+              if: :has_rules?
 
             with_options allow_nil: true do
               validates :tags, array_of_strings: true
@@ -32,17 +40,19 @@ module Gitlab
               validates :parallel, numericality: { only_integer: true,
                                                    greater_than_or_equal_to: 2,
                                                    less_than_or_equal_to: 50 }
-              validates :when,
-                inclusion: { in: %w[on_success on_failure always manual delayed],
-                             message: 'should be on_success, on_failure, ' \
-                                      'always, manual or delayed' }
+              validates :when, inclusion: {
+                in: ALLOWED_WHEN,
+                message: "should be one of: #{ALLOWED_WHEN.join(', ')}"
+              }
+
               validates :dependencies, array_of_strings: true
               validates :needs, array_of_strings: true
               validates :extends, array_of_strings_or_string: true
+              validates :rules, array_of_hashes: true
             end
 
             validates :start_in, duration: { limit: '1 day' }, if: :delayed?
-            validates :start_in, absence: true, unless: :delayed?
+            validates :start_in, absence: true, if: -> { has_rules? || !delayed? }
 
             validate do
               next unless dependencies.present?
@@ -91,6 +101,9 @@ module Gitlab
           entry :except, Entry::Policy,
             description: 'Refs policy this job will be executed for.'
 
+          entry :rules, Entry::Rules,
+            description: 'List of evaluable Rules to determine job inclusion.'
+
           entry :variables, Entry::Variables,
             description: 'Environment variables available for this job.'
 
@@ -112,7 +125,7 @@ module Gitlab
                   :parallel, :needs
 
           attributes :script, :tags, :allow_failure, :when, :dependencies,
-                     :needs, :retry, :parallel, :extends, :start_in
+                     :needs, :retry, :parallel, :extends, :start_in, :rules
 
           def self.matching?(name, config)
             !name.to_s.start_with?('.') &&
@@ -151,6 +164,10 @@ module Gitlab
             self.when == 'delayed'
           end
 
+          def has_rules?
+            @config.try(:key?, :rules)
+          end
+
           def ignored?
             allow_failure.nil? ? manual_action? : allow_failure
           end
diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb
new file mode 100644
index 00000000000..65cad0880f5
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/rules.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Rules < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+
+          validations do
+            validates :config, presence: true
+            validates :config, type: Array
+          end
+
+          def compose!(deps = nil)
+            super(deps) do
+              @config.each_with_index do |rule, index|
+                @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule)
+                  .value(rule)
+                  .with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord
+                  .create!
+              end
+
+              @entries.each_value do |entry|
+                entry.compose!(deps)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb
new file mode 100644
index 00000000000..1f2a34ec90e
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/rules/rule.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Rules::Rule < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+          include ::Gitlab::Config::Entry::Attributable
+
+          CLAUSES      = %i[if changes].freeze
+          ALLOWED_KEYS = %i[if changes when start_in].freeze
+          ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze
+
+          attributes :if, :changes, :when, :start_in
+
+          validations do
+            validates :config, presence: true
+            validates :config, type: { with: Hash }
+            validates :config, allowed_keys: ALLOWED_KEYS
+            validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay?
+            validates :start_in, presence: true, if: :specifies_delay?
+            validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay?
+
+            with_options allow_nil: true do
+              validates :if, expression: true
+              validates :changes, array_of_strings: true
+              validates :when, allowed_values: { in: ALLOWED_WHEN }
+            end
+          end
+
+          def specifies_delay?
+            self.when == 'delayed'
+          end
+
+          def default
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 7ec03d132c0..1066331062b 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -7,7 +7,7 @@ module Gitlab
         class Build < Seed::Base
           include Gitlab::Utils::StrongMemoize
 
-          delegate :dig, to: :@attributes
+          delegate :dig, to: :@seed_attributes
 
           # When the `ci_dag_limit_needs` is enabled it uses the lower limit
           LOW_NEEDS_LIMIT = 5
@@ -15,14 +15,20 @@ module Gitlab
 
           def initialize(pipeline, attributes, previous_stages)
             @pipeline = pipeline
-            @attributes = attributes
+            @seed_attributes = attributes
             @previous_stages = previous_stages
             @needs_attributes = dig(:needs_attributes)
 
+            @using_rules  = attributes.key?(:rules)
+            @using_only   = attributes.key?(:only)
+            @using_except = attributes.key?(:except)
+
             @only = Gitlab::Ci::Build::Policy
               .fabricate(attributes.delete(:only))
             @except = Gitlab::Ci::Build::Policy
               .fabricate(attributes.delete(:except))
+            @rules = Gitlab::Ci::Build::Rules
+              .new(attributes.delete(:rules))
           end
 
           def name
@@ -31,8 +37,13 @@ module Gitlab
 
           def included?
             strong_memoize(:inclusion) do
-              all_of_only? &&
-                none_of_except?
+              if @using_rules
+                included_by_rules?
+              elsif @using_only || @using_except
+                all_of_only? && none_of_except?
+              else
+                true
+              end
             end
           end
 
@@ -45,19 +56,13 @@ module Gitlab
           end
 
           def attributes
-            @attributes.merge(
-              pipeline: @pipeline,
-              project: @pipeline.project,
-              user: @pipeline.user,
-              ref: @pipeline.ref,
-              tag: @pipeline.tag,
-              trigger_request: @pipeline.legacy_trigger,
-              protected: @pipeline.protected_ref?
-            )
+            @seed_attributes
+              .deep_merge(pipeline_attributes)
+              .deep_merge(rules_attributes)
           end
 
           def bridge?
-            attributes_hash = @attributes.to_h
+            attributes_hash = @seed_attributes.to_h
             attributes_hash.dig(:options, :trigger).present? ||
               (attributes_hash.dig(:options, :bridge_needs).instance_of?(Hash) &&
                attributes_hash.dig(:options, :bridge_needs, :pipeline).present?)
@@ -73,6 +78,18 @@ module Gitlab
             end
           end
 
+          def scoped_variables_hash
+            strong_memoize(:scoped_variables_hash) do
+              # This is a temporary piece of technical debt to allow us access
+              # to the CI variables to evaluate rules before we persist a Build
+              # with the result. We should refactor away the extra Build.new,
+              # but be able to get CI Variables directly from the Seed::Build.
+              ::Ci::Build.new(
+                @seed_attributes.merge(pipeline_attributes)
+              ).scoped_variables_hash
+            end
+          end
+
           private
 
           def all_of_only?
@@ -109,6 +126,28 @@ module Gitlab
               HARD_NEEDS_LIMIT
             end
           end
+
+          def pipeline_attributes
+            {
+              pipeline: @pipeline,
+              project: @pipeline.project,
+              user: @pipeline.user,
+              ref: @pipeline.ref,
+              tag: @pipeline.tag,
+              trigger_request: @pipeline.legacy_trigger,
+              protected: @pipeline.protected_ref?
+            }
+          end
+
+          def included_by_rules?
+            rules_attributes[:when] != 'never'
+          end
+
+          def rules_attributes
+            strong_memoize(:rules_attributes) do
+              @using_rules ? @rules.evaluate(@pipeline, self).to_h.compact : {}
+            end
+          end
         end
       end
     end
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
index 0289e675c6b..374f929878e 100644
--- a/lib/gitlab/config/entry/validators.rb
+++ b/lib/gitlab/config/entry/validators.rb
@@ -20,8 +20,10 @@ module Gitlab
             present_keys = value.try(:keys).to_a & options[:in]
 
             if present_keys.any?
-              record.errors.add(attribute, "contains disallowed keys: " +
-                present_keys.join(', '))
+              message = options[:message] || "contains disallowed keys"
+              message += ": #{present_keys.join(', ')}"
+
+              record.errors.add(attribute, message)
             end
           end
         end
@@ -65,6 +67,16 @@ module Gitlab
           end
         end
 
+        class ArrayOfHashesValidator < ActiveModel::EachValidator
+          include LegacyValidationHelpers
+
+          def validate_each(record, attribute, value)
+            unless value.is_a?(Array) && value.map { |hsh| hsh.is_a?(Hash) }.all?
+              record.errors.add(attribute, 'should be an array of hashes')
+            end
+          end
+        end
+
         class ArrayOrStringValidator < ActiveModel::EachValidator
           def validate_each(record, attribute, value)
             unless value.is_a?(Array) || value.is_a?(String)
@@ -231,6 +243,14 @@ module Gitlab
           end
         end
 
+        class ExpressionValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            unless value.is_a?(String) && ::Gitlab::Ci::Pipeline::Expression::Statement.new(value).valid?
+              record.errors.add(attribute, 'Invalid expression syntax')
+            end
+          end
+        end
+
         class PortNamePresentAndUniqueValidator < ActiveModel::EachValidator
           def validate_each(record, attribute, value)
             return unless value.is_a?(Array)