Add latest changes from gitlab-org/gitlab@master

6 files changed, 174 insertions, 5 deletions

diff --git a/lib/gitlab/ci/config/entry/files.rb b/lib/gitlab/ci/config/entry/files.rb
new file mode 100644
index 00000000000..d0d6a36d754
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/files.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents an array of file paths.
+        #
+        class Files < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+
+          validations do
+            validates :config, array_of_strings: true
+            validates :config, length: {
+              minimum: 1,
+              maximum: 2,
+              too_short: 'requires at least %{count} item',
+              too_long: 'has too many items (maximum is %{count})'
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb
index 0c10967e629..f12f0919348 100644
--- a/lib/gitlab/ci/config/entry/key.rb
+++ b/lib/gitlab/ci/config/entry/key.rb
@@ -7,11 +7,48 @@ module Gitlab
         ##
         # Entry that represents a key.
         #
-        class Key < ::Gitlab::Config::Entry::Node
-          include ::Gitlab::Config::Entry::Validatable
+        class Key < ::Gitlab::Config::Entry::Simplifiable
+          strategy :SimpleKey, if: -> (config) { config.is_a?(String) || config.is_a?(Symbol) }
+          strategy :ComplexKey, if: -> (config) { config.is_a?(Hash) }
 
-          validations do
-            validates :config, key: true
+          class SimpleKey < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+
+            validations do
+              validates :config, key: true
+            end
+
+            def self.default
+              'default'
+            end
+
+            def value
+              super.to_s
+            end
+          end
+
+          class ComplexKey < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Attributable
+            include ::Gitlab::Config::Entry::Configurable
+
+            ALLOWED_KEYS = %i[files prefix].freeze
+            REQUIRED_KEYS = %i[files].freeze
+
+            validations do
+              validates :config, allowed_keys: ALLOWED_KEYS
+              validates :config, required_keys: REQUIRED_KEYS
+            end
+
+            entry :files, Entry::Files,
+              description: 'Files that should be used to build the key'
+            entry :prefix, Entry::Prefix,
+              description: 'Prefix that is added to the final cache key'
+          end
+
+          class UnknownStrategy < ::Gitlab::Config::Entry::Node
+            def errors
+              ["#{location} should be a hash, a string or a symbol"]
+            end
           end
 
           def self.default
diff --git a/lib/gitlab/ci/config/entry/prefix.rb b/lib/gitlab/ci/config/entry/prefix.rb
new file mode 100644
index 00000000000..3244ad6d611
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/prefix.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a key prefix.
+        #
+        class Prefix < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+
+          validations do
+            validates :config, key: true
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index fc9c540088b..1d698a32ba8 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -29,6 +29,8 @@ module Gitlab
               .fabricate(attributes.delete(:except))
             @rules = Gitlab::Ci::Build::Rules
               .new(attributes.delete(:rules))
+            @cache = Seed::Build::Cache
+              .new(pipeline, attributes.delete(:cache))
           end
 
           def name
@@ -59,6 +61,7 @@ module Gitlab
             @seed_attributes
               .deep_merge(pipeline_attributes)
               .deep_merge(rules_attributes)
+              .deep_merge(cache_attributes)
           end
 
           def bridge?
@@ -150,6 +153,12 @@ module Gitlab
               @using_rules ? @rules.evaluate(@pipeline, self).build_attributes : {}
             end
           end
+
+          def cache_attributes
+            strong_memoize(:cache_attributes) do
+              @cache.build_attributes
+            end
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb
new file mode 100644
index 00000000000..7671035b896
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Seed
+        class Build
+          class Cache
+            def initialize(pipeline, cache)
+              @pipeline = pipeline
+              local_cache = cache.to_h.deep_dup
+              @key = local_cache.delete(:key)
+              @paths = local_cache.delete(:paths)
+              @policy = local_cache.delete(:policy)
+              @untracked = local_cache.delete(:untracked)
+
+              raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any?
+            end
+
+            def build_attributes
+              {
+                options: {
+                  cache: {
+                    key: key_string,
+                    paths: @paths,
+                    policy: @policy,
+                    untracked: @untracked
+                  }.compact.presence
+                }.compact
+              }
+            end
+
+            private
+
+            def key_string
+              key_from_string || key_from_files
+            end
+
+            def key_from_string
+              @key.to_s if @key.is_a?(String) || @key.is_a?(Symbol)
+            end
+
+            def key_from_files
+              return unless @key.is_a?(Hash)
+
+              [@key[:prefix], files_digest].select(&:present?).join('-')
+            end
+
+            def files_digest
+              hash_of_the_latest_changes || 'default'
+            end
+
+            def hash_of_the_latest_changes
+              return unless Feature.enabled?(:ci_file_based_cache, @pipeline.project, default_enabled: true)
+
+              ids = files.map { |path| last_commit_id_for_path(path) }
+              ids = ids.compact.sort.uniq
+
+              Digest::SHA1.hexdigest(ids.join('-')) if ids.any?
+            end
+
+            def files
+              @key[:files]
+                .to_a
+                .select(&:present?)
+                .uniq
+            end
+
+            def last_commit_id_for_path(path)
+              @pipeline.project.repository.last_commit_id_for_path(@pipeline.sha, path)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index c2a55fa8b1b..f48ffa9c1f7 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -43,11 +43,11 @@ module Gitlab
           needs_attributes: job.dig(:needs, :job),
           interruptible: job[:interruptible],
           rules: job[:rules],
+          cache: job[:cache],
           options: {
             image: job[:image],
             services: job[:services],
             artifacts: job[:artifacts],
-            cache: job[:cache],
             dependencies: job[:dependencies],
             job_timeout: job[:timeout],
             before_script: job[:before_script],