Merge branch 'include-templates' into 'master'

Include templates

Closes #53445

See merge request gitlab-org/gitlab-ce!23495

7 files changed, 121 insertions, 21 deletions

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 6333799a491..11e0352975d 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -83,7 +83,7 @@ module Gitlab
 
       def process_external_files(config, project, opts)
         sha = opts.fetch(:sha) { project.repository.root_ref_sha }
-        Config::External::Processor.new(config, project, sha).perform
+        Config::External::Processor.new(config, project: project, sha: sha).perform
       end
     end
   end
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index ee4ea9bbb1d..2ac6656a703 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -8,20 +8,26 @@ module Gitlab
           class Base
             include Gitlab::Utils::StrongMemoize
 
-            attr_reader :location, :opts, :errors
+            attr_reader :location, :params, :context, :errors
 
             YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
 
-            def initialize(location, opts = {})
-              @location = location
-              @opts = opts
+            Context = Struct.new(:project, :sha)
+
+            def initialize(params, context)
+              @params = params
+              @context = context
               @errors = []
 
               validate!
             end
 
+            def matching?
+              location.present?
+            end
+
             def invalid_extension?
-              !::File.basename(location).match(YAML_WHITELIST_EXTENSION)
+              location.nil? || !::File.basename(location).match?(YAML_WHITELIST_EXTENSION)
             end
 
             def valid?
diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb
index 2a256aff65c..2535d178ba8 100644
--- a/lib/gitlab/ci/config/external/file/local.rb
+++ b/lib/gitlab/ci/config/external/file/local.rb
@@ -8,11 +8,8 @@ module Gitlab
           class Local < Base
             include Gitlab::Utils::StrongMemoize
 
-            attr_reader :project, :sha
-
-            def initialize(location, opts = {})
-              @project = opts.fetch(:project)
-              @sha = opts.fetch(:sha)
+            def initialize(params, context)
+              @location = params[:local]
 
               super
             end
@@ -32,7 +29,7 @@ module Gitlab
             end
 
             def fetch_local_content
-              project.repository.blob_data_at(sha, location)
+              context.project.repository.blob_data_at(context.sha, location)
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb
index 86fa5ad8800..567a86c47e5 100644
--- a/lib/gitlab/ci/config/external/file/remote.rb
+++ b/lib/gitlab/ci/config/external/file/remote.rb
@@ -8,6 +8,12 @@ module Gitlab
           class Remote < Base
             include Gitlab::Utils::StrongMemoize
 
+            def initialize(params, context)
+              @location = params[:remote]
+
+              super
+            end
+
             def content
               strong_memoize(:content) { fetch_remote_content }
             end
diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb
new file mode 100644
index 00000000000..54f4cf74c4d
--- /dev/null
+++ b/lib/gitlab/ci/config/external/file/template.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        module File
+          class Template < Base
+            attr_reader :location, :project
+
+            SUFFIX = '.gitlab-ci.yml'.freeze
+
+            def initialize(params, context)
+              @location = params[:template]
+
+              super
+            end
+
+            def content
+              strong_memoize(:content) { fetch_template_content }
+            end
+
+            private
+
+            def validate_location!
+              super
+
+              unless template_name_valid?
+                errors.push("Template file `#{location}` is not a valid location!")
+              end
+            end
+
+            def template_name
+              return unless template_name_valid?
+
+              location.first(-SUFFIX.length)
+            end
+
+            def template_name_valid?
+              location.to_s.end_with?(SUFFIX)
+            end
+
+            def fetch_template_content
+              Gitlab::Template::GitlabCiYmlTemplate.find(template_name, project)&.content
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index def3563e505..74bd927da39 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -5,25 +5,63 @@ module Gitlab
     class Config
       module External
         class Mapper
-          def initialize(values, project, sha)
-            @locations = Array(values.fetch(:include, []))
+          include Gitlab::Utils::StrongMemoize
+
+          FILE_CLASSES = [
+            External::File::Remote,
+            External::File::Template,
+            External::File::Local
+          ].freeze
+
+          AmbigiousSpecificationError = Class.new(StandardError)
+
+          def initialize(values, project:, sha:)
+            @locations = Array.wrap(values.fetch(:include, []))
             @project = project
             @sha = sha
           end
 
           def process
-            locations.map { |location| build_external_file(location) }
+            locations
+              .compact
+              .map(&method(:normalize_location))
+              .map(&method(:select_first_matching))
           end
 
           private
 
-          attr_reader :locations, :project, :sha
+          attr_reader :locations, :project, :sha, :user
+
+          # convert location if String to canonical form
+          def normalize_location(location)
+            if location.is_a?(String)
+              normalize_location_string(location)
+            else
+              location.deep_symbolize_keys
+            end
+          end
 
-          def build_external_file(location)
+          def normalize_location_string(location)
             if ::Gitlab::UrlSanitizer.valid?(location)
-              External::File::Remote.new(location)
+              { remote: location }
             else
-              External::File::Local.new(location, project: project, sha: sha)
+              { local: location }
+            end
+          end
+
+          def select_first_matching(location)
+            matching = FILE_CLASSES.map do |file_class|
+              file_class.new(location, context)
+            end.select(&:matching?)
+
+            raise AmbigiousSpecificationError, "Include `#{location.to_json}` needs to match exactly one accessor!" unless matching.one?
+
+            matching.first
+          end
+
+          def context
+            strong_memoize(:context) do
+              External::File::Base::Context.new(project, sha)
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb
index eae0bdeb644..1d310b29dc8 100644
--- a/lib/gitlab/ci/config/external/processor.rb
+++ b/lib/gitlab/ci/config/external/processor.rb
@@ -7,10 +7,12 @@ module Gitlab
         class Processor
           IncludeError = Class.new(StandardError)
 
-          def initialize(values, project, sha)
+          def initialize(values, project:, sha:)
             @values = values
-            @external_files = External::Mapper.new(values, project, sha).process
+            @external_files = External::Mapper.new(values, project: project, sha: sha).process
             @content = {}
+          rescue External::Mapper::AmbigiousSpecificationError => e
+            raise IncludeError, e.message
           end
 
           def perform