Allow to recursively expand includes

This change introduces a support for nesting the includes,
allowing to evaluate them in context of the target,
by properly respecting the relative inclusions and user permissions
of another projects, or templates.

6 files changed, 77 insertions, 12 deletions

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 5875479183e..15643fa03ac 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -84,7 +84,8 @@ module Gitlab
         Config::External::Processor.new(config,
           project: project,
           sha: sha || project.repository.root_ref_sha,
-          user: user).perform
+          user: user,
+          expandset: Set.new).perform
       end
     end
   end
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index a747886093c..2ffbb214a92 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -12,7 +12,7 @@ module Gitlab
 
             YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
 
-            Context = Struct.new(:project, :sha, :user)
+            Context = Struct.new(:project, :sha, :user, :expandset)
 
             def initialize(params, context)
               @params = params
@@ -43,13 +43,27 @@ module Gitlab
             end
 
             def to_hash
-              @hash ||= Gitlab::Config::Loader::Yaml.new(content).load!
-            rescue Gitlab::Config::Loader::FormatError
-              nil
+              expanded_content_hash
             end
 
             protected
 
+            def expanded_content_hash
+              return unless content_hash
+
+              strong_memoize(:expanded_content_yaml) do
+                expand_includes(content_hash)
+              end
+            end
+
+            def content_hash
+              strong_memoize(:content_yaml) do
+                Gitlab::Config::Loader::Yaml.new(content).load!
+              end
+            rescue Gitlab::Config::Loader::FormatError
+              nil
+            end
+
             def validate!
               validate_location!
               validate_content! if errors.none?
@@ -73,6 +87,14 @@ module Gitlab
                 errors.push("Included file `#{location}` does not have valid YAML syntax!")
               end
             end
+
+            def expand_includes(hash)
+              External::Processor.new(hash, **expand_context).perform
+            end
+
+            def expand_context
+              { project: nil, sha: nil, user: nil, expandset: context.expandset }
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb
index 2535d178ba8..229a06451e8 100644
--- a/lib/gitlab/ci/config/external/file/local.rb
+++ b/lib/gitlab/ci/config/external/file/local.rb
@@ -31,6 +31,13 @@ module Gitlab
             def fetch_local_content
               context.project.repository.blob_data_at(context.sha, location)
             end
+
+            def expand_context
+              super.merge(
+                project: context.project,
+                sha: context.sha,
+                user: context.user)
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index e75540dbe5a..b828f77835c 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -64,6 +64,13 @@ module Gitlab
                 project.commit(ref_name).try(:sha)
               end
             end
+
+            def expand_context
+              super.merge(
+                project: project,
+                sha: sha,
+                user: context.user)
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 108bfd5eb43..aff5c5b9651 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -7,6 +7,8 @@ module Gitlab
         class Mapper
           include Gitlab::Utils::StrongMemoize
 
+          MAX_INCLUDES = 50
+
           FILE_CLASSES = [
             External::File::Remote,
             External::File::Template,
@@ -14,25 +16,34 @@ module Gitlab
             External::File::Project
           ].freeze
 
-          AmbigiousSpecificationError = Class.new(StandardError)
+          Error = Class.new(StandardError)
+          AmbigiousSpecificationError = Class.new(Error)
+          DuplicateIncludesError = Class.new(Error)
+          TooManyIncludesError = Class.new(Error)
+
+          def initialize(values, project:, sha:, user:, expandset:)
+            raise Error, 'Expanded needs to be `Set`' unless expandset.is_a?(Set)
 
-          def initialize(values, project:, sha:, user:)
             @locations = Array.wrap(values.fetch(:include, []))
             @project = project
             @sha = sha
             @user = user
+            @expandset = expandset
           end
 
           def process
+            return [] if locations.empty?
+
             locations
               .compact
               .map(&method(:normalize_location))
+              .each(&method(:verify_duplicates!))
               .map(&method(:select_first_matching))
           end
 
           private
 
-          attr_reader :locations, :project, :sha, :user
+          attr_reader :locations, :project, :sha, :user, :expandset
 
           # convert location if String to canonical form
           def normalize_location(location)
@@ -51,6 +62,23 @@ module Gitlab
             end
           end
 
+          def verify_duplicates!(location)
+            if expandset.count >= MAX_INCLUDES
+              raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!"
+            end
+
+            # We scope location to context, as this allows us to properly support
+            # relative incldues, and similarly looking relative in another project
+            # does not trigger duplicate error
+            scoped_location = location.merge(
+              context_project: project,
+              context_sha: sha)
+
+            unless expandset.add?(scoped_location)
+              raise DuplicateIncludesError, "Include `#{location.to_json}` was already included!"
+            end
+          end
+
           def select_first_matching(location)
             matching = FILE_CLASSES.map do |file_class|
               file_class.new(location, context)
@@ -63,7 +91,7 @@ module Gitlab
 
           def context
             strong_memoize(:context) do
-              External::File::Base::Context.new(project, sha, user)
+              External::File::Base::Context.new(project, sha, user, expandset)
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb
index 69bc164a039..1dd2d42016a 100644
--- a/lib/gitlab/ci/config/external/processor.rb
+++ b/lib/gitlab/ci/config/external/processor.rb
@@ -7,11 +7,11 @@ module Gitlab
         class Processor
           IncludeError = Class.new(StandardError)
 
-          def initialize(values, project:, sha:, user:)
+          def initialize(values, project:, sha:, user:, expandset:)
             @values = values
-            @external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process
+            @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process
             @content = {}
-          rescue External::Mapper::AmbigiousSpecificationError => e
+          rescue External::Mapper::Error => e
             raise IncludeError, e.message
           end