Merge branch 'feature/gb/allow-to-extend-keys-in-gitlab-ci-yml' into 'master'

Add support for advanced CI/CD config extension with `extends:`

Closes gitlab-ee#6136

See merge request gitlab-org/gitlab-ce!21243

5 files changed, 144 insertions, 7 deletions

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 66ac4a40616..46dad59eb8c 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -4,12 +4,17 @@ module Gitlab
     # Base GitLab CI Configuration facade
     #
     class Config
-      # EE would override this and utilize opts argument
+      ConfigError = Class.new(StandardError)
+
       def initialize(config, opts = {})
-        @config = Loader.new(config).load!
+        @config = Config::Extendable
+          .new(build_config(config, opts))
+          .to_hash
 
         @global = Entry::Global.new(@config)
         @global.compose!
+      rescue Loader::FormatError, Extendable::ExtensionError => e
+        raise Config::ConfigError, e.message
       end
 
       def valid?
@@ -58,6 +63,11 @@ module Gitlab
       def jobs
         @global.jobs_value
       end
+
+      # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb
+      def build_config(config, opts = {})
+        Loader.new(config).load!
+      end
     end
   end
 end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 91aac6df4b1..016a896bde5 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -9,9 +9,10 @@ module Gitlab
           include Configurable
           include Attributable
 
-          ALLOWED_KEYS = %i[tags script only except type image services allow_failure
-                            type stage when artifacts cache dependencies before_script
-                            after_script variables environment coverage retry].freeze
+          ALLOWED_KEYS = %i[tags script only except type image services
+                            allow_failure type stage when artifacts cache
+                            dependencies before_script after_script variables
+                            environment coverage retry extends].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -32,6 +33,7 @@ module Gitlab
                                       'always or manual' }
 
               validates :dependencies, array_of_strings: true
+              validates :extends, type: String
             end
           end
 
@@ -81,7 +83,8 @@ module Gitlab
                   :cache, :image, :services, :only, :except, :variables,
                   :artifacts, :commands, :environment, :coverage, :retry
 
-          attributes :script, :tags, :allow_failure, :when, :dependencies, :retry
+          attributes :script, :tags, :allow_failure, :when, :dependencies,
+                     :retry, :extends
 
           def compose!(deps = nil)
             super do
diff --git a/lib/gitlab/ci/config/extendable.rb b/lib/gitlab/ci/config/extendable.rb
new file mode 100644
index 00000000000..a43901c69fe
--- /dev/null
+++ b/lib/gitlab/ci/config/extendable.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      class Extendable
+        include Enumerable
+
+        ExtensionError = Class.new(StandardError)
+
+        def initialize(hash)
+          @hash = hash.to_h.deep_dup
+
+          each { |entry| entry.extend! if entry.extensible? }
+        end
+
+        def each
+          @hash.each_key do |key|
+            yield Extendable::Entry.new(key, @hash)
+          end
+        end
+
+        def to_hash
+          @hash.to_h
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/extendable/entry.rb b/lib/gitlab/ci/config/extendable/entry.rb
new file mode 100644
index 00000000000..7793db09d33
--- /dev/null
+++ b/lib/gitlab/ci/config/extendable/entry.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      class Extendable
+        class Entry
+          InvalidExtensionError = Class.new(Extendable::ExtensionError)
+          CircularDependencyError = Class.new(Extendable::ExtensionError)
+          NestingTooDeepError = Class.new(Extendable::ExtensionError)
+
+          MAX_NESTING_LEVELS = 10
+
+          attr_reader :key
+
+          def initialize(key, context, parent = nil)
+            @key = key
+            @context = context
+            @parent = parent
+
+            unless @context.key?(@key)
+              raise StandardError, 'Invalid entry key!'
+            end
+          end
+
+          def extensible?
+            value.is_a?(Hash) && value.key?(:extends)
+          end
+
+          def value
+            @value ||= @context.fetch(@key)
+          end
+
+          def base_hash!
+            @base ||= Extendable::Entry
+              .new(extends_key, @context, self)
+              .extend!
+          end
+
+          def extends_key
+            value.fetch(:extends).to_s.to_sym if extensible?
+          end
+
+          def ancestors
+            @ancestors ||= Array(@parent&.ancestors) + Array(@parent&.key)
+          end
+
+          def extend!
+            return value unless extensible?
+
+            if unknown_extension?
+              raise Entry::InvalidExtensionError,
+                    "#{key}: unknown key in `extends`"
+            end
+
+            if invalid_base?
+              raise Entry::InvalidExtensionError,
+                    "#{key}: invalid base hash in `extends`"
+            end
+
+            if nesting_too_deep?
+              raise Entry::NestingTooDeepError,
+                    "#{key}: nesting too deep in `extends`"
+            end
+
+            if circular_dependency?
+              raise Entry::CircularDependencyError,
+                    "#{key}: circular dependency detected in `extends`"
+            end
+
+            @context[key] = base_hash!.deep_merge(value)
+          end
+
+          private
+
+          def nesting_too_deep?
+            ancestors.count > MAX_NESTING_LEVELS
+          end
+
+          def circular_dependency?
+            ancestors.include?(key)
+          end
+
+          def unknown_extension?
+            !@context.key?(extends_key)
+          end
+
+          def invalid_base?
+            !@context[extends_key].is_a?(Hash)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index e829f2a95f8..5d1864ae9e2 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -16,7 +16,7 @@ module Gitlab
         end
 
         initial_parsing
-      rescue Gitlab::Ci::Config::Loader::FormatError => e
+      rescue Gitlab::Ci::Config::ConfigError => e
         raise ValidationError, e.message
       end