Add latest changes from gitlab-org/gitlab@master

8 files changed, 130 insertions, 44 deletions

diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index d5f0ddb0805..a5ec3e12b99 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -78,6 +78,10 @@ module API
             receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i
             if receive_max_input_size > 0
               payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}"
+
+              if Feature.enabled?(:gitaly_upload_pack_filter, project)
+                payload[:git_config_options] << "uploadpack.allowFilter=true" << "uploadpack.allowAnySHA1InWant=true"
+              end
             end
 
             response_with_status(**payload)
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 668e4a5e246..342dcb2f784 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -7,6 +7,8 @@ module Gitlab
     #
     class Config
       ConfigError = Class.new(StandardError)
+      TIMEOUT_SECONDS = 30.seconds
+      TIMEOUT_MESSAGE = 'Resolving config took longer than expected'
 
       RESCUE_ERRORS = [
         Gitlab::Config::Loader::FormatError,
@@ -17,17 +19,17 @@ module Gitlab
       attr_reader :root
 
       def initialize(config, project: nil, sha: nil, user: nil)
-        @config = Config::Extendable
-          .new(build_config(config, project: project, sha: sha, user: user))
-          .to_hash
+        @context = build_context(project: project, sha: sha, user: user)
+
+        if Feature.enabled?(:ci_limit_yaml_expansion, project, default_enabled: true)
+          @context.set_deadline(TIMEOUT_SECONDS)
+        end
+
+        @config = expand_config(config)
 
         @root = Entry::Root.new(@config)
         @root.compose!
 
-      rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e
-        Gitlab::Sentry.track_exception(e, extra: { user: user.inspect, project: project.inspect })
-        raise Config::ConfigError, e.message
-
       rescue *rescue_errors => e
         raise Config::ConfigError, e.message
       end
@@ -61,18 +63,34 @@ module Gitlab
 
       private
 
-      def build_config(config, project:, sha:, user:)
+      def expand_config(config)
+        build_config(config)
+
+      rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e
+        track_exception(e)
+        raise Config::ConfigError, e.message
+
+      rescue Gitlab::Ci::Config::External::Context::TimeoutError => e
+        track_exception(e)
+        raise Config::ConfigError, TIMEOUT_MESSAGE
+      end
+
+      def build_config(config)
         initial_config = Gitlab::Config::Loader::Yaml.new(config).load!
+        initial_config = Config::External::Processor.new(initial_config, @context).perform
 
-        process_external_files(initial_config, project: project, sha: sha, user: user)
+        Config::Extendable.new(initial_config).to_hash
       end
 
-      def process_external_files(config, project:, sha:, user:)
-        Config::External::Processor.new(config,
+      def build_context(project:, sha:, user:)
+        Config::External::Context.new(
           project: project,
           sha: sha || project&.repository&.root_ref_sha,
-          user: user,
-          expandset: Set.new).perform
+          user: user)
+      end
+
+      def track_exception(error)
+        Gitlab::Sentry.track_exception(error, extra: @context.sentry_payload)
       end
 
       # Overriden in EE
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
new file mode 100644
index 00000000000..bb4439cd069
--- /dev/null
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        class Context
+          TimeoutError = Class.new(StandardError)
+
+          attr_reader :project, :sha, :user
+          attr_reader :expandset, :execution_deadline
+
+          def initialize(project: nil, sha: nil, user: nil)
+            @project = project
+            @sha = sha
+            @user = user
+            @expandset = Set.new
+            @execution_deadline = 0
+
+            yield self if block_given?
+          end
+
+          def mutate(attrs = {})
+            self.class.new(**attrs) do |ctx|
+              ctx.expandset = expandset
+              ctx.execution_deadline = execution_deadline
+            end
+          end
+
+          def set_deadline(timeout_seconds)
+            @execution_deadline = current_monotonic_time + timeout_seconds.to_f
+          end
+
+          def check_execution_time!
+            raise TimeoutError if execution_expired?
+          end
+
+          def sentry_payload
+            {
+              user: user.inspect,
+              project: project.inspect
+            }
+          end
+
+          protected
+
+          attr_writer :expandset, :execution_deadline
+
+          private
+
+          def current_monotonic_time
+            Gitlab::Metrics::System.monotonic_time
+          end
+
+          def execution_expired?
+            return false if execution_deadline.zero?
+
+            current_monotonic_time > execution_deadline
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index c56d33544ba..4684a9eb981 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -12,8 +12,6 @@ module Gitlab
 
             YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
 
-            Context = Struct.new(:project, :sha, :user, :expandset)
-
             def initialize(params, context)
               @params = params
               @context = context
@@ -69,11 +67,16 @@ module Gitlab
             end
 
             def validate!
+              validate_execution_time!
               validate_location!
               validate_content! if errors.none?
               validate_hash! if errors.none?
             end
 
+            def validate_execution_time!
+              context.check_execution_time!
+            end
+
             def validate_location!
               if invalid_location_type?
                 errors.push("Included file `#{location}` needs to be a string")
@@ -95,11 +98,11 @@ module Gitlab
             end
 
             def expand_includes(hash)
-              External::Processor.new(hash, **expand_context).perform
+              External::Processor.new(hash, context.mutate(expand_context_attrs)).perform
             end
 
-            def expand_context
-              { project: nil, sha: nil, user: nil, expandset: context.expandset }
+            def expand_context_attrs
+              {}
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb
index cac321ec4a6..8cb1575a3e1 100644
--- a/lib/gitlab/ci/config/external/file/local.rb
+++ b/lib/gitlab/ci/config/external/file/local.rb
@@ -6,6 +6,7 @@ module Gitlab
       module External
         module File
           class Local < Base
+            extend ::Gitlab::Utils::Override
             include Gitlab::Utils::StrongMemoize
 
             def initialize(params, context)
@@ -34,11 +35,13 @@ module Gitlab
               context.project.repository.blob_data_at(context.sha, location)
             end
 
-            def expand_context
-              super.merge(
+            override :expand_context_attrs
+            def expand_context_attrs
+              {
                 project: context.project,
                 sha: context.sha,
-                user: context.user)
+                user: context.user
+              }
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index b828f77835c..c7b49b495fa 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -6,11 +6,12 @@ module Gitlab
       module External
         module File
           class Project < Base
+            extend ::Gitlab::Utils::Override
             include Gitlab::Utils::StrongMemoize
 
             attr_reader :project_name, :ref_name
 
-            def initialize(params, context = {})
+            def initialize(params, context)
               @location = params[:file]
               @project_name = params[:project]
               @ref_name = params[:ref] || 'HEAD'
@@ -65,11 +66,13 @@ module Gitlab
               end
             end
 
-            def expand_context
-              super.merge(
+            override :expand_context_attrs
+            def expand_context_attrs
+              {
                 project: project,
                 sha: sha,
-                user: context.user)
+                user: context.user
+              }
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index aff5c5b9651..5f7694e2cc6 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -21,14 +21,9 @@ module Gitlab
           DuplicateIncludesError = Class.new(Error)
           TooManyIncludesError = Class.new(Error)
 
-          def initialize(values, project:, sha:, user:, expandset:)
-            raise Error, 'Expanded needs to be `Set`' unless expandset.is_a?(Set)
-
+          def initialize(values, context)
             @locations = Array.wrap(values.fetch(:include, []))
-            @project = project
-            @sha = sha
-            @user = user
-            @expandset = expandset
+            @context = context
           end
 
           def process
@@ -43,7 +38,9 @@ module Gitlab
 
           private
 
-          attr_reader :locations, :project, :sha, :user, :expandset
+          attr_reader :locations, :context
+
+          delegate :expandset, to: :context
 
           # convert location if String to canonical form
           def normalize_location(location)
@@ -68,11 +65,11 @@ module Gitlab
             end
 
             # We scope location to context, as this allows us to properly support
-            # relative incldues, and similarly looking relative in another project
+            # relative includes, and similarly looking relative in another project
             # does not trigger duplicate error
             scoped_location = location.merge(
-              context_project: project,
-              context_sha: sha)
+              context_project: context.project,
+              context_sha: context.sha)
 
             unless expandset.add?(scoped_location)
               raise DuplicateIncludesError, "Include `#{location.to_json}` was already included!"
@@ -88,12 +85,6 @@ module Gitlab
 
             matching.first
           end
-
-          def context
-            strong_memoize(:context) do
-              External::File::Base::Context.new(project, sha, user, expandset)
-            end
-          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb
index 4a049ecae49..de69a1b1e8f 100644
--- a/lib/gitlab/ci/config/external/processor.rb
+++ b/lib/gitlab/ci/config/external/processor.rb
@@ -7,9 +7,9 @@ module Gitlab
         class Processor
           IncludeError = Class.new(StandardError)
 
-          def initialize(values, project:, sha:, user:, expandset:)
+          def initialize(values, context)
             @values = values
-            @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process
+            @external_files = External::Mapper.new(values, context).process
             @content = {}
           rescue External::Mapper::Error,
                  OpenSSL::SSL::SSLError => e