Add latest changes from gitlab-org/gitlab@master

8 files changed, 62 insertions, 25 deletions

diff --git a/lib/api/entities/snippet.rb b/lib/api/entities/snippet.rb
index 709566944ed..ee652225ba0 100644
--- a/lib/api/entities/snippet.rb
+++ b/lib/api/entities/snippet.rb
@@ -26,3 +26,5 @@ module API
     end
   end
 end
+
+API::Entities::Snippet.prepend_mod_with('API::Entities::Snippet', with_descendants: true)
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index e1207e7e222..b7f21bd6c22 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -757,6 +757,9 @@ module API
         @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user! }
       rescue Gitlab::Auth::UnauthorizedError
         unauthorized!
+
+        # Explicitly return `nil`, otherwise an instance of `Rack::Response` is returned when reporting an error
+        nil
       end
     end
     # rubocop:enable Gitlab/ModuleWithInstanceVariables
diff --git a/lib/api/helpers/snippets_helpers.rb b/lib/api/helpers/snippets_helpers.rb
index fe20fb3cbe2..241e92e9d10 100644
--- a/lib/api/helpers/snippets_helpers.rb
+++ b/lib/api/helpers/snippets_helpers.rb
@@ -46,6 +46,9 @@ module API
         at_least_one_of :content, :description, :files, :file_name, :title, :visibility
       end
 
+      params :optional_list_params_ee do # rubocop:disable Lint/EmptyBlock
+      end
+
       def content_for(snippet)
         if snippet.empty_repo?
           env['api.format'] = :txt
@@ -96,3 +99,5 @@ module API
     end
   end
 end
+
+API::Helpers::SnippetsHelpers.prepend_mod_with('API::Helpers::SnippetsHelpers')
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
index c17d8462988..4f3c1499549 100644
--- a/lib/api/snippets.rb
+++ b/lib/api/snippets.rb
@@ -8,18 +8,19 @@ module API
     feature_category :source_code_management
     urgency :low
 
+    helpers do
+      def find_snippets(user: current_user, params: {})
+        SnippetsFinder.new(user, params).execute
+      end
+
+      def snippets_for_current_user
+        find_snippets(params: { author: current_user })
+      end
+    end
+
     resource :snippets do
       helpers Helpers::SnippetsHelpers
       helpers SpammableActions::CaptchaCheck::RestApiActionsSupport
-      helpers do
-        def snippets_for_current_user
-          SnippetsFinder.new(current_user, author: current_user).execute
-        end
-
-        def snippets
-          SnippetsFinder.new(current_user).execute
-        end
-      end
 
       desc 'Get a snippets list for an authenticated user' do
         detail 'This feature was introduced in GitLab 8.15.'
@@ -41,7 +42,7 @@ module API
 
         filter_params = declared_params(include_missing: false).merge(author: current_user)
 
-        present paginate(SnippetsFinder.new(current_user, filter_params).execute), with: Entities::Snippet, current_user: current_user
+        present paginate(find_snippets(params: filter_params)), with: Entities::Snippet, current_user: current_user
       end
 
       desc 'List all public personal snippets current_user has access to' do
@@ -64,7 +65,7 @@ module API
 
         filter_params = declared_params(include_missing: false).merge(only_personal: true)
 
-        present paginate(SnippetsFinder.new(nil, filter_params).execute), with: Entities::PersonalSnippet, current_user: current_user
+        present paginate(find_snippets(user: nil, params: filter_params)), with: Entities::PersonalSnippet, current_user: current_user
       end
 
       desc 'List all snippets current_user has access to' do
@@ -81,13 +82,14 @@ module API
         optional :created_before, type: DateTime, desc: 'Return snippets created before the specified time'
 
         use :pagination
+        use :optional_list_params_ee
       end
       get 'all' do
         authenticate!
 
         filter_params = declared_params(include_missing: false).merge(all_available: true)
 
-        present paginate(SnippetsFinder.new(current_user, filter_params).execute), with: Entities::Snippet, current_user: current_user
+        present paginate(find_snippets(params: filter_params)), with: Entities::Snippet, current_user: current_user
       end
 
       desc 'Get a single snippet' do
@@ -102,7 +104,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of a snippet'
       end
       get ':id' do
-        snippet = snippets.find_by_id(params[:id])
+        snippet = find_snippets.find_by_id(params[:id])
 
         break not_found!('Snippet') unless snippet
 
@@ -126,6 +128,7 @@ module API
                               values: Gitlab::VisibilityLevel.string_values,
                               default: 'internal',
                               desc: 'The visibility of the snippet'
+
         use :create_file_params
       end
       post do
@@ -156,7 +159,6 @@ module API
         ]
         tags %w[snippets]
       end
-
       params do
         requires :id, type: Integer, desc: 'The ID of a snippet'
         optional :content, type: String, allow_blank: false, desc: 'The content of a snippet'
@@ -235,7 +237,7 @@ module API
         requires :id, type: Integer, desc: 'The ID of a snippet'
       end
       get ":id/raw" do
-        snippet = snippets.find_by_id(params.delete(:id))
+        snippet = find_snippets.find_by_id(params.delete(:id))
         not_found!('Snippet') unless snippet
 
         present content_for(snippet)
@@ -251,7 +253,7 @@ module API
         use :raw_file_params
       end
       get ":id/files/:ref/:file_path/raw", requirements: { file_path: API::NO_SLASH_URL_PART_REGEX } do
-        snippet = snippets.find_by_id(params.delete(:id))
+        snippet = find_snippets.find_by_id(params.delete(:id))
         not_found!('Snippet') unless snippet&.repo_exists?
 
         present file_content_for(snippet)
@@ -279,3 +281,5 @@ module API
     end
   end
 end
+
+API::Snippets.prepend_mod_with('API::Snippets')
diff --git a/lib/bulk_imports/visibility_level.rb b/lib/bulk_imports/visibility_level.rb
index 6b0af15dd7b..13bf25ff662 100644
--- a/lib/bulk_imports/visibility_level.rb
+++ b/lib/bulk_imports/visibility_level.rb
@@ -4,23 +4,24 @@ module BulkImports
   module VisibilityLevel
     private
 
+    # Calculates visbility level based on the source and the destination namespace visbility levels
+    # If there are visibility_level restrictions on the destination instance,
+    # the highest allowed level less than the calculated level is returned
     def visibility_level(entity, namespace, visibility_string)
       requested = requested_visibility_level(entity, visibility_string)
-      max_allowed = max_allowed_visibility_level(namespace)
+      namespace_level = namespace&.visibility_level
 
-      return requested if max_allowed >= requested
+      lowest_level = [requested, namespace_level].compact.min
 
-      max_allowed
+      closet_allowed_level(lowest_level)
     end
 
     def requested_visibility_level(entity, visibility_string)
       Gitlab::VisibilityLevel.string_options[visibility_string] || entity.default_visibility_level
     end
 
-    def max_allowed_visibility_level(namespace)
-      return Gitlab::VisibilityLevel.allowed_levels.max if namespace.blank?
-
-      Gitlab::VisibilityLevel.closest_allowed_level(namespace.visibility_level)
+    def closet_allowed_level(level)
+      Gitlab::VisibilityLevel.closest_allowed_level(level)
     end
   end
 end
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index b8e012ec851..c57391d355c 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -14,7 +14,9 @@ module Gitlab
           include ::Gitlab::Utils::StrongMemoize
 
           attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config
-          attr_reader :expandset, :execution_deadline, :logger, :max_includes
+          attr_reader :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes
+
+          attr_accessor :total_file_size_in_bytes
 
           delegate :instrument, to: :logger
 
@@ -32,6 +34,9 @@ module Gitlab
             @execution_deadline = 0
             @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project)
             @max_includes = Gitlab::CurrentSettings.current_application_settings.ci_max_includes
+            @max_total_yaml_size_bytes =
+              Gitlab::CurrentSettings.current_application_settings.ci_max_total_yaml_size_bytes
+            @total_file_size_in_bytes = 0
             yield self if block_given?
           end
 
@@ -59,6 +64,7 @@ module Gitlab
               ctx.execution_deadline = execution_deadline
               ctx.logger = logger
               ctx.max_includes = max_includes
+              ctx.max_total_yaml_size_bytes = max_total_yaml_size_bytes
             end
           end
 
@@ -100,7 +106,7 @@ module Gitlab
 
           protected
 
-          attr_writer :expandset, :execution_deadline, :logger, :max_includes
+          attr_writer :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes
 
           private
 
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 61b4d1ada10..cff7954235f 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -10,6 +10,7 @@ module Gitlab
           Error = Class.new(StandardError)
           AmbigiousSpecificationError = Class.new(Error)
           TooManyIncludesError = Class.new(Error)
+          TooMuchDataInPipelineTreeError = Class.new(Error)
 
           def initialize(values, context)
             @locations = Array.wrap(values.fetch(:include, [])).compact
diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb
index 95975e4661b..580cae8a207 100644
--- a/lib/gitlab/ci/config/external/mapper/verifier.rb
+++ b/lib/gitlab/ci/config/external/mapper/verifier.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'objspace'
+
 module Gitlab
   module Ci
     class Config
@@ -37,6 +39,13 @@ module Gitlab
 
                 file.validate_content! if file.valid?
                 file.load_and_validate_expanded_hash! if file.valid?
+
+                next unless Feature.enabled?(:introduce_ci_max_total_yaml_size_bytes, context.project) && file.valid?
+
+                # We are checking the file.content.to_s because that is returning the actual content of the file,
+                # whereas file.content would return the BatchLoader.
+                context.total_file_size_in_bytes += ObjectSpace.memsize_of(file.content.to_s)
+                verify_max_total_pipeline_size!
               end
             end
             # rubocop: enable Metrics/CyclomaticComplexity
@@ -50,6 +59,12 @@ module Gitlab
             def verify_execution_time!
               context.check_execution_time!
             end
+
+            def verify_max_total_pipeline_size!
+              return if context.total_file_size_in_bytes <= context.max_total_yaml_size_bytes
+
+              raise Mapper::TooMuchDataInPipelineTreeError, "Total size of combined CI/CD configuration is too big"
+            end
           end
         end
       end