Add latest changes from gitlab-org/gitlab@master

8 files changed, 98 insertions, 41 deletions

diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb
index 140cbfac5c1..8d557463a9a 100644
--- a/lib/gitlab/ci/config/external/file/artifact.rb
+++ b/lib/gitlab/ci/config/external/file/artifact.rb
@@ -36,8 +36,6 @@ module Gitlab
               )
             end
 
-            private
-
             def validate_context!
               context.logger.instrument(:config_file_artifact_validate_context) do
                 if !creating_child_pipeline?
@@ -54,6 +52,8 @@ module Gitlab
               errors.push("File `#{masked_location}` is empty!") unless content.present?
             end
 
+            private
+
             def artifact_job
               strong_memoize(:artifact_job) do
                 next unless creating_child_pipeline?
diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb
index 7899fe0ff73..c80344bef8a 100644
--- a/lib/gitlab/ci/config/external/file/base.rb
+++ b/lib/gitlab/ci/config/external/file/base.rb
@@ -46,6 +46,7 @@ module Gitlab
               expanded_content_hash
             end
 
+            # Will be removed with the FF ci_batch_request_for_local_and_project_includes
             def validate!
               validate_location!
               validate_context! if valid?
@@ -68,22 +69,16 @@ module Gitlab
               [params, context.project&.full_path, context.sha].hash
             end
 
-            protected
-
-            def expanded_content_hash
-              return unless content_hash
-
-              strong_memoize(:expanded_content_yaml) do
-                expand_includes(content_hash)
+            def load_and_validate_expanded_hash!
+              context.logger.instrument(:config_file_fetch_content_hash) do
+                content_hash # calling the method loads then memoizes the result
               end
-            end
 
-            def content_hash
-              strong_memoize(:content_yaml) do
-                ::Gitlab::Ci::Config::Yaml.load!(content)
+              context.logger.instrument(:config_file_expand_content_includes) do
+                expanded_content_hash # calling the method expands then memoizes the result
               end
-            rescue Gitlab::Config::Loader::FormatError
-              nil
+
+              validate_hash!
             end
 
             def validate_location!
@@ -98,33 +93,40 @@ module Gitlab
               raise NotImplementedError, 'subclass must implement validate_context'
             end
 
-            def fetch_and_validate_content!
-              context.logger.instrument(:config_file_fetch_content) do
-                content # calling the method fetches then memoizes the result
+            def validate_content!
+              if content.blank?
+                errors.push("Included file `#{masked_location}` is empty or does not exist!")
               end
+            end
 
-              return if errors.any?
+            protected
 
-              context.logger.instrument(:config_file_validate_content) do
-                validate_content!
+            def expanded_content_hash
+              return unless content_hash
+
+              strong_memoize(:expanded_content_hash) do
+                expand_includes(content_hash)
               end
             end
 
-            def load_and_validate_expanded_hash!
-              context.logger.instrument(:config_file_fetch_content_hash) do
-                content_hash # calling the method loads then memoizes the result
+            def content_hash
+              strong_memoize(:content_hash) do
+                ::Gitlab::Ci::Config::Yaml.load!(content)
               end
+            rescue Gitlab::Config::Loader::FormatError
+              nil
+            end
 
-              context.logger.instrument(:config_file_expand_content_includes) do
-                expanded_content_hash # calling the method expands then memoizes the result
+            # Will be removed with the FF ci_batch_request_for_local_and_project_includes
+            def fetch_and_validate_content!
+              context.logger.instrument(:config_file_fetch_content) do
+                content # calling the method fetches then memoizes the result
               end
 
-              validate_hash!
-            end
+              return if errors.any?
 
-            def validate_content!
-              if content.blank?
-                errors.push("Included file `#{masked_location}` is empty or does not exist!")
+              context.logger.instrument(:config_file_validate_content) do
+                validate_content!
               end
             end
 
diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb
index 0912a732158..96f9cba419c 100644
--- a/lib/gitlab/ci/config/external/file/local.rb
+++ b/lib/gitlab/ci/config/external/file/local.rb
@@ -10,7 +10,12 @@ module Gitlab
             include Gitlab::Utils::StrongMemoize
 
             def initialize(params, context)
-              @location = params[:local]
+              @location = if ::Feature.enabled?(:ci_batch_request_for_local_and_project_includes, context.project)
+                            # `Repository#blobs_at` does not support files with the `/` prefix.
+                            Gitlab::Utils.remove_leading_slashes(params[:local])
+                          else
+                            params[:local]
+                          end
 
               super
             end
@@ -29,8 +34,6 @@ module Gitlab
               )
             end
 
-            private
-
             def validate_context!
               return if context.project&.repository
 
@@ -45,7 +48,27 @@ module Gitlab
               end
             end
 
+            private
+
             def fetch_local_content
+              if ::Feature.disabled?(:ci_batch_request_for_local_and_project_includes, context.project)
+                return legacy_fetch_local_content
+              end
+
+              BatchLoader.for([context.sha, location])
+                         .batch(key: context.project) do |locations, loader, args|
+                context.logger.instrument(:config_file_fetch_local_content) do
+                  args[:key].repository.blobs_at(locations).each do |blob|
+                    loader.call([blob.commit_id, blob.path], blob.data)
+                  end
+                end
+              rescue GRPC::InvalidArgument
+                # no-op
+              end
+            end
+
+            # Will be removed with the FF ci_batch_request_for_local_and_project_includes
+            def legacy_fetch_local_content
               context.logger.instrument(:config_file_fetch_local_content) do
                 context.project.repository.blob_data_at(context.sha, location)
               end
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index 553cbd819ad..ba1a56754eb 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -37,8 +37,6 @@ module Gitlab
               )
             end
 
-            private
-
             def validate_context!
               if !can_access_local_content?
                 errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.")
@@ -55,6 +53,8 @@ module Gitlab
               end
             end
 
+            private
+
             def project
               strong_memoize(:project) do
                 ::Project.find_by_full_path(project_name)
diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb
index ed37357dc53..bc8cebb8c3e 100644
--- a/lib/gitlab/ci/config/external/file/remote.rb
+++ b/lib/gitlab/ci/config/external/file/remote.rb
@@ -28,8 +28,6 @@ module Gitlab
               )
             end
 
-            private
-
             def validate_context!
               # no-op
             end
@@ -42,6 +40,8 @@ module Gitlab
               end
             end
 
+            private
+
             def fetch_remote_content
               begin
                 response = context.logger.instrument(:config_file_fetch_remote_content) do
diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb
index 53236cb317b..093b945da4b 100644
--- a/lib/gitlab/ci/config/external/file/template.rb
+++ b/lib/gitlab/ci/config/external/file/template.rb
@@ -31,8 +31,6 @@ module Gitlab
               )
             end
 
-            private
-
             def validate_context!
               # no-op
             end
@@ -45,6 +43,8 @@ module Gitlab
               end
             end
 
+            private
+
             def template_name
               return unless template_name_valid?
 
diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb
index 6d6f227b940..beb4dd1add7 100644
--- a/lib/gitlab/ci/config/external/mapper/verifier.rb
+++ b/lib/gitlab/ci/config/external/mapper/verifier.rb
@@ -10,6 +10,34 @@ module Gitlab
             private
 
             def process_without_instrumentation(files)
+              if ::Feature.disabled?(:ci_batch_request_for_local_and_project_includes, context.project)
+                return legacy_process_without_instrumentation(files)
+              end
+
+              files.each do |file|
+                verify_execution_time!
+
+                file.validate_location!
+                file.validate_context! if file.valid?
+                file.content if file.valid?
+              end
+
+              # We do not combine the loops because we need to load the content of all files before continuing
+              # to call `BatchLoader` for all locations.
+              files.each do |file| # rubocop:disable Style/CombinableLoops
+                # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150
+                verify_max_includes!
+                verify_execution_time!
+
+                file.validate_content! if file.valid?
+                file.load_and_validate_expanded_hash! if file.valid?
+
+                context.expandset.add(file)
+              end
+            end
+
+            # Will be removed with the FF ci_batch_request_for_local_and_project_includes
+            def legacy_process_without_instrumentation(files)
               files.select do |file|
                 verify_max_includes!
                 verify_execution_time!
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index d3055569ece..65572c237ec 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -82,7 +82,11 @@ module Gitlab
 
     # Append path to host, making sure there's one single / in between
     def append_path(host, path)
-      "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}"
+      "#{host.to_s.sub(%r{\/+$}, '')}/#{remove_leading_slashes(path)}"
+    end
+
+    def remove_leading_slashes(str)
+      str.to_s.sub(%r{^/+}, '')
     end
 
     # A slugified version of the string, suitable for inclusion in URLs and