Add latest changes from gitlab-org/gitlab@master

1 files changed, 29 insertions, 1 deletions

diff --git a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb
index 5ef7e03ea02..48a21bf94ba 100644
--- a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb
@@ -7,8 +7,13 @@ module Projects
     class LfsDownloadLinkListService < BaseService
       DOWNLOAD_ACTION = 'download'
 
+      # This could be different per server, but it seems like a reasonable value to start with.
+      # https://github.com/git-lfs/git-lfs/issues/419
+      REQUEST_BATCH_SIZE = 100
+
       DownloadLinksError = Class.new(StandardError)
       DownloadLinkNotFound = Class.new(StandardError)
+      DownloadLinksRequestEntityTooLargeError = Class.new(StandardError)
 
       attr_reader :remote_uri
 
@@ -25,16 +30,39 @@ module Projects
       def execute(oids)
         return [] unless project&.lfs_enabled? && remote_uri && oids.present?
 
-        get_download_links(oids)
+        get_download_links_in_batches(oids)
       end
 
       private
 
+      def get_download_links_in_batches(oids, batch_size = REQUEST_BATCH_SIZE)
+        download_links = []
+
+        oids.each_slice(batch_size) do |batch|
+          download_links += get_download_links(batch)
+        end
+
+        download_links
+
+      rescue DownloadLinksRequestEntityTooLargeError => e
+        # Log this exceptions to see how open it happens
+        Gitlab::ErrorTracking
+          .track_exception(e, project_id: project&.id, batch_size: batch_size, oids_count: oids.count)
+
+        # Try again with a smaller batch
+        batch_size /= 2
+
+        retry if batch_size > REQUEST_BATCH_SIZE / 3
+
+        raise DownloadLinksError, 'Unable to download due to RequestEntityTooLarge errors'
+      end
+
       def get_download_links(oids)
         response = Gitlab::HTTP.post(remote_uri,
                                      body: request_body(oids),
                                      headers: headers)
 
+        raise DownloadLinksRequestEntityTooLargeError if response.request_entity_too_large?
         raise DownloadLinksError, response.message unless response.success?
 
         # Since the LFS Batch API may return a Content-Ttpe of