diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-12 00:09:19 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-12 00:09:19 +0300 |
commit | fca89bb73ff5b1d14c98c72481f9268fee107ea0 (patch) | |
tree | e1c8a2c4fe5df7f054fd09e49f53bcfb51e51c84 /app/services | |
parent | 76e9fc7b29c1ce716c26932e9fbec0f3c99f53f4 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app/services')
-rw-r--r-- | app/services/projects/lfs_pointers/lfs_download_link_list_service.rb | 30 |
1 files changed, 29 insertions, 1 deletions
diff --git a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb index 5ef7e03ea02..48a21bf94ba 100644 --- a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb +++ b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb @@ -7,8 +7,13 @@ module Projects class LfsDownloadLinkListService < BaseService DOWNLOAD_ACTION = 'download' + # This could be different per server, but it seems like a reasonable value to start with. + # https://github.com/git-lfs/git-lfs/issues/419 + REQUEST_BATCH_SIZE = 100 + DownloadLinksError = Class.new(StandardError) DownloadLinkNotFound = Class.new(StandardError) + DownloadLinksRequestEntityTooLargeError = Class.new(StandardError) attr_reader :remote_uri @@ -25,16 +30,39 @@ module Projects def execute(oids) return [] unless project&.lfs_enabled? && remote_uri && oids.present? - get_download_links(oids) + get_download_links_in_batches(oids) end private + def get_download_links_in_batches(oids, batch_size = REQUEST_BATCH_SIZE) + download_links = [] + + oids.each_slice(batch_size) do |batch| + download_links += get_download_links(batch) + end + + download_links + + rescue DownloadLinksRequestEntityTooLargeError => e + # Log this exceptions to see how open it happens + Gitlab::ErrorTracking + .track_exception(e, project_id: project&.id, batch_size: batch_size, oids_count: oids.count) + + # Try again with a smaller batch + batch_size /= 2 + + retry if batch_size > REQUEST_BATCH_SIZE / 3 + + raise DownloadLinksError, 'Unable to download due to RequestEntityTooLarge errors' + end + def get_download_links(oids) response = Gitlab::HTTP.post(remote_uri, body: request_body(oids), headers: headers) + raise DownloadLinksRequestEntityTooLargeError if response.request_entity_too_large? raise DownloadLinksError, response.message unless response.success? # Since the LFS Batch API may return a Content-Ttpe of |