Paginate Bitbucket Server importer projects

To prevent delays in loading the page and reduce memory usage, limit the
number of projects shown at 25 per page.

Part of https://gitlab.com/gitlab-org/gitlab-ce/issues/50021

3 files changed, 49 insertions, 10 deletions

diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb
index 15e59f93141..83e8808db07 100644
--- a/lib/bitbucket_server/client.rb
+++ b/lib/bitbucket_server/client.rb
@@ -35,9 +35,9 @@ module BitbucketServer
       BitbucketServer::Representation::Repo.new(parsed_response)
     end
 
-    def repos
+    def repos(page_offset: 0, limit: nil)
       path = "/repos"
-      get_collection(path, :repo)
+      get_collection(path, :repo, page_offset: page_offset, limit: limit)
     end
 
     def create_branch(project_key, repo, branch_name, sha)
@@ -61,8 +61,8 @@ module BitbucketServer
 
     private
 
-    def get_collection(path, type)
-      paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type)
+    def get_collection(path, type, page_offset: 0, limit: nil)
+      paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type, page_offset: page_offset, limit: limit)
       BitbucketServer::Collection.new(paginator)
     rescue *SERVER_ERRORS => e
       raise ServerError, e
diff --git a/lib/bitbucket_server/collection.rb b/lib/bitbucket_server/collection.rb
index b50c5dde352..7e4b2277bbe 100644
--- a/lib/bitbucket_server/collection.rb
+++ b/lib/bitbucket_server/collection.rb
@@ -2,7 +2,13 @@
 
 module BitbucketServer
   class Collection < Enumerator
+    attr_reader :paginator
+
+    delegate :page_offset, :has_next_page?, to: :paginator
+
     def initialize(paginator)
+      @paginator = paginator
+
       super() do |yielder|
         loop do
           paginator.items.each { |item| yielder << item }
@@ -12,6 +18,24 @@ module BitbucketServer
       lazy
     end
 
+    def current_page
+      return 1 if page_offset <= 1
+
+      [1, page_offset].max
+    end
+
+    def prev_page
+      return nil unless current_page > 1
+
+      current_page - 1
+    end
+
+    def next_page
+      return nil unless has_next_page?
+
+      current_page + 1
+    end
+
     def method_missing(method, *args)
       return super unless self.respond_to?(method)
 
diff --git a/lib/bitbucket_server/paginator.rb b/lib/bitbucket_server/paginator.rb
index c351fb2f11f..aa5f84f44b3 100644
--- a/lib/bitbucket_server/paginator.rb
+++ b/lib/bitbucket_server/paginator.rb
@@ -4,34 +4,49 @@ module BitbucketServer
   class Paginator
     PAGE_LENGTH = 25
 
-    def initialize(connection, url, type)
+    attr_reader :page_offset
+
+    def initialize(connection, url, type, page_offset: 0, limit: nil)
       @connection = connection
       @type = type
       @url = url
       @page = nil
+      @page_offset = page_offset
+      @limit = limit || PAGE_LENGTH
+      @total = 0
     end
 
     def items
       raise StopIteration unless has_next_page?
+      raise StopIteration if over_limit?
 
       @page = fetch_next_page
+      @total += @page.items.count
       @page.items
     end
 
+    def has_next_page?
+      page.nil? || page.next?
+    end
+
     private
 
-    attr_reader :connection, :page, :url, :type
+    attr_reader :connection, :page, :url, :type, :limit
 
-    def has_next_page?
-      page.nil? || page.next?
+    def over_limit?
+      @limit.positive? && @total >= @limit
     end
 
     def next_offset
-      page.nil? ? 0 : page.next
+      page.nil? ? starting_offset : page.next
+    end
+
+    def starting_offset
+      [0, page_offset - 1].max * limit
     end
 
     def fetch_next_page
-      parsed_response = connection.get(@url, start: next_offset, limit: PAGE_LENGTH)
+      parsed_response = connection.get(@url, start: next_offset, limit: @limit)
       Page.new(parsed_response, type)
     end
   end