Merge branch 'sh-optimize-groups-api' into 'master'

Optimize API /groups/:id/projects by preloading assocations

Closes #40308

See merge request gitlab-org/gitlab-ce!15475

7 files changed, 95 insertions, 12 deletions

diff --git a/app/services/base_count_service.rb b/app/services/base_count_service.rb
index 19873fe09a5..f2844854112 100644
--- a/app/services/base_count_service.rb
+++ b/app/services/base_count_service.rb
@@ -12,8 +12,12 @@ class BaseCountService
     Rails.cache.fetch(cache_key, cache_options) { uncached_count }.to_i
   end
 
-  def refresh_cache
-    Rails.cache.write(cache_key, uncached_count, raw: raw?)
+  def count_stored?
+    Rails.cache.read(cache_key).present?
+  end
+
+  def refresh_cache(&block)
+    Rails.cache.write(cache_key, block_given? ? yield : uncached_count, raw: raw?)
   end
 
   def uncached_count
diff --git a/app/services/projects/batch_count_service.rb b/app/services/projects/batch_count_service.rb
new file mode 100644
index 00000000000..178ebc5a143
--- /dev/null
+++ b/app/services/projects/batch_count_service.rb
@@ -0,0 +1,31 @@
+# Service class for getting and caching the number of elements of several projects
+# Warning: do not user this service with a really large set of projects
+# because the service use maps to retrieve the project ids.
+module Projects
+  class BatchCountService
+    def initialize(projects)
+      @projects = projects
+    end
+
+    def refresh_cache
+      @projects.each do |project|
+        service = count_service.new(project)
+        unless service.count_stored?
+          service.refresh_cache { global_count[project.id].to_i }
+        end
+      end
+    end
+
+    def project_ids
+      @projects.map(&:id)
+    end
+
+    def global_count(project)
+      raise NotImplementedError, 'global_count must be implemented and return an hash indexed by the project id'
+    end
+
+    def count_service
+      raise NotImplementedError, 'count_service must be implemented and return a Projects::CountService object'
+    end
+  end
+end
diff --git a/app/services/projects/batch_forks_count_service.rb b/app/services/projects/batch_forks_count_service.rb
new file mode 100644
index 00000000000..e61fe6c86b2
--- /dev/null
+++ b/app/services/projects/batch_forks_count_service.rb
@@ -0,0 +1,18 @@
+# Service class for getting and caching the number of forks of several projects
+# Warning: do not user this service with a really large set of projects
+# because the service use maps to retrieve the project ids
+module Projects
+  class BatchForksCountService < Projects::BatchCountService
+    def global_count
+      @global_count ||= begin
+        count_service.query(project_ids)
+                     .group(:forked_from_project_id)
+                     .count
+      end
+    end
+
+    def count_service
+      ::Projects::ForksCountService
+    end
+  end
+end
diff --git a/app/services/projects/batch_open_issues_count_service.rb b/app/services/projects/batch_open_issues_count_service.rb
new file mode 100644
index 00000000000..3b0ade2419b
--- /dev/null
+++ b/app/services/projects/batch_open_issues_count_service.rb
@@ -0,0 +1,16 @@
+# Service class for getting and caching the number of issues of several projects
+# Warning: do not user this service with a really large set of projects
+# because the service use maps to retrieve the project ids
+module Projects
+  class BatchOpenIssuesCountService < Projects::BatchCountService
+    def global_count
+      @global_count ||= begin
+        count_service.query(project_ids).group(:project_id).count
+      end
+    end
+
+    def count_service
+      ::Projects::OpenIssuesCountService
+    end
+  end
+end
diff --git a/app/services/projects/count_service.rb b/app/services/projects/count_service.rb
index 7e575b2d6f3..933829b557b 100644
--- a/app/services/projects/count_service.rb
+++ b/app/services/projects/count_service.rb
@@ -11,6 +11,10 @@ module Projects
       @project = project
     end
 
+    def relation_for_count
+      self.class.query(@project.id)
+    end
+
     def cache_key_name
       raise(
         NotImplementedError,
@@ -21,5 +25,12 @@ module Projects
     def cache_key
       ['projects', 'count_service', VERSION, @project.id, cache_key_name]
     end
+
+    def self.query(project_ids)
+      raise(
+        NotImplementedError,
+        '"query" must be implemented and return an ActiveRecord::Relation'
+      )
+    end
   end
 end
diff --git a/app/services/projects/forks_count_service.rb b/app/services/projects/forks_count_service.rb
index d9bdf3a8ad7..dc6eb19affd 100644
--- a/app/services/projects/forks_count_service.rb
+++ b/app/services/projects/forks_count_service.rb
@@ -1,12 +1,15 @@
 module Projects
   # Service class for getting and caching the number of forks of a project.
   class ForksCountService < Projects::CountService
-    def relation_for_count
-      @project.forks
-    end
-
     def cache_key_name
       'forks_count'
     end
+
+    def self.query(project_ids)
+      # We can't directly change ForkedProjectLink to ForkNetworkMember here
+      # Nowadays, when a call using v3 to projects/:id/fork is made,
+      # the relationship to ForkNetworkMember is not updated
+      ForkedProjectLink.where(forked_from_project: project_ids)
+    end
   end
 end
diff --git a/app/services/projects/open_issues_count_service.rb b/app/services/projects/open_issues_count_service.rb
index 25de97325e2..a975a06a05c 100644
--- a/app/services/projects/open_issues_count_service.rb
+++ b/app/services/projects/open_issues_count_service.rb
@@ -2,14 +2,14 @@ module Projects
   # Service class for counting and caching the number of open issues of a
   # project.
   class OpenIssuesCountService < Projects::CountService
-    def relation_for_count
-      # We don't include confidential issues in this number since this would
-      # expose the number of confidential issues to non project members.
-      @project.issues.opened.public_only
-    end
-
     def cache_key_name
       'open_issues_count'
     end
+
+    def self.query(project_ids)
+      # We don't include confidential issues in this number since this would
+      # expose the number of confidential issues to non project members.
+      Issue.opened.public_only.where(project: project_ids)
+    end
   end
 end