From 0eff75fa2b6691b6fba31fcc2842f51debd249a9 Mon Sep 17 00:00:00 2001
From: Nick Thomas <nick@gitlab.com>
Date: Mon, 8 Jul 2019 17:11:59 +0100
Subject: Cache branch and tag names as Redis sets

This allows us to check inclusion for the *_exists? methods without
downloading the full list of branch names, which is over 100KiB in size
for gitlab-ce at the moment.
---
 lib/gitlab/repository_cache_adapter.rb | 53 +++++++++++++++++++++++++++
 lib/gitlab/repository_set_cache.rb     | 67 ++++++++++++++++++++++++++++++++++
 2 files changed, 120 insertions(+)
 create mode 100644 lib/gitlab/repository_set_cache.rb

(limited to 'lib')

diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb
index e40c366ed02..75503ee1789 100644
--- a/lib/gitlab/repository_cache_adapter.rb
+++ b/lib/gitlab/repository_cache_adapter.rb
@@ -23,6 +23,37 @@ module Gitlab
         end
       end
 
+      # Caches and strongly memoizes the method as a Redis Set.
+      #
+      # This only works for methods that do not take any arguments. The method
+      # should return an Array of Strings to be cached.
+      #
+      # In addition to overriding the named method, a "name_include?" method is
+      # defined. This uses the "SISMEMBER" query to efficiently check membership
+      # without needing to load the entire set into memory.
+      #
+      # name     - The name of the method to be cached.
+      # fallback - A value to fall back to if the repository does not exist, or
+      #            in case of a Git error. Defaults to nil.
+      def cache_method_as_redis_set(name, fallback: nil)
+        uncached_name = alias_uncached_method(name)
+
+        define_method(name) do
+          cache_method_output_as_redis_set(name, fallback: fallback) do
+            __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        define_method("#{name}_include?") do |value|
+          # If the cache isn't populated, we can't rely on it
+          return redis_set_cache.include?(name, value) if redis_set_cache.exist?(name)
+
+          # Since we have to pull all branch names to populate the cache, use
+          # the data we already have to answer the query just this once
+          __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+
       # Caches truthy values from the method. All values are strongly memoized,
       # and cached in RequestStore.
       #
@@ -84,6 +115,11 @@ module Gitlab
       raise NotImplementedError
     end
 
+    # RepositorySetCache to be used. Should be overridden by the including class
+    def redis_set_cache
+      raise NotImplementedError
+    end
+
     # List of cached methods. Should be overridden by the including class
     def cached_methods
       raise NotImplementedError
@@ -100,6 +136,18 @@ module Gitlab
       end
     end
 
+    # Caches and strongly memoizes the supplied block as a Redis Set. The result
+    # will be provided as a sorted array.
+    #
+    # name     - The name of the method to be cached.
+    # fallback - A value to fall back to if the repository does not exist, or
+    #            in case of a Git error. Defaults to nil.
+    def cache_method_output_as_redis_set(name, fallback: nil, &block)
+      memoize_method_output(name, fallback: fallback) do
+        redis_set_cache.fetch(name, &block).sort
+      end
+    end
+
     # Caches truthy values from the supplied block. All values are strongly
     # memoized, and cached in RequestStore.
     #
@@ -154,6 +202,7 @@ module Gitlab
         clear_memoization(memoizable_name(name))
       end
 
+      expire_redis_set_method_caches(methods)
       expire_request_store_method_caches(methods)
     end
 
@@ -169,6 +218,10 @@ module Gitlab
       end
     end
 
+    def expire_redis_set_method_caches(methods)
+      methods.each { |name| redis_set_cache.expire(name) }
+    end
+
     # All cached repository methods depend on the existence of a Git repository,
     # so if the repository doesn't exist, we already know not to call it.
     def fallback_early?(method_name)
diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb
new file mode 100644
index 00000000000..fb634328a95
--- /dev/null
+++ b/lib/gitlab/repository_set_cache.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# Interface to the Redis-backed cache store for keys that use a Redis set
+module Gitlab
+  class RepositorySetCache
+    attr_reader :repository, :namespace, :expires_in
+
+    def initialize(repository, extra_namespace: nil, expires_in: 2.weeks)
+      @repository = repository
+      @namespace = "#{repository.full_path}:#{repository.project.id}"
+      @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace
+      @expires_in = expires_in
+    end
+
+    def cache_key(type)
+      [type, namespace, 'set'].join(':')
+    end
+
+    def expire(key)
+      with { |redis| redis.del(cache_key(key)) }
+    end
+
+    def exist?(key)
+      with { |redis| redis.exists(cache_key(key)) }
+    end
+
+    def read(key)
+      with { |redis| redis.smembers(cache_key(key)) }
+    end
+
+    def write(key, value)
+      full_key = cache_key(key)
+
+      with do |redis|
+        redis.multi do
+          redis.del(full_key)
+
+          # Splitting into groups of 1000 prevents us from creating a too-long
+          # Redis command
+          value.in_groups_of(1000, false) { |subset| redis.sadd(full_key, subset) }
+
+          redis.expire(full_key, expires_in)
+        end
+      end
+
+      value
+    end
+
+    def fetch(key, &block)
+      if exist?(key)
+        read(key)
+      else
+        write(key, yield)
+      end
+    end
+
+    def include?(key, value)
+      with { |redis| redis.sismember(cache_key(key), value) }
+    end
+
+    private
+
+    def with(&blk)
+      Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
+    end
+  end
+end
-- 
cgit v1.2.3