Cache branch and tag names as Redis sets

This allows us to check inclusion for the *_exists? methods without
downloading the full list of branch names, which is over 100KiB in size
for gitlab-ce at the moment.

3 files changed, 119 insertions, 13 deletions

diff --git a/spec/lib/gitlab/repository_cache_adapter_spec.rb b/spec/lib/gitlab/repository_cache_adapter_spec.rb
index 0295138fc3a..04fc24b6205 100644
--- a/spec/lib/gitlab/repository_cache_adapter_spec.rb
+++ b/spec/lib/gitlab/repository_cache_adapter_spec.rb
@@ -4,6 +4,7 @@ describe Gitlab::RepositoryCacheAdapter do
   let(:project) { create(:project, :repository) }
   let(:repository) { project.repository }
   let(:cache) { repository.send(:cache) }
+  let(:redis_set_cache) { repository.send(:redis_set_cache) }
 
   describe '#cache_method_output', :use_clean_rails_memory_store_caching do
     let(:fallback) { 10 }
@@ -206,9 +207,11 @@ describe Gitlab::RepositoryCacheAdapter do
   describe '#expire_method_caches' do
     it 'expires the caches of the given methods' do
       expect(cache).to receive(:expire).with(:rendered_readme)
-      expect(cache).to receive(:expire).with(:gitignore)
+      expect(cache).to receive(:expire).with(:branch_names)
+      expect(redis_set_cache).to receive(:expire).with(:rendered_readme)
+      expect(redis_set_cache).to receive(:expire).with(:branch_names)
 
-      repository.expire_method_caches(%i(rendered_readme gitignore))
+      repository.expire_method_caches(%i(rendered_readme branch_names))
     end
 
     it 'does not expire caches for non-existent methods' do
diff --git a/spec/lib/gitlab/repository_set_cache_spec.rb b/spec/lib/gitlab/repository_set_cache_spec.rb
new file mode 100644
index 00000000000..9695e13d842
--- /dev/null
+++ b/spec/lib/gitlab/repository_set_cache_spec.rb
@@ -0,0 +1,73 @@
+require 'spec_helper'
+
+describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do
+  let(:project) { create(:project) }
+  let(:repository) { project.repository }
+  let(:namespace) { "#{repository.full_path}:#{project.id}" }
+  let(:cache) { described_class.new(repository) }
+
+  describe '#cache_key' do
+    subject { cache.cache_key(:foo) }
+
+    it 'includes the namespace' do
+      is_expected.to eq("foo:#{namespace}:set")
+    end
+
+    context 'with a given namespace' do
+      let(:extra_namespace) { 'my:data' }
+      let(:cache) { described_class.new(repository, extra_namespace: extra_namespace) }
+
+      it 'includes the full namespace' do
+        is_expected.to eq("foo:#{namespace}:#{extra_namespace}:set")
+      end
+    end
+  end
+
+  describe '#expire' do
+    it 'expires the given key from the cache' do
+      cache.write(:foo, ['value'])
+
+      expect(cache.read(:foo)).to contain_exactly('value')
+      expect(cache.expire(:foo)).to eq(1)
+      expect(cache.read(:foo)).to be_empty
+    end
+  end
+
+  describe '#exist?' do
+    it 'checks whether the key exists' do
+      expect(cache.exist?(:foo)).to be(false)
+
+      cache.write(:foo, ['value'])
+
+      expect(cache.exist?(:foo)).to be(true)
+    end
+  end
+
+  describe '#fetch' do
+    let(:blk) { -> { ['block value'] } }
+
+    subject { cache.fetch(:foo, &blk) }
+
+    it 'fetches the key from the cache when filled' do
+      cache.write(:foo, ['value'])
+
+      is_expected.to contain_exactly('value')
+    end
+
+    it 'writes the value of the provided block when empty' do
+      cache.expire(:foo)
+
+      is_expected.to contain_exactly('block value')
+      expect(cache.read(:foo)).to contain_exactly('block value')
+    end
+  end
+
+  describe '#include?' do
+    it 'checks inclusion in the Redis set' do
+      cache.write(:foo, ['value'])
+
+      expect(cache.include?(:foo, 'value')).to be(true)
+      expect(cache.include?(:foo, 'bar')).to be(false)
+    end
+  end
+end
diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb
index e68de2e73a8..423334c66f2 100644
--- a/spec/models/repository_spec.rb
+++ b/spec/models/repository_spec.rb
@@ -1223,36 +1223,66 @@ describe Repository do
   end
 
   describe '#branch_exists?' do
-    it 'uses branch_names' do
-      allow(repository).to receive(:branch_names).and_return(['foobar'])
+    let(:branch) { repository.root_ref }
 
-      expect(repository.branch_exists?('foobar')).to eq(true)
-      expect(repository.branch_exists?('master')).to eq(false)
+    subject { repository.branch_exists?(branch) }
+
+    it 'delegates to branch_names when the cache is empty' do
+      repository.expire_branches_cache
+
+      expect(repository).to receive(:branch_names).and_call_original
+      is_expected.to eq(true)
+    end
+
+    it 'uses redis set caching when the cache is filled' do
+      repository.branch_names # ensure the branch name cache is filled
+
+      expect(repository)
+        .to receive(:branch_names_include?)
+        .with(branch)
+        .and_call_original
+
+      is_expected.to eq(true)
     end
   end
 
   describe '#tag_exists?' do
-    it 'uses tag_names' do
-      allow(repository).to receive(:tag_names).and_return(['foobar'])
+    let(:tag) { repository.tags.first.name }
+
+    subject { repository.tag_exists?(tag) }
+
+    it 'delegates to tag_names when the cache is empty' do
+      repository.expire_tags_cache
+
+      expect(repository).to receive(:tag_names).and_call_original
+      is_expected.to eq(true)
+    end
+
+    it 'uses redis set caching when the cache is filled' do
+      repository.tag_names # ensure the tag name cache is filled
+
+      expect(repository)
+        .to receive(:tag_names_include?)
+        .with(tag)
+        .and_call_original
 
-      expect(repository.tag_exists?('foobar')).to eq(true)
-      expect(repository.tag_exists?('master')).to eq(false)
+      is_expected.to eq(true)
     end
   end
 
-  describe '#branch_names', :use_clean_rails_memory_store_caching do
+  describe '#branch_names', :clean_gitlab_redis_cache do
     let(:fake_branch_names) { ['foobar'] }
 
     it 'gets cached across Repository instances' do
       allow(repository.raw_repository).to receive(:branch_names).once.and_return(fake_branch_names)
 
-      expect(repository.branch_names).to eq(fake_branch_names)
+      expect(repository.branch_names).to match_array(fake_branch_names)
 
       fresh_repository = Project.find(project.id).repository
       expect(fresh_repository.object_id).not_to eq(repository.object_id)
 
       expect(fresh_repository.raw_repository).not_to receive(:branch_names)
-      expect(fresh_repository.branch_names).to eq(fake_branch_names)
+      expect(fresh_repository.branch_names).to match_array(fake_branch_names)
     end
   end