From 0eff75fa2b6691b6fba31fcc2842f51debd249a9 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Mon, 8 Jul 2019 17:11:59 +0100 Subject: Cache branch and tag names as Redis sets This allows us to check inclusion for the *_exists? methods without downloading the full list of branch names, which is over 100KiB in size for gitlab-ce at the moment. --- spec/lib/gitlab/repository_cache_adapter_spec.rb | 7 ++- spec/lib/gitlab/repository_set_cache_spec.rb | 73 ++++++++++++++++++++++++ spec/models/repository_spec.rb | 52 +++++++++++++---- 3 files changed, 119 insertions(+), 13 deletions(-) create mode 100644 spec/lib/gitlab/repository_set_cache_spec.rb (limited to 'spec') diff --git a/spec/lib/gitlab/repository_cache_adapter_spec.rb b/spec/lib/gitlab/repository_cache_adapter_spec.rb index 0295138fc3a..04fc24b6205 100644 --- a/spec/lib/gitlab/repository_cache_adapter_spec.rb +++ b/spec/lib/gitlab/repository_cache_adapter_spec.rb @@ -4,6 +4,7 @@ describe Gitlab::RepositoryCacheAdapter do let(:project) { create(:project, :repository) } let(:repository) { project.repository } let(:cache) { repository.send(:cache) } + let(:redis_set_cache) { repository.send(:redis_set_cache) } describe '#cache_method_output', :use_clean_rails_memory_store_caching do let(:fallback) { 10 } @@ -206,9 +207,11 @@ describe Gitlab::RepositoryCacheAdapter do describe '#expire_method_caches' do it 'expires the caches of the given methods' do expect(cache).to receive(:expire).with(:rendered_readme) - expect(cache).to receive(:expire).with(:gitignore) + expect(cache).to receive(:expire).with(:branch_names) + expect(redis_set_cache).to receive(:expire).with(:rendered_readme) + expect(redis_set_cache).to receive(:expire).with(:branch_names) - repository.expire_method_caches(%i(rendered_readme gitignore)) + repository.expire_method_caches(%i(rendered_readme branch_names)) end it 'does not expire caches for non-existent methods' do diff --git a/spec/lib/gitlab/repository_set_cache_spec.rb b/spec/lib/gitlab/repository_set_cache_spec.rb new file mode 100644 index 00000000000..9695e13d842 --- /dev/null +++ b/spec/lib/gitlab/repository_set_cache_spec.rb @@ -0,0 +1,73 @@ +require 'spec_helper' + +describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do + let(:project) { create(:project) } + let(:repository) { project.repository } + let(:namespace) { "#{repository.full_path}:#{project.id}" } + let(:cache) { described_class.new(repository) } + + describe '#cache_key' do + subject { cache.cache_key(:foo) } + + it 'includes the namespace' do + is_expected.to eq("foo:#{namespace}:set") + end + + context 'with a given namespace' do + let(:extra_namespace) { 'my:data' } + let(:cache) { described_class.new(repository, extra_namespace: extra_namespace) } + + it 'includes the full namespace' do + is_expected.to eq("foo:#{namespace}:#{extra_namespace}:set") + end + end + end + + describe '#expire' do + it 'expires the given key from the cache' do + cache.write(:foo, ['value']) + + expect(cache.read(:foo)).to contain_exactly('value') + expect(cache.expire(:foo)).to eq(1) + expect(cache.read(:foo)).to be_empty + end + end + + describe '#exist?' do + it 'checks whether the key exists' do + expect(cache.exist?(:foo)).to be(false) + + cache.write(:foo, ['value']) + + expect(cache.exist?(:foo)).to be(true) + end + end + + describe '#fetch' do + let(:blk) { -> { ['block value'] } } + + subject { cache.fetch(:foo, &blk) } + + it 'fetches the key from the cache when filled' do + cache.write(:foo, ['value']) + + is_expected.to contain_exactly('value') + end + + it 'writes the value of the provided block when empty' do + cache.expire(:foo) + + is_expected.to contain_exactly('block value') + expect(cache.read(:foo)).to contain_exactly('block value') + end + end + + describe '#include?' do + it 'checks inclusion in the Redis set' do + cache.write(:foo, ['value']) + + expect(cache.include?(:foo, 'value')).to be(true) + expect(cache.include?(:foo, 'bar')).to be(false) + end + end +end diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index e68de2e73a8..423334c66f2 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -1223,36 +1223,66 @@ describe Repository do end describe '#branch_exists?' do - it 'uses branch_names' do - allow(repository).to receive(:branch_names).and_return(['foobar']) + let(:branch) { repository.root_ref } - expect(repository.branch_exists?('foobar')).to eq(true) - expect(repository.branch_exists?('master')).to eq(false) + subject { repository.branch_exists?(branch) } + + it 'delegates to branch_names when the cache is empty' do + repository.expire_branches_cache + + expect(repository).to receive(:branch_names).and_call_original + is_expected.to eq(true) + end + + it 'uses redis set caching when the cache is filled' do + repository.branch_names # ensure the branch name cache is filled + + expect(repository) + .to receive(:branch_names_include?) + .with(branch) + .and_call_original + + is_expected.to eq(true) end end describe '#tag_exists?' do - it 'uses tag_names' do - allow(repository).to receive(:tag_names).and_return(['foobar']) + let(:tag) { repository.tags.first.name } + + subject { repository.tag_exists?(tag) } + + it 'delegates to tag_names when the cache is empty' do + repository.expire_tags_cache + + expect(repository).to receive(:tag_names).and_call_original + is_expected.to eq(true) + end + + it 'uses redis set caching when the cache is filled' do + repository.tag_names # ensure the tag name cache is filled + + expect(repository) + .to receive(:tag_names_include?) + .with(tag) + .and_call_original - expect(repository.tag_exists?('foobar')).to eq(true) - expect(repository.tag_exists?('master')).to eq(false) + is_expected.to eq(true) end end - describe '#branch_names', :use_clean_rails_memory_store_caching do + describe '#branch_names', :clean_gitlab_redis_cache do let(:fake_branch_names) { ['foobar'] } it 'gets cached across Repository instances' do allow(repository.raw_repository).to receive(:branch_names).once.and_return(fake_branch_names) - expect(repository.branch_names).to eq(fake_branch_names) + expect(repository.branch_names).to match_array(fake_branch_names) fresh_repository = Project.find(project.id).repository expect(fresh_repository.object_id).not_to eq(repository.object_id) expect(fresh_repository.raw_repository).not_to receive(:branch_names) - expect(fresh_repository.branch_names).to eq(fake_branch_names) + expect(fresh_repository.branch_names).to match_array(fake_branch_names) end end -- cgit v1.2.3