Add latest changes from gitlab-org/gitlab@master

13 files changed, 78 insertions, 32 deletions

diff --git a/spec/lib/gitlab/container_repository/tags/cache_spec.rb b/spec/lib/gitlab/container_repository/tags/cache_spec.rb
index fcfc8e7a348..4b8c843eb3a 100644
--- a/spec/lib/gitlab/container_repository/tags/cache_spec.rb
+++ b/spec/lib/gitlab/container_repository/tags/cache_spec.rb
@@ -81,7 +81,9 @@ RSpec.describe ::Gitlab::ContainerRepository::Tags::Cache, :clean_gitlab_redis_c
         ::Gitlab::Redis::Cache.with do |redis|
           expect(redis).to receive(:pipelined).and_call_original
 
-          expect_next_instance_of(Redis::PipelinedConnection) do |pipeline|
+          times = Gitlab::Redis::ClusterUtil.cluster?(redis) ? 2 : 1
+
+          expect_next_instances_of(Redis::PipelinedConnection, times) do |pipeline|
             expect(pipeline)
               .to receive(:set)
                     .with(cache_key(tag), rfc3339(tag.created_at), ex: ttl.to_i)
diff --git a/spec/lib/gitlab/diff/highlight_cache_spec.rb b/spec/lib/gitlab/diff/highlight_cache_spec.rb
index 43e4f28b4df..c51eaa4fa18 100644
--- a/spec/lib/gitlab/diff/highlight_cache_spec.rb
+++ b/spec/lib/gitlab/diff/highlight_cache_spec.rb
@@ -217,7 +217,7 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache, feature_
 
   describe '#clear' do
     it 'clears cache' do
-      expect_any_instance_of(Redis).to receive(:del).with(cache_key)
+      Gitlab::Redis::Cache.with { |r| expect(r).to receive(:del).with(cache_key) }
 
       cache.clear
     end
@@ -241,7 +241,8 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache, feature_
     end
 
     it "uses ActiveSupport::Gzip to compress data when writing to cache" do
-      expect(ActiveSupport::Gzip).to receive(:compress).and_call_original
+      # at least once as Gitlab::Redis::Cache is a multistore
+      expect(ActiveSupport::Gzip).to receive(:compress).at_least(1).and_call_original
 
       cache.send(:write_to_redis_hash, diff_hash)
     end
diff --git a/spec/lib/gitlab/external_authorization/cache_spec.rb b/spec/lib/gitlab/external_authorization/cache_spec.rb
index a8e7932b82c..186bf7d7ec1 100644
--- a/spec/lib/gitlab/external_authorization/cache_spec.rb
+++ b/spec/lib/gitlab/external_authorization/cache_spec.rb
@@ -16,7 +16,7 @@ RSpec.describe Gitlab::ExternalAuthorization::Cache, :clean_gitlab_redis_cache d
 
   def set_in_redis(key, value)
     Gitlab::Redis::Cache.with do |redis|
-      redis.hmset(cache_key, key, value)
+      redis.hset(cache_key, key, value)
     end
   end
 
diff --git a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
index 0b8b1922d94..6b3d4485ea5 100644
--- a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
+++ b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter do
+RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter, feature_category: :importers do
   let(:repository) { double(:repository) }
   let(:import_state) { double(:import_state) }
   let(:client) { double(:client) }
@@ -23,6 +23,7 @@ RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter do
   let(:project) do
     double(
       :project,
+      id: 1,
       import_url: 'foo.git',
       import_source: 'foo/bar',
       repository_storage: 'foo',
@@ -204,6 +205,8 @@ RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter do
         .to receive(:fetch_as_mirror)
         .with(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: true)
 
+      expect(importer).to receive(:validate_repository_size!)
+
       service = double
       expect(Repositories::HousekeepingService)
         .to receive(:new).with(project, :gc).and_return(service)
diff --git a/spec/lib/gitlab/markdown_cache/redis/extension_spec.rb b/spec/lib/gitlab/markdown_cache/redis/extension_spec.rb
index 071f6e090c6..da5431a370b 100644
--- a/spec/lib/gitlab/markdown_cache/redis/extension_spec.rb
+++ b/spec/lib/gitlab/markdown_cache/redis/extension_spec.rb
@@ -65,7 +65,9 @@ RSpec.describe Gitlab::MarkdownCache::Redis::Extension, :clean_gitlab_redis_cach
       Gitlab::Redis::Cache.with do |redis|
         expect(redis).to receive(:pipelined).and_call_original
 
-        expect_next_instance_of(Redis::PipelinedConnection) do |pipeline|
+        times = Gitlab::Redis::ClusterUtil.cluster?(redis) ? 2 : 1
+
+        expect_next_instances_of(Redis::PipelinedConnection, times) do |pipeline|
           expect(pipeline).to receive(:mapped_hmget).once.and_call_original
         end
       end
diff --git a/spec/lib/gitlab/patch/redis_cache_store_spec.rb b/spec/lib/gitlab/patch/redis_cache_store_spec.rb
index 4095cbbfcde..5a674d443bb 100644
--- a/spec/lib/gitlab/patch/redis_cache_store_spec.rb
+++ b/spec/lib/gitlab/patch/redis_cache_store_spec.rb
@@ -36,7 +36,11 @@ RSpec.describe Gitlab::Patch::RedisCacheStore, :use_clean_rails_redis_caching, f
       context 'when reading large amount of keys' do
         it 'batches get into pipelines of 100' do
           cache.redis.with do |redis|
-            if Gitlab::Redis::ClusterUtil.cluster?(redis)
+            normal_cluster = !redis.is_a?(Gitlab::Redis::MultiStore) && Gitlab::Redis::ClusterUtil.cluster?(redis)
+            multistore_cluster = redis.is_a?(Gitlab::Redis::MultiStore) &&
+              ::Gitlab::Redis::ClusterUtil.cluster?(redis.default_store)
+
+            if normal_cluster || multistore_cluster
               expect(redis).to receive(:pipelined).at_least(2).and_call_original
             else
               expect(redis).to receive(:mget).and_call_original
@@ -53,6 +57,14 @@ RSpec.describe Gitlab::Patch::RedisCacheStore, :use_clean_rails_redis_caching, f
     context 'when cache is Rails.cache' do
       let(:cache) { Rails.cache }
 
+      context 'when reading using secondary store as default' do
+        before do
+          stub_feature_flags(use_primary_store_as_default_for_cache: false)
+        end
+
+        it_behaves_like 'reading using cache stores'
+      end
+
       it_behaves_like 'reading using cache stores'
     end
 
diff --git a/spec/lib/gitlab/redis/cluster_cache_spec.rb b/spec/lib/gitlab/redis/cluster_cache_spec.rb
new file mode 100644
index 00000000000..e448d608c53
--- /dev/null
+++ b/spec/lib/gitlab/redis/cluster_cache_spec.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Redis::ClusterCache, feature_category: :redis do
+  include_examples "redis_new_instance_shared_examples", 'cluster_cache', Gitlab::Redis::Cache
+end
diff --git a/spec/lib/gitlab/redis/multi_store_spec.rb b/spec/lib/gitlab/redis/multi_store_spec.rb
index 75e6e2db6e7..80d5915b819 100644
--- a/spec/lib/gitlab/redis/multi_store_spec.rb
+++ b/spec/lib/gitlab/redis/multi_store_spec.rb
@@ -138,6 +138,9 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do
     let_it_be(:hvalmapped) { { "item1" => value1 } }
     let_it_be(:sscanargs) { [skey2, 0] }
     let_it_be(:sscanval) { ["0", [value1]] }
+    let_it_be(:scanargs) { ["0"] }
+    let_it_be(:scankwargs) { { match: '*:set:key2*' } }
+    let_it_be(:scanval) { ["0", [skey2]] }
     let_it_be(:sscan_eachval) { [value1] }
     let_it_be(:sscan_each_arg) { { match: '*1*' } }
     let_it_be(:hscan_eachval) { [[hitem1, value1]] }
@@ -162,6 +165,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do
       'execute :hmget command'        | :hmget        | ref(:hgetargs)    | ref(:hmgetval)   | {} | nil
       'execute :mapped_hmget command' | :mapped_hmget | ref(:mhmgetargs)  | ref(:hvalmapped) | {} | nil
       'execute :sscan command'        | :sscan        | ref(:sscanargs)   | ref(:sscanval)   | {} | nil
+      'execute :scan command'         | :scan         | ref(:scanargs)    | ref(:scanval)    | ref(:scankwargs) | nil
 
       # we run *scan_each here as they are reads too
       'execute :scan_each command'    | :scan_each    | nil         | ref(:scan_each_val) | ref(:scan_each_arg)  | nil
@@ -489,6 +493,7 @@ RSpec.describe Gitlab::Redis::MultiStore, feature_category: :redis do
       'execute :setnx command'        | :setnx          | ref(:key1_value2)      | ref(:value1)   | :get      | ref(:key2)
       'execute :setex command'        | :setex          | ref(:key1_ttl_value1)  | ref(:ttl)      | :ttl      | ref(:key1)
       'execute :sadd command'         | :sadd           | ref(:skey_value2)      | ref(:svalues1) | :smembers | ref(:skey)
+      'execute :sadd? command'        | :sadd?          | ref(:skey_value2)      | ref(:svalues1) | :smembers | ref(:skey)
       'execute :srem command'         | :srem           | ref(:skey_value1)      | []             | :smembers | ref(:skey)
       'execute :del command'          | :del            | ref(:key2)             | nil            | :get      | ref(:key2)
       'execute :unlink command'       | :unlink         | ref(:key3)             | nil            | :get      | ref(:key3)
diff --git a/spec/services/projects/container_repository/third_party/cleanup_tags_service_spec.rb b/spec/services/projects/container_repository/third_party/cleanup_tags_service_spec.rb
index 836e722eb99..78343490e3a 100644
--- a/spec/services/projects/container_repository/third_party/cleanup_tags_service_spec.rb
+++ b/spec/services/projects/container_repository/third_party/cleanup_tags_service_spec.rb
@@ -325,9 +325,13 @@ RSpec.describe Projects::ContainerRepository::ThirdParty::CleanupTagsService, :c
         Gitlab::Redis::Cache.with do |redis|
           expect(redis).to receive(:pipelined).and_call_original
 
-          expect_next_instance_of(Redis::PipelinedConnection) do |pipeline|
+          times = Gitlab::Redis::ClusterUtil.cluster?(redis) ? 2 : 1
+
+          # Set 2 instances as redis is a MultiStore.
+          # Redis Cluster uses only 1 pipeline as the keys have hash-tags
+          expect_next_instances_of(Redis::PipelinedConnection, times) do |pipeline|
             selected_tags.each do |tag_name, created_at, ex|
-              expect(pipeline).to receive(:set).with(cache_key(tag_name), rfc3339(created_at), ex: ex)
+              expect(pipeline).to receive(:set).with(cache_key(tag_name), rfc3339(created_at), ex: ex).and_call_original
             end
           end
         end
@@ -372,7 +376,11 @@ RSpec.describe Projects::ContainerRepository::ThirdParty::CleanupTagsService, :c
       expect(redis).to receive(:mget).and_call_original
       expect(redis).to receive(:pipelined).and_call_original
 
-      expect_next_instance_of(Redis::PipelinedConnection) do |pipeline|
+      times = Gitlab::Redis::ClusterUtil.cluster?(redis) ? 2 : 1
+
+      # Set 2 instances as redis is a MultiStore
+      # Redis Cluster uses only 1 pipeline as the keys have hash-tags
+      expect_next_instances_of(Redis::PipelinedConnection, times) do |pipeline|
         expect(pipeline).to receive(:set).and_call_original
       end
     end
diff --git a/spec/support/caching.rb b/spec/support/caching.rb
index 119c521f732..46b6c7afa90 100644
--- a/spec/support/caching.rb
+++ b/spec/support/caching.rb
@@ -23,9 +23,7 @@ RSpec.configure do |config|
 
   config.around(:each, :use_clean_rails_redis_caching) do |example|
     original_null_store = Rails.cache
-    caching_config_hash = Gitlab::Redis::Cache.params
-    caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
-    Rails.cache = ActiveSupport::Cache::RedisCacheStore.new(**caching_config_hash)
+    Rails.cache = ActiveSupport::Cache::RedisCacheStore.new(**Gitlab::Redis::Cache.active_support_config)
 
     redis_cache_cleanup!
 
diff --git a/spec/support_specs/helpers/redis_commands/recorder_spec.rb b/spec/support_specs/helpers/redis_commands/recorder_spec.rb
index 6f93ed2fcf0..f41624d8dcc 100644
--- a/spec/support_specs/helpers/redis_commands/recorder_spec.rb
+++ b/spec/support_specs/helpers/redis_commands/recorder_spec.rb
@@ -8,6 +8,12 @@ RSpec.describe RedisCommands::Recorder, :use_clean_rails_redis_caching do
   let(:cache) { Rails.cache }
   let(:pattern) { nil }
 
+  before do
+    # do not need to test for positive case since this is testing
+    # a spec support class
+    stub_feature_flags(use_primary_and_secondary_stores_for_cache: false)
+  end
+
   describe '#initialize' do
     context 'with a block' do
       it 'records Redis commands' do
@@ -35,7 +41,7 @@ RSpec.describe RedisCommands::Recorder, :use_clean_rails_redis_caching do
         cache.delete('key1')
       end
 
-      expect(recorder.log).to include([:set, 'cache:gitlab:key1', anything])
+      expect(recorder.log).to include([:set, 'cache:gitlab:key1', anything, anything, anything])
       expect(recorder.log).to include([:get, 'cache:gitlab:key1'])
       expect(recorder.log).to include([:get, 'cache:gitlab:key2'])
       expect(recorder.log).to include([:del, 'cache:gitlab:key1'])
@@ -91,7 +97,7 @@ RSpec.describe RedisCommands::Recorder, :use_clean_rails_redis_caching do
         cache.delete('key2')
       end
 
-      expect(recorder.log).to include([:set, 'cache:gitlab:key1', anything])
+      expect(recorder.log).to include([:set, 'cache:gitlab:key1', anything, anything, anything])
       expect(recorder.log).to include([:get, 'cache:gitlab:key1'])
       expect(recorder.log).not_to include([:get, 'cache:gitlab:key2'])
       expect(recorder.log).not_to include([:del, 'cache:gitlab:key2'])
diff --git a/spec/support_specs/matchers/exceed_redis_call_limit_spec.rb b/spec/support_specs/matchers/exceed_redis_call_limit_spec.rb
index 819f50e26b6..e49a26c9b99 100644
--- a/spec/support_specs/matchers/exceed_redis_call_limit_spec.rb
+++ b/spec/support_specs/matchers/exceed_redis_call_limit_spec.rb
@@ -2,12 +2,14 @@
 
 require 'spec_helper'
 
-RSpec.describe 'RedisCommand matchers', :use_clean_rails_redis_caching, feature_category: :source_code_management do
+RSpec.describe 'RedisCommand matchers', :use_clean_rails_repository_cache_store_caching, feature_category: :source_code_management do
+  let_it_be(:cache) { Gitlab::Redis::RepositoryCache.cache_store }
+
   let(:control) do
     RedisCommands::Recorder.new do
-      Rails.cache.read('test')
-      Rails.cache.read('test')
-      Rails.cache.write('test', 1)
+      cache.read('test')
+      cache.read('test')
+      cache.write('test', 1)
     end
   end
 
@@ -31,13 +33,13 @@ RSpec.describe 'RedisCommand matchers', :use_clean_rails_redis_caching, feature_
 
   context 'with Recorder matching only some Redis calls' do
     it 'counts only Redis calls captured by Recorder' do
-      Rails.cache.write('ignored', 1)
+      cache.write('ignored', 1)
 
       control = RedisCommands::Recorder.new do
-        Rails.cache.read('recorded')
+        cache.read('recorded')
       end
 
-      Rails.cache.write('also_ignored', 1)
+      cache.write('also_ignored', 1)
 
       expect(control).not_to exceed_redis_calls_limit(1)
       expect(control).not_to exceed_redis_command_calls_limit(:set, 0)
@@ -48,8 +50,8 @@ RSpec.describe 'RedisCommand matchers', :use_clean_rails_redis_caching, feature_
   context 'when expect part is a function' do
     it 'automatically enables RedisCommand::Recorder for it' do
       func = -> do
-        Rails.cache.read('test')
-        Rails.cache.read('test')
+        cache.read('test')
+        cache.read('test')
       end
 
       expect { func.call }.not_to exceed_redis_calls_limit(2)
diff --git a/spec/tasks/cache_rake_spec.rb b/spec/tasks/cache_rake_spec.rb
index 046f8b107f2..7e4397ce3f4 100644
--- a/spec/tasks/cache_rake_spec.rb
+++ b/spec/tasks/cache_rake_spec.rb
@@ -10,8 +10,8 @@ RSpec.describe 'clearing redis cache', :clean_gitlab_redis_repository_cache, :cl
 
   let(:keys_size_changed) { -1 }
 
-  shared_examples 'clears the cache' do
-    it { expect { run_rake_task('cache:clear:redis') }.to change { redis_keys.size }.by(keys_size_changed) }
+  shared_examples 'clears the cache' do |redis|
+    it { expect { run_rake_task('cache:clear:redis') }.to change { redis_keys(redis).size }.by(keys_size_changed) }
   end
 
   describe 'clearing pipeline status cache' do
@@ -24,7 +24,7 @@ RSpec.describe 'clearing redis cache', :clean_gitlab_redis_repository_cache, :cl
       allow(pipeline_status).to receive(:loaded).and_return(nil)
     end
 
-    it_behaves_like 'clears the cache'
+    it_behaves_like 'clears the cache', Gitlab::Redis::Cache
   end
 
   describe 'clearing set caches' do
@@ -38,7 +38,7 @@ RSpec.describe 'clearing redis cache', :clean_gitlab_redis_repository_cache, :cl
         cache.write(:foo, [:bar])
       end
 
-      it_behaves_like 'clears the cache'
+      it_behaves_like 'clears the cache', Gitlab::Redis::RepositoryCache
     end
 
     context 'reactive cache set' do
@@ -48,15 +48,15 @@ RSpec.describe 'clearing redis cache', :clean_gitlab_redis_repository_cache, :cl
         cache.write(:foo, :bar)
       end
 
-      it_behaves_like 'clears the cache'
+      it_behaves_like 'clears the cache', Gitlab::Redis::Cache
     end
   end
 
-  def redis_keys
+  def redis_keys(redis_instance)
     # multiple scans to look across different shards if cache is using a Redis Cluster
-    cursor, scanned_keys = Gitlab::Redis::Cache.with { |redis| redis.scan(0, match: "*") }
+    cursor, scanned_keys = redis_instance.with { |redis| redis.scan(0, match: "*") }
     while cursor != "0"
-      cursor, keys = Gitlab::Redis::Cache.with { |redis| redis.scan(cursor, match: "*") }
+      cursor, keys = redis_instance.with { |redis| redis.scan(cursor, match: "*") }
       scanned_keys << keys
     end
     scanned_keys.flatten