Add latest changes from gitlab-org/gitlab@master

16 files changed, 72 insertions, 52 deletions

diff --git a/lib/gitlab/application_rate_limiter/increment_per_action.rb b/lib/gitlab/application_rate_limiter/increment_per_action.rb
index c99d03f1344..a3343c8a97c 100644
--- a/lib/gitlab/application_rate_limiter/increment_per_action.rb
+++ b/lib/gitlab/application_rate_limiter/increment_per_action.rb
@@ -5,9 +5,9 @@ module Gitlab
     class IncrementPerAction < BaseStrategy
       def increment(cache_key, expiry)
         with_redis do |redis|
-          redis.pipelined do
-            redis.incr(cache_key)
-            redis.expire(cache_key, expiry)
+          redis.pipelined do |pipeline|
+            pipeline.incr(cache_key)
+            pipeline.expire(cache_key, expiry)
           end.first
         end
       end
diff --git a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb
index 8b4197cfff9..7a68dd104a8 100644
--- a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb
+++ b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb
@@ -9,10 +9,10 @@ module Gitlab
 
       def increment(cache_key, expiry)
         with_redis do |redis|
-          redis.pipelined do
-            redis.sadd(cache_key, resource_key)
-            redis.expire(cache_key, expiry)
-            redis.scard(cache_key)
+          redis.pipelined do |pipeline|
+            pipeline.sadd(cache_key, resource_key)
+            pipeline.expire(cache_key, expiry)
+            pipeline.scard(cache_key)
           end.last
         end
       end
diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb
new file mode 100644
index 00000000000..5b1d630bb03
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Removes obsolete wiki notes
+    class RemoveSelfManagedWikiNotes < BatchedMigrationJob
+      def perform
+        each_sub_batch(
+          operation_name: :delete_all
+        ) do |sub_batch|
+          sub_batch.where(noteable_type: 'Wiki').delete_all
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb
index 4dbce0b05e1..87988279c8d 100644
--- a/lib/gitlab/cache/import/caching.rb
+++ b/lib/gitlab/cache/import/caching.rb
@@ -153,11 +153,11 @@ module Gitlab
         # timeout - The time after which the cache key should expire.
         def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT)
           Redis::Cache.with do |redis|
-            redis.pipelined do |multi|
+            redis.pipelined do |pipeline|
               mapping.each do |raw_key, value|
                 key = cache_key_for("#{key_prefix}#{raw_key}")
 
-                multi.set(key, value, ex: timeout)
+                pipeline.set(key, value, ex: timeout)
               end
             end
           end
diff --git a/lib/gitlab/container_repository/tags/cache.rb b/lib/gitlab/container_repository/tags/cache.rb
index ff457fb9219..47a6e67a5a1 100644
--- a/lib/gitlab/container_repository/tags/cache.rb
+++ b/lib/gitlab/container_repository/tags/cache.rb
@@ -48,14 +48,14 @@ module Gitlab
           ::Gitlab::Redis::Cache.with do |redis|
             # we use a pipeline instead of a MSET because each tag has
             # a specific ttl
-            redis.pipelined do
+            redis.pipelined do |pipeline|
               cacheable_tags.each do |tag|
                 created_at = tag.created_at
                 # ttl is the max_ttl_in_seconds reduced by the number
                 # of seconds that the tag has already existed
                 ttl = max_ttl_in_seconds - (now - created_at).seconds
                 ttl = ttl.to_i
-                redis.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0
+                pipeline.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0
               end
             end
           end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index 44e3e9987f7..084ce63e36a 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -135,9 +135,9 @@ module Gitlab
       #
       def write_to_redis_hash(hash)
         Gitlab::Redis::Cache.with do |redis|
-          redis.pipelined do
+          redis.pipelined do |pipeline|
             hash.each do |diff_file_id, highlighted_diff_lines_hash|
-              redis.hset(
+              pipeline.hset(
                 key,
                 diff_file_id,
                 gzip_compress(highlighted_diff_lines_hash.to_json)
@@ -147,7 +147,7 @@ module Gitlab
             end
 
             # HSETs have to have their expiration date manually updated
-            redis.expire(key, expiration)
+            pipeline.expire(key, expiration)
           end
 
           record_memory_usage(fetch_memory_usage(redis, key))
@@ -202,9 +202,9 @@ module Gitlab
         expiration_period = expiration
 
         Gitlab::Redis::Cache.with do |redis|
-          redis.pipelined do
-            results = redis.hmget(cache_key, file_paths)
-            redis.expire(key, expiration_period) if highlight_diffs_renewable_expiration_enabled
+          redis.pipelined do |pipeline|
+            results = pipeline.hmget(cache_key, file_paths)
+            pipeline.expire(key, expiration_period) if highlight_diffs_renewable_expiration_enabled
           end
         end
 
diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb
index 44c6984c09b..437d577e70e 100644
--- a/lib/gitlab/etag_caching/store.rb
+++ b/lib/gitlab/etag_caching/store.rb
@@ -16,9 +16,9 @@ module Gitlab
         etags = keys.map { generate_etag }
 
         Gitlab::Redis::SharedState.with do |redis|
-          redis.pipelined do
+          redis.pipelined do |pipeline|
             keys.each_with_index do |key, i|
-              redis.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing)
+              pipeline.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing)
             end
           end
         end
diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb
index 509daeb0248..c06711d16f8 100644
--- a/lib/gitlab/external_authorization/cache.rb
+++ b/lib/gitlab/external_authorization/cache.rb
@@ -20,8 +20,8 @@ module Gitlab
 
       def store(new_access, new_reason, new_refreshed_at)
         ::Gitlab::Redis::Cache.with do |redis|
-          redis.pipelined do
-            redis.mapped_hmset(
+          redis.pipelined do |pipeline|
+            pipeline.mapped_hmset(
               cache_key,
               {
                 access: new_access.to_s,
@@ -30,7 +30,7 @@ module Gitlab
               }
             )
 
-            redis.expire(cache_key, VALIDITY_TIME)
+            pipeline.expire(cache_key, VALIDITY_TIME)
           end
         end
       end
diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb
index 80dff075391..6fe9bb10cdf 100644
--- a/lib/gitlab/manifest_import/metadata.rb
+++ b/lib/gitlab/manifest_import/metadata.rb
@@ -14,9 +14,9 @@ module Gitlab
 
       def save(repositories, group_id)
         Gitlab::Redis::SharedState.with do |redis|
-          redis.multi do
-            redis.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME)
-            redis.set(key_for('group_id'), group_id, ex: EXPIRY_TIME)
+          redis.multi do |multi|
+            multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME)
+            multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME)
           end
         end
       end
diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb
index 5a8efa34097..752ab153f37 100644
--- a/lib/gitlab/markdown_cache/redis/store.rb
+++ b/lib/gitlab/markdown_cache/redis/store.rb
@@ -10,9 +10,9 @@ module Gitlab
           results = {}
 
           Gitlab::Redis::Cache.with do |r|
-            r.pipelined do
+            r.pipelined do |pipeline|
               subjects.each do |subject|
-                results[subject.cache_key] = new(subject).read
+                results[subject.cache_key] = new(subject).read(pipeline)
               end
             end
           end
@@ -34,11 +34,15 @@ module Gitlab
           end
         end
 
-        def read
+        def read(pipeline = nil)
           @loaded = true
 
-          Gitlab::Redis::Cache.with do |r|
-            r.mapped_hmget(markdown_cache_key, *fields)
+          if pipeline
+            pipeline.mapped_hmget(markdown_cache_key, *fields)
+          else
+            Gitlab::Redis::Cache.with do |r|
+              r.mapped_hmget(markdown_cache_key, *fields)
+            end
           end
         end
 
diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb
index 7ccbeadfd8a..fbf96023f30 100644
--- a/lib/gitlab/reactive_cache_set_cache.rb
+++ b/lib/gitlab/reactive_cache_set_cache.rb
@@ -15,8 +15,8 @@ module Gitlab
         keys = read(key).map { |value| "#{cache_namespace}:#{value}" }
         keys << cache_key(key)
 
-        redis.pipelined do
-          keys.each_slice(1000) { |subset| redis.unlink(*subset) }
+        redis.pipelined do |pipeline|
+          keys.each_slice(1000) { |subset| pipeline.unlink(*subset) }
         end
       end
     end
diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb
index cdd2ac6100e..a7c36786d2d 100644
--- a/lib/gitlab/redis/multi_store.rb
+++ b/lib/gitlab/redis/multi_store.rb
@@ -267,7 +267,7 @@ module Gitlab
 
       def same_redis_store?
         strong_memoize(:same_redis_store) do
-          # <Redis client v4.4.0 for redis:///path_to/redis/redis.socket/5>"
+          # <Redis client v4.7.1 for unix:///path_to/redis/redis.socket/5>"
           primary_store.inspect == secondary_store.inspect
         end
       end
diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb
index 430f3e8d162..1ecdf506208 100644
--- a/lib/gitlab/repository_hash_cache.rb
+++ b/lib/gitlab/repository_hash_cache.rb
@@ -83,14 +83,14 @@ module Gitlab
       full_key = cache_key(key)
 
       with do |redis|
-        results = redis.pipelined do
+        results = redis.pipelined do |pipeline|
           # Set each hash key to the provided value
           hash.each do |h_key, h_value|
-            redis.hset(full_key, h_key, h_value)
+            pipeline.hset(full_key, h_key, h_value)
           end
 
           # Update the expiry time for this hset
-          redis.expire(full_key, expires_in)
+          pipeline.expire(full_key, expires_in)
         end
 
         results.all?
diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb
index 3061fb96190..33c7d96c45b 100644
--- a/lib/gitlab/repository_set_cache.rb
+++ b/lib/gitlab/repository_set_cache.rb
@@ -21,14 +21,14 @@ module Gitlab
       full_key = cache_key(key)
 
       with do |redis|
-        redis.multi do
-          redis.unlink(full_key)
+        redis.multi do |multi|
+          multi.unlink(full_key)
 
           # Splitting into groups of 1000 prevents us from creating a too-long
           # Redis command
-          value.each_slice(1000) { |subset| redis.sadd(full_key, subset) }
+          value.each_slice(1000) { |subset| multi.sadd(full_key, subset) }
 
-          redis.expire(full_key, expires_in)
+          multi.expire(full_key, expires_in)
         end
       end
 
@@ -39,9 +39,9 @@ module Gitlab
       full_key = cache_key(key)
 
       smembers, exists = with do |redis|
-        redis.multi do
-          redis.smembers(full_key)
-          redis.exists(full_key)
+        redis.multi do |multi|
+          multi.smembers(full_key)
+          multi.exists(full_key)
         end
       end
 
diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb
index feb2c3c1d7d..09c451f4dcc 100644
--- a/lib/gitlab/set_cache.rb
+++ b/lib/gitlab/set_cache.rb
@@ -33,10 +33,10 @@ module Gitlab
 
     def write(key, value)
       with do |redis|
-        redis.pipelined do
-          redis.sadd(cache_key(key), value)
+        redis.pipelined do |pipeline|
+          pipeline.sadd(cache_key(key), value)
 
-          redis.expire(cache_key(key), expires_in)
+          pipeline.expire(cache_key(key), expires_in)
         end
       end
 
@@ -57,9 +57,9 @@ module Gitlab
       full_key = cache_key(key)
 
       with do |redis|
-        redis.multi do
-          redis.sismember(full_key, value)
-          redis.exists(full_key)
+        redis.multi do |multi|
+          multi.sismember(full_key, value)
+          multi.exists(full_key)
         end
       end
     end
diff --git a/lib/gitlab/web_hooks/recursion_detection.rb b/lib/gitlab/web_hooks/recursion_detection.rb
index 1b5350d4a4e..031d9ec6ec4 100644
--- a/lib/gitlab/web_hooks/recursion_detection.rb
+++ b/lib/gitlab/web_hooks/recursion_detection.rb
@@ -40,9 +40,9 @@ module Gitlab
           cache_key = cache_key_for_hook(hook)
 
           ::Gitlab::Redis::SharedState.with do |redis|
-            redis.multi do
-              redis.sadd(cache_key, hook.id)
-              redis.expire(cache_key, TOUCH_CACHE_TTL)
+            redis.multi do |multi|
+              multi.sadd(cache_key, hook.id)
+              multi.expire(cache_key, TOUCH_CACHE_TTL)
             end
           end
         end