Merge branch '30093-apply-bfg-object-map-to-database' into 'master'

Remove cleaned up OIDs from database and cache

Closes #30093

See merge request gitlab-org/gitlab-ce!26555

3 files changed, 35 insertions, 15 deletions

diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb
index 270cfb89488..369c6b87fb4 100644
--- a/lib/gitlab/discussions_diff/highlight_cache.rb
+++ b/lib/gitlab/discussions_diff/highlight_cache.rb
@@ -52,6 +52,19 @@ module Gitlab
           end
         end
 
+        # Clears multiple cache keys at once.
+        #
+        # raw_keys - An Array of unique cache keys, without namespaces.
+        #
+        # It returns the number of cache keys cleared. Ex.: 42
+        def clear_multiple(raw_keys)
+          return [] if raw_keys.empty?
+
+          keys = raw_keys.map { |id| cache_key_for(id) }
+
+          Redis::Cache.with { |redis| redis.del(keys) }
+        end
+
         def cache_key_for(raw_key)
           "#{cache_key_prefix}:#{raw_key}"
         end
diff --git a/lib/gitlab/git/repository_cleaner.rb b/lib/gitlab/git/repository_cleaner.rb
index 2d1d8435cf3..9dd0ddfb44b 100644
--- a/lib/gitlab/git/repository_cleaner.rb
+++ b/lib/gitlab/git/repository_cleaner.rb
@@ -12,9 +12,9 @@ module Gitlab
         @repository = repository
       end
 
-      def apply_bfg_object_map(io)
+      def apply_bfg_object_map_stream(io, &blk)
         wrapped_gitaly_errors do
-          gitaly_cleanup_client.apply_bfg_object_map(io)
+          gitaly_cleanup_client.apply_bfg_object_map_stream(io, &blk)
         end
       end
 
diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb
index 3e8d6a773ca..a56bc35f6d7 100644
--- a/lib/gitlab/gitaly_client/cleanup_service.rb
+++ b/lib/gitlab/gitaly_client/cleanup_service.rb
@@ -12,25 +12,32 @@ module Gitlab
         @storage = repository.storage
       end
 
-      def apply_bfg_object_map(io)
-        first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo)
+      def apply_bfg_object_map_stream(io, &blk)
+        responses = GitalyClient.call(
+          storage,
+          :cleanup_service,
+          :apply_bfg_object_map_stream,
+          build_object_map_enum(io),
+          timeout: GitalyClient.no_timeout
+        )
+
+        responses.each(&blk)
+      end
+
+      private
 
-        enum = Enumerator.new do |y|
-          y.yield first_request
+      def build_object_map_enum(io)
+        Enumerator.new do |y|
+          # First request. For simplicity, doesn't include any object map data
+          y << Gitaly::ApplyBfgObjectMapStreamRequest.new(repository: gitaly_repo)
 
+          # Now stream the BFG object map file to gitaly in chunks
           while data = io.read(RepositoryService::MAX_MSG_SIZE)
-            y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data)
+            y << Gitaly::ApplyBfgObjectMapStreamRequest.new(object_map: data)
+
             break if io&.eof?
           end
         end
-
-        GitalyClient.call(
-          storage,
-          :cleanup_service,
-          :apply_bfg_object_map,
-          enum,
-          timeout: GitalyClient.no_timeout
-        )
       end
     end
   end