Merge branch '46535-orphaned-uploads' into 'master'

Rake task for checking orphaned object store uploads

Closes #46535

See merge request gitlab-org/gitlab-ce!20918

2 files changed, 90 insertions, 0 deletions

diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb
new file mode 100644
index 00000000000..45a5aea4fcd
--- /dev/null
+++ b/lib/gitlab/cleanup/remote_uploads.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+module Gitlab
+  module Cleanup
+    class RemoteUploads
+      attr_reader :logger
+
+      BATCH_SIZE = 100
+
+      def initialize(logger: nil)
+        @logger = logger || Rails.logger
+      end
+
+      def run!(dry_run: false)
+        unless configuration.enabled
+          logger.warn "Object storage not enabled. Exit".color(:yellow)
+
+          return
+        end
+
+        logger.info "Looking for orphaned remote uploads to remove#{'. Dry run' if dry_run}..."
+
+        each_orphan_file do |file|
+          info = if dry_run
+                   "Can be moved to lost and found: #{file.key}"
+                 else
+                   new_path = move_to_lost_and_found(file)
+                   "Moved to lost and found: #{file.key} -> #{new_path}"
+                 end
+
+          logger.info(info)
+        end
+      end
+
+      private
+
+      def each_orphan_file
+        # we want to skip files already moved to lost_and_found directory
+        lost_dir_match = "^#{lost_and_found_dir}\/"
+
+        remote_directory.files.each_slice(BATCH_SIZE) do |remote_files|
+          remote_files.reject! { |file| file.key.match(/#{lost_dir_match}/) }
+          file_paths = remote_files.map(&:key)
+          tracked_paths = Upload
+            .where(store: ObjectStorage::Store::REMOTE, path: file_paths)
+            .pluck(:path)
+
+          remote_files.reject! { |file| tracked_paths.include?(file.key) }
+          remote_files.each do |file|
+            yield file
+          end
+        end
+      end
+
+      def move_to_lost_and_found(file)
+        new_path = "#{lost_and_found_dir}/#{file.key}"
+
+        file.copy(configuration['remote_directory'], new_path)
+        file.destroy
+
+        new_path
+      end
+
+      def lost_and_found_dir
+        'lost_and_found'
+      end
+
+      def remote_directory
+        connection.directories.get(configuration['remote_directory'])
+      end
+
+      def connection
+        ::Fog::Storage.new(configuration['connection'].symbolize_keys)
+      end
+
+      def configuration
+        Gitlab.config.uploads.object_store
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index a2feb074b1d..c8a8863443e 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -116,6 +116,16 @@ namespace :gitlab do
       end
     end
 
+    desc 'GitLab | Cleanup | Clean orphan remote upload files that do not exist in the db'
+    task remote_upload_files: :environment do
+      cleaner = Gitlab::Cleanup::RemoteUploads.new(logger: logger)
+      cleaner.run!(dry_run: dry_run?)
+
+      if dry_run?
+        logger.info "To cleanup these files run this command with DRY_RUN=false".color(:yellow)
+      end
+    end
+
     def remove?
       ENV['REMOVE'] == 'true'
     end