diff options
author | Jarka Kadlecová <jarka@gitlab.com> | 2018-07-30 21:14:38 +0300 |
---|---|---|
committer | Jarka Kadlecová <jarka@gitlab.com> | 2018-08-07 13:51:58 +0300 |
commit | 6aaeb6dc411d6a92e9dc8d7968aa774c9e8ae044 (patch) | |
tree | f9877c182926456459693cd09c0aa63da21baa01 /lib | |
parent | ab08f9986de070b8d6bc15c36115653bc3ef3000 (diff) |
Clean orphaned files in object storage
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/cleanup/remote_uploads.rb | 80 | ||||
-rw-r--r-- | lib/tasks/gitlab/cleanup.rake | 10 |
2 files changed, 90 insertions, 0 deletions
diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb new file mode 100644 index 00000000000..45a5aea4fcd --- /dev/null +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true +module Gitlab + module Cleanup + class RemoteUploads + attr_reader :logger + + BATCH_SIZE = 100 + + def initialize(logger: nil) + @logger = logger || Rails.logger + end + + def run!(dry_run: false) + unless configuration.enabled + logger.warn "Object storage not enabled. Exit".color(:yellow) + + return + end + + logger.info "Looking for orphaned remote uploads to remove#{'. Dry run' if dry_run}..." + + each_orphan_file do |file| + info = if dry_run + "Can be moved to lost and found: #{file.key}" + else + new_path = move_to_lost_and_found(file) + "Moved to lost and found: #{file.key} -> #{new_path}" + end + + logger.info(info) + end + end + + private + + def each_orphan_file + # we want to skip files already moved to lost_and_found directory + lost_dir_match = "^#{lost_and_found_dir}\/" + + remote_directory.files.each_slice(BATCH_SIZE) do |remote_files| + remote_files.reject! { |file| file.key.match(/#{lost_dir_match}/) } + file_paths = remote_files.map(&:key) + tracked_paths = Upload + .where(store: ObjectStorage::Store::REMOTE, path: file_paths) + .pluck(:path) + + remote_files.reject! { |file| tracked_paths.include?(file.key) } + remote_files.each do |file| + yield file + end + end + end + + def move_to_lost_and_found(file) + new_path = "#{lost_and_found_dir}/#{file.key}" + + file.copy(configuration['remote_directory'], new_path) + file.destroy + + new_path + end + + def lost_and_found_dir + 'lost_and_found' + end + + def remote_directory + connection.directories.get(configuration['remote_directory']) + end + + def connection + ::Fog::Storage.new(configuration['connection'].symbolize_keys) + end + + def configuration + Gitlab.config.uploads.object_store + end + end + end +end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index a2feb074b1d..c8a8863443e 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -116,6 +116,16 @@ namespace :gitlab do end end + desc 'GitLab | Cleanup | Clean orphan remote upload files that do not exist in the db' + task remote_upload_files: :environment do + cleaner = Gitlab::Cleanup::RemoteUploads.new(logger: logger) + cleaner.run!(dry_run: dry_run?) + + if dry_run? + logger.info "To cleanup these files run this command with DRY_RUN=false".color(:yellow) + end + end + def remove? ENV['REMOVE'] == 'true' end |