diff options
author | Jan Provaznik <jprovaznik@gitlab.com> | 2018-10-07 21:31:08 +0300 |
---|---|---|
committer | Jan Provaznik <jprovaznik@gitlab.com> | 2018-12-07 00:00:19 +0300 |
commit | 239fdc78b1ced1861cdcf00b8927963e30ef2095 (patch) | |
tree | f5888314cdedf47b21a259d334fc739be0e38d5a /app/models/uploads | |
parent | c3bbad762d418857e3f5b52222f5eedd62663229 (diff) |
Use FastDestroy for deleting uploads
It gathers list of file paths to delete before destroying
the parent object. Then after the parent_object is destroyed
these paths are scheduled for deletion asynchronously.
Carrierwave needed associated model for deleting upload file.
To avoid this requirement, simple Fog/File layer is used directly
for file deletion, this allows us to use just a simple list of paths.
Diffstat (limited to 'app/models/uploads')
-rw-r--r-- | app/models/uploads/base.rb | 19 | ||||
-rw-r--r-- | app/models/uploads/fog.rb | 43 | ||||
-rw-r--r-- | app/models/uploads/local.rb | 56 |
3 files changed, 118 insertions, 0 deletions
diff --git a/app/models/uploads/base.rb b/app/models/uploads/base.rb new file mode 100644 index 00000000000..f9814159958 --- /dev/null +++ b/app/models/uploads/base.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Uploads + class Base + BATCH_SIZE = 100 + + attr_reader :logger + + def initialize(logger: nil) + @logger ||= Rails.logger + end + + def delete_keys_async(keys_to_delete) + keys_to_delete.each_slice(BATCH_SIZE) do |batch| + DeleteStoredFilesWorker.perform_async(self.class, batch) + end + end + end +end diff --git a/app/models/uploads/fog.rb b/app/models/uploads/fog.rb new file mode 100644 index 00000000000..b44e273e9ab --- /dev/null +++ b/app/models/uploads/fog.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Uploads + class Fog < Base + include ::Gitlab::Utils::StrongMemoize + + def available? + object_store.enabled + end + + def keys(relation) + return [] unless available? + + relation.pluck(:path) + end + + def delete_keys(keys) + keys.each do |key| + connection.delete_object(bucket_name, key) + end + end + + private + + def object_store + Gitlab.config.uploads.object_store + end + + def bucket_name + return unless available? + + object_store.remote_directory + end + + def connection + return unless available? + + strong_memoize(:connection) do + ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys) + end + end + end +end diff --git a/app/models/uploads/local.rb b/app/models/uploads/local.rb new file mode 100644 index 00000000000..ffbf840a9d2 --- /dev/null +++ b/app/models/uploads/local.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Uploads + class Local < Base + def keys(relation) + relation.includes(:model).find_each.map {|u| u.absolute_path } + end + + def delete_keys(keys) + keys.each do |path| + delete_file(path) + end + end + + private + + def delete_file(path) + unless exists?(path) + logger.warn("File '#{path}' doesn't exist, skipping") + return + end + + unless in_uploads?(path) + message = "Path '#{path}' is not in uploads dir, skipping" + logger.warn(message) + Gitlab::Sentry.track_exception(RuntimeError.new(message), extra: { uploads_dir: storage_dir }) + return + end + + FileUtils.rm(path) + delete_dir!(File.dirname(path)) + end + + def exists?(path) + path.present? && File.exist?(path) + end + + def in_uploads?(path) + path.start_with?(storage_dir) + end + + def delete_dir!(path) + Dir.rmdir(path) + rescue Errno::ENOENT + # Ignore: path does not exist + rescue Errno::ENOTDIR + # Ignore: path is not a dir + rescue Errno::ENOTEMPTY, Errno::EEXIST + # Ignore: dir is not empty + end + + def storage_dir + @storage_dir ||= File.realpath(Gitlab.config.uploads.storage_path) + end + end +end |