diff options
author | Micaël Bergeron <mbergeron@gitlab.com> | 2018-01-29 20:57:34 +0300 |
---|---|---|
committer | Micaël Bergeron <mbergeron@gitlab.com> | 2018-02-01 20:14:46 +0300 |
commit | 2057a6acdee7c1f6824ff6289b0d979e8cb15f35 (patch) | |
tree | 25b911b147bfa0797dfee4cacf99f23ae5f55281 /app | |
parent | 402f3dfc0a962dc89d8334d4d61410e67e14a55f (diff) |
port of 594e6a0a625^..f74c90f68c6
Diffstat (limited to 'app')
27 files changed, 416 insertions, 340 deletions
diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb index a6fb1f40001..61554029d09 100644 --- a/app/controllers/concerns/uploads_actions.rb +++ b/app/controllers/concerns/uploads_actions.rb @@ -1,6 +1,8 @@ module UploadsActions include Gitlab::Utils::StrongMemoize + UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo).freeze + def create link_to_file = UploadService.new(model, params[:file], uploader_class).execute @@ -17,34 +19,71 @@ module UploadsActions end end + # This should either + # - send the file directly + # - or redirect to its URL + # def show return render_404 unless uploader.exists? - disposition = uploader.image_or_video? ? 'inline' : 'attachment' - - expires_in 0.seconds, must_revalidate: true, private: true + if uploader.file_storage? + disposition = uploader.image_or_video? ? 'inline' : 'attachment' + expires_in 0.seconds, must_revalidate: true, private: true - send_file uploader.file.path, disposition: disposition + send_file uploader.file.path, disposition: disposition + else + redirect_to uploader.url + end end private + def uploader_class + raise NotImplementedError + end + + def upload_mount + mounted_as = params[:mounted_as] + mounted_as if UPLOAD_MOUNTS.include?(mounted_as) + end + + def uploader_mounted? + upload_model_class < CarrierWave::Mount::Extension && !upload_mount.nil? + end + def uploader strong_memoize(:uploader) do - return if show_model.nil? + if uploader_mounted? + model.public_send(upload_mount) # rubocop:disable GitlabSecurity/PublicSend + else + build_uploader_from_upload || build_uploader_from_params + end + end + end - file_uploader = FileUploader.new(show_model, params[:secret]) - file_uploader.retrieve_from_store!(params[:filename]) + def build_uploader_from_upload + return nil unless params[:secret] && params[:filename] - file_uploader - end + upload_path = uploader_class.upload_path(params[:secret], params[:filename]) + upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_path) + upload&.build_uploader + end + + def build_uploader_from_params + uploader = uploader_class.new(model, params[:secret]) + uploader.retrieve_from_store!(params[:filename]) + uploader end def image_or_video? uploader && uploader.exists? && uploader.image_or_video? end - def uploader_class - FileUploader + def find_model + nil + end + + def model + strong_memoize(:model) { find_model } end end diff --git a/app/controllers/groups/uploads_controller.rb b/app/controllers/groups/uploads_controller.rb index e6bd9806401..f1578f75e88 100644 --- a/app/controllers/groups/uploads_controller.rb +++ b/app/controllers/groups/uploads_controller.rb @@ -7,29 +7,23 @@ class Groups::UploadsController < Groups::ApplicationController private - def show_model - strong_memoize(:show_model) do - group_id = params[:group_id] - - Group.find_by_full_path(group_id) - end + def upload_model_class + Group end - def authorize_upload_file! - render_404 unless can?(current_user, :upload_file, group) + def uploader_class + NamespaceFileUploader end - def uploader - strong_memoize(:uploader) do - file_uploader = uploader_class.new(show_model, params[:secret]) - file_uploader.retrieve_from_store!(params[:filename]) - file_uploader - end - end + def find_model + return @group if @group - def uploader_class - NamespaceFileUploader + group_id = params[:group_id] + + Group.find_by_full_path(group_id) end - alias_method :model, :group + def authorize_upload_file! + render_404 unless can?(current_user, :upload_file, group) + end end diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb index 293869345bd..941638db427 100644 --- a/app/controllers/projects/lfs_storage_controller.rb +++ b/app/controllers/projects/lfs_storage_controller.rb @@ -60,7 +60,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController def store_file(oid, size, tmp_file) # Define tmp_file_path early because we use it in "ensure" - tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) + tmp_file_path = File.join(LfsObjectUploader.workhorse_upload_path, tmp_file) object = LfsObject.find_or_create_by(oid: oid, size: size) file_exists = object.file.exists? || move_tmp_file_to_storage(object, tmp_file_path) diff --git a/app/controllers/projects/uploads_controller.rb b/app/controllers/projects/uploads_controller.rb index 4685bbe80b4..f5cf089ad98 100644 --- a/app/controllers/projects/uploads_controller.rb +++ b/app/controllers/projects/uploads_controller.rb @@ -1,6 +1,7 @@ class Projects::UploadsController < Projects::ApplicationController include UploadsActions + # These will kick you out if you don't have access. skip_before_action :project, :repository, if: -> { action_name == 'show' && image_or_video? } @@ -8,14 +9,20 @@ class Projects::UploadsController < Projects::ApplicationController private - def show_model - strong_memoize(:show_model) do - namespace = params[:namespace_id] - id = params[:project_id] + def upload_model_class + Project + end - Project.find_by_full_path("#{namespace}/#{id}") - end + def uploader_class + FileUploader end - alias_method :model, :project + def find_model + return @project if @project + + namespace = params[:namespace_id] + id = params[:project_id] + + Project.find_by_full_path("#{namespace}/#{id}") + end end diff --git a/app/controllers/uploads_controller.rb b/app/controllers/uploads_controller.rb index 16a74f82d3f..3d227b0a955 100644 --- a/app/controllers/uploads_controller.rb +++ b/app/controllers/uploads_controller.rb @@ -1,19 +1,34 @@ class UploadsController < ApplicationController include UploadsActions + UnknownUploadModelError = Class.new(StandardError) + + MODEL_CLASSES = { + "user" => User, + "project" => Project, + "note" => Note, + "group" => Group, + "appearance" => Appearance, + "personal_snippet" => PersonalSnippet, + nil => PersonalSnippet + }.freeze + + rescue_from UnknownUploadModelError, with: :render_404 + skip_before_action :authenticate_user! + before_action :upload_mount_satisfied? before_action :find_model before_action :authorize_access!, only: [:show] before_action :authorize_create_access!, only: [:create] - private + def uploader_class + PersonalFileUploader + end def find_model return nil unless params[:id] - return render_404 unless upload_model && upload_mount - - @model = upload_model.find(params[:id]) + upload_model_class.find(params[:id]) end def authorize_access! @@ -53,55 +68,17 @@ class UploadsController < ApplicationController end end - def upload_model - upload_models = { - "user" => User, - "project" => Project, - "note" => Note, - "group" => Group, - "appearance" => Appearance, - "personal_snippet" => PersonalSnippet - } - - upload_models[params[:model]] - end - - def upload_mount - return true unless params[:mounted_as] - - upload_mounts = %w(avatar attachment file logo header_logo) - - if upload_mounts.include?(params[:mounted_as]) - params[:mounted_as] - end + def upload_model_class + MODEL_CLASSES[params[:model]] || raise(UnknownUploadModelError) end - def uploader - return @uploader if defined?(@uploader) - - case model - when nil - @uploader = PersonalFileUploader.new(nil, params[:secret]) - - @uploader.retrieve_from_store!(params[:filename]) - when PersonalSnippet - @uploader = PersonalFileUploader.new(model, params[:secret]) - - @uploader.retrieve_from_store!(params[:filename]) - else - @uploader = @model.public_send(upload_mount) # rubocop:disable GitlabSecurity/PublicSend - - redirect_to @uploader.url unless @uploader.file_storage? - end - - @uploader + def upload_model_class_has_mounts? + upload_model_class < CarrierWave::Mount::Extension end - def uploader_class - PersonalFileUploader - end + def upload_mount_satisfied? + return true unless upload_model_class_has_mounts? - def model - @model ||= find_model + upload_model_class.uploader_options.has_key?(upload_mount) end end diff --git a/app/models/appearance.rb b/app/models/appearance.rb index 76cfe28742a..dcd14c08f3c 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -11,6 +11,7 @@ class Appearance < ActiveRecord::Base mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent CACHE_KEY = 'current_appearance'.freeze diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index 10659030910..d35e37935fb 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -1,6 +1,30 @@ module Avatarable extend ActiveSupport::Concern + included do + prepend ShadowMethods + + validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } + validates :avatar, file_size: { maximum: 200.kilobytes.to_i } + + mount_uploader :avatar, AvatarUploader + end + + module ShadowMethods + def avatar_url(**args) + # We use avatar_path instead of overriding avatar_url because of carrierwave. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 + + avatar_path(only_path: args.fetch(:only_path, true)) || super + end + end + + def avatar_type + unless self.avatar.image? + self.errors.add :avatar, "only images allowed" + end + end + def avatar_path(only_path: true) return unless self[:avatar].present? diff --git a/app/models/group.rb b/app/models/group.rb index fddace03387..415507a6aea 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -29,17 +29,14 @@ class Group < Namespace has_many :variables, class_name: 'Ci::GroupVariable' has_many :custom_attributes, class_name: 'GroupCustomAttribute' - validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + validate :visibility_level_allowed_by_projects validate :visibility_level_allowed_by_sub_groups validate :visibility_level_allowed_by_parent - validates :avatar, file_size: { maximum: 200.kilobytes.to_i } - validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } - mount_uploader :avatar, AvatarUploader - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent after_create :post_create_hook after_destroy :post_destroy_hook @@ -116,12 +113,6 @@ class Group < Namespace visibility_level_allowed_by_sub_groups?(level) end - def avatar_url(**args) - # We use avatar_path instead of overriding avatar_url because of carrierwave. - # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 - avatar_path(args) - end - def lfs_enabled? return false unless Gitlab.config.lfs.enabled return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil? diff --git a/app/models/note.rb b/app/models/note.rb index 184fbd5f5ae..a84db8982e5 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -88,6 +88,7 @@ class Note < ActiveRecord::Base end end + # @deprecated attachments are handler by the MarkdownUploader mount_uploader :attachment, AttachmentUploader # Scopes diff --git a/app/models/project.rb b/app/models/project.rb index 4def590a7a9..90f5df6265d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -256,9 +256,6 @@ class Project < ActiveRecord::Base validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } - validate :avatar_type, - if: ->(project) { project.avatar.present? && project.avatar_changed? } - validates :avatar, file_size: { maximum: 200.kilobytes.to_i } validate :visibility_level_allowed_by_group validate :visibility_level_allowed_as_fork validate :check_wiki_path_conflict @@ -266,7 +263,6 @@ class Project < ActiveRecord::Base presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } - mount_uploader :avatar, AvatarUploader has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # Scopes @@ -289,7 +285,6 @@ class Project < ActiveRecord::Base scope :non_archived, -> { where(archived: false) } scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } - scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } scope :with_statistics, -> { includes(:statistics) } scope :with_shared_runners, -> { where(shared_runners_enabled: true) } @@ -923,20 +918,12 @@ class Project < ActiveRecord::Base issues_tracker.to_param == 'jira' end - def avatar_type - unless self.avatar.image? - self.errors.add :avatar, 'only images allowed' - end - end - def avatar_in_git repository.avatar end def avatar_url(**args) - # We use avatar_path instead of overriding avatar_url because of carrierwave. - # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 - avatar_path(args) || (Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git) + Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git end # For compatibility with old code diff --git a/app/models/upload.rb b/app/models/upload.rb index f194d7bdb80..a9f144d2fa2 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -9,22 +9,11 @@ class Upload < ActiveRecord::Base validates :model, presence: true validates :uploader, presence: true - before_save :calculate_checksum, if: :foreground_checksum? - after_commit :schedule_checksum, unless: :foreground_checksum? + before_save :calculate_checksum!, if: :foreground_checksummable? + after_commit :schedule_checksum, if: :checksummable? - def self.remove_path(path) - where(path: path).destroy_all - end - - def self.record(uploader) - remove_path(uploader.relative_path) - - create( - size: uploader.file.size, - path: uploader.relative_path, - model: uploader.model, - uploader: uploader.class.to_s - ) + def self.hexdigest(path) + Digest::SHA256.file(path).hexdigest end def absolute_path @@ -33,10 +22,18 @@ class Upload < ActiveRecord::Base uploader_class.absolute_path(self) end - def calculate_checksum - return unless exist? + def calculate_checksum! + self.checksum = nil + return unless checksummable? - self.checksum = Digest::SHA256.file(absolute_path).hexdigest + self.checksum = self.class.hexdigest(absolute_path) + end + + def build_uploader + uploader_class.new(model).tap do |uploader| + uploader.upload = self + uploader.retrieve_from_store!(identifier) + end end def exist? @@ -45,8 +42,16 @@ class Upload < ActiveRecord::Base private - def foreground_checksum? - size <= CHECKSUM_THRESHOLD + def checksummable? + checksum.nil? && local? && exist? + end + + def local? + return true + end + + def foreground_checksummable? + checksummable? && size <= CHECKSUM_THRESHOLD end def schedule_checksum @@ -57,6 +62,10 @@ class Upload < ActiveRecord::Base !path.start_with?('/') end + def identifier + File.basename(path) + end + def uploader_class Object.const_get(uploader) end diff --git a/app/models/user.rb b/app/models/user.rb index fb5d56a68b0..89e787c3274 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -137,6 +137,7 @@ class User < ActiveRecord::Base has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent has_many :custom_attributes, class_name: 'UserCustomAttribute' + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # # Validations @@ -159,12 +160,10 @@ class User < ActiveRecord::Base validate :namespace_uniq, if: :username_changed? validate :namespace_move_dir_allowed, if: :username_changed? - validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :unique_email, if: :email_changed? validate :owns_notification_email, if: :notification_email_changed? validate :owns_public_email, if: :public_email_changed? validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id } - validates :avatar, file_size: { maximum: 200.kilobytes.to_i } before_validation :sanitize_attrs before_validation :set_notification_email, if: :email_changed? @@ -225,9 +224,6 @@ class User < ActiveRecord::Base end end - mount_uploader :avatar, AvatarUploader - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - # Scopes scope :admins, -> { where(admin: true) } scope :blocked, -> { with_states(:blocked, :ldap_blocked) } @@ -527,12 +523,6 @@ class User < ActiveRecord::Base end end - def avatar_type - unless avatar.image? - errors.add :avatar, "only images allowed" - end - end - def unique_email if !emails.exists?(email: email) && Email.exists?(email: email) errors.add(:email, 'has already been taken') @@ -860,9 +850,7 @@ class User < ActiveRecord::Base end def avatar_url(size: nil, scale: 2, **args) - # We use avatar_path instead of overriding avatar_url because of carrierwave. - # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 - avatar_path(args) || GravatarService.new.execute(email, size, scale, username: username) + GravatarService.new.execute(email, size, scale, username: username) end def primary_email_verified? diff --git a/app/services/projects/hashed_storage/migrate_attachments_service.rb b/app/services/projects/hashed_storage/migrate_attachments_service.rb index f8aaec8a9c0..bc897d891d5 100644 --- a/app/services/projects/hashed_storage/migrate_attachments_service.rb +++ b/app/services/projects/hashed_storage/migrate_attachments_service.rb @@ -14,9 +14,9 @@ module Projects @old_path = project.full_path @new_path = project.disk_path - origin = FileUploader.dynamic_path_segment(project) + origin = FileUploader.absolute_base_dir(project) project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] - target = FileUploader.dynamic_path_segment(project) + target = FileUploader.absolute_base_dir(project) result = move_folder!(origin, target) project.save! diff --git a/app/uploaders/attachment_uploader.rb b/app/uploaders/attachment_uploader.rb index 109eb2fea0b..4930fb2fca7 100644 --- a/app/uploaders/attachment_uploader.rb +++ b/app/uploaders/attachment_uploader.rb @@ -1,10 +1,12 @@ class AttachmentUploader < GitlabUploader - include RecordsUploads include UploaderHelper + include RecordsUploads::Concern storage :file - def store_dir - "#{base_dir}/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}" + private + + def dynamic_segment + File.join(model.class.to_s.underscore, mounted_as.to_s, model.id.to_s) end end diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb index cbb79376d5f..5c8e1cea62e 100644 --- a/app/uploaders/avatar_uploader.rb +++ b/app/uploaders/avatar_uploader.rb @@ -1,25 +1,24 @@ class AvatarUploader < GitlabUploader - include RecordsUploads include UploaderHelper + include RecordsUploads::Concern storage :file - def store_dir - "#{base_dir}/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}" - end - def exists? model.avatar.file && model.avatar.file.present? end - # We set move_to_store and move_to_cache to 'false' to prevent stealing - # the avatar file from a project when forking it. - # https://gitlab.com/gitlab-org/gitlab-ce/issues/26158 - def move_to_store + def move_to_cache false end - def move_to_cache + def move_to_store false end + + private + + def dynamic_segment + File.join(model.class.to_s.underscore, mounted_as.to_s, model.id.to_s) + end end diff --git a/app/uploaders/file_mover.rb b/app/uploaders/file_mover.rb index 00c2888d224..e7af1483d23 100644 --- a/app/uploaders/file_mover.rb +++ b/app/uploaders/file_mover.rb @@ -21,7 +21,8 @@ class FileMover end def update_markdown - updated_text = model.read_attribute(update_field).gsub(temp_file_uploader.to_markdown, uploader.to_markdown) + updated_text = model.read_attribute(update_field) + .gsub(temp_file_uploader.markdown_link, uploader.markdown_link) model.update_attribute(update_field, updated_text) true diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb index 0b591e3bbbb..85ae9863b13 100644 --- a/app/uploaders/file_uploader.rb +++ b/app/uploaders/file_uploader.rb @@ -1,23 +1,38 @@ +# This class breaks the actual CarrierWave concept. +# Every uploader should use a base_dir that is model agnostic so we can build +# back URLs from base_dir-relative paths saved in the `Upload` model. +# +# As the `.base_dir` is model dependent and **not** saved in the upload model (see #upload_path) +# there is no way to build back the correct file path without the model, which defies +# CarrierWave way of storing files. +# class FileUploader < GitlabUploader - include RecordsUploads include UploaderHelper + include RecordsUploads::Concern MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)} + DYNAMIC_PATH_PATTERN = %r{(?<secret>\h{32})/(?<identifier>.*)} storage :file - def self.absolute_path(upload_record) + def self.root + File.join(options.storage_path, 'uploads') + end + + def self.absolute_path(upload) File.join( - self.dynamic_path_segment(upload_record.model), - upload_record.path + absolute_base_dir(upload.model), + upload.path # already contain the dynamic_segment, see #upload_path ) end - # Not using `GitlabUploader.base_dir` because all project namespaces are in - # the `public/uploads` dir. - # - def self.base_dir - root_dir + def self.base_dir(model) + model_path_segment(model) + end + + # used in migrations and import/exports + def self.absolute_base_dir(model) + File.join(root, base_dir(model)) end # Returns the part of `store_dir` that can change based on the model's current @@ -29,63 +44,96 @@ class FileUploader < GitlabUploader # model - Object that responds to `full_path` and `disk_path` # # Returns a String without a trailing slash - def self.dynamic_path_segment(model) + def self.model_path_segment(model) if model.hashed_storage?(:attachments) - dynamic_path_builder(model.disk_path) + model.disk_path else - dynamic_path_builder(model.full_path) + model.full_path end end - # Auxiliary method to build dynamic path segment when not using a project model - # - # Prefer to use the `.dynamic_path_segment` as it includes Hashed Storage specific logic - def self.dynamic_path_builder(path) - File.join(CarrierWave.root, base_dir, path) + def self.upload_path(secret, identifier) + File.join(secret, identifier) + end + + def self.generate_secret + SecureRandom.hex end attr_accessor :model - attr_reader :secret def initialize(model, secret = nil) @model = model - @secret = secret || generate_secret + @secret = secret end - def store_dir - File.join(dynamic_path_segment, @secret) + def base_dir + self.class.base_dir(@model) end - def relative_path - self.file.path.sub("#{dynamic_path_segment}/", '') + # we don't need to know the actual path, an uploader instance should be + # able to yield the file content on demand, so we should build the digest + def absolute_path + self.class.absolute_path(@upload) end - def to_markdown - to_h[:markdown] + def upload_path + self.class.upload_path(dynamic_segment, identifier) end - def to_h - filename = image_or_video? ? self.file.basename : self.file.filename - escaped_filename = filename.gsub("]", "\\]") + def model_path_segment + self.class.model_path_segment(@model) + end + + def store_dir + File.join(base_dir, dynamic_segment) + end - markdown = "[#{escaped_filename}](#{secure_url})" + def markdown_link + markdown = "[#{markdown_name}](#{secure_url})" markdown.prepend("!") if image_or_video? || dangerous? + markdown + end + def to_h { - alt: filename, + alt: markdown_name, url: secure_url, - markdown: markdown + markdown: markdown_link } end + def filename + self.file.filename + end + + # the upload does not hold the secret, but holds the path + # which contains the secret: extract it + def upload=(value) + if matches = DYNAMIC_PATH_PATTERN.match(value.path) + @secret = matches[:secret] + @identifier = matches[:identifier] + end + + super + end + + def secret + @secret ||= self.class.generate_secret + end + private - def dynamic_path_segment - self.class.dynamic_path_segment(model) + def markdown_name + (image_or_video? ? File.basename(filename, File.extname(filename)) : filename).gsub("]", "\\]") end - def generate_secret - SecureRandom.hex + def identifier + @identifier ||= filename + end + + def dynamic_segment + secret end def secure_url diff --git a/app/uploaders/gitlab_uploader.rb b/app/uploaders/gitlab_uploader.rb index 7f72b3ce471..be8ab05b036 100644 --- a/app/uploaders/gitlab_uploader.rb +++ b/app/uploaders/gitlab_uploader.rb @@ -1,28 +1,32 @@ class GitlabUploader < CarrierWave::Uploader::Base - def self.absolute_path(upload_record) - File.join(CarrierWave.root, upload_record.path) - end + class_attribute :options - def self.root_dir - 'uploads' - end + class << self + # DSL setter + def storage_options(options) + self.options = options + end - # When object storage is used, keep the `root_dir` as `base_dir`. - # The files aren't really in folders there, they just have a name. - # The files that contain user input in their name, also contain a hash, so - # the names are still unique - # - # This method is overridden in the `FileUploader` - def self.base_dir - return root_dir unless file_storage? + def root + options.storage_path + end - File.join(root_dir, '-', 'system') - end + # represent the directory namespacing at the class level + def base_dir + options.fetch('base_dir', '') + end - def self.file_storage? - self.storage == CarrierWave::Storage::File + def file_storage? + storage == CarrierWave::Storage::File + end + + def absolute_path(upload_record) + File.join(root, upload_record.path) + end end + storage_options Gitlab.config.uploads + delegate :base_dir, :file_storage?, to: :class def file_cache_storage? @@ -31,34 +35,28 @@ class GitlabUploader < CarrierWave::Uploader::Base # Reduce disk IO def move_to_cache - true + super || true end # Reduce disk IO def move_to_store - true - end - - # Designed to be overridden by child uploaders that have a dynamic path - # segment -- that is, a path that changes based on mutable attributes of its - # associated model - # - # For example, `FileUploader` builds the storage path based on the associated - # project model's `path_with_namespace` value, which can change when the - # project or its containing namespace is moved or renamed. - def relative_path - self.file.path.sub("#{root}/", '') + super || true end def exists? file.present? end - # Override this if you don't want to save files by default to the Rails.root directory + def store_dir + File.join(base_dir, dynamic_segment) + end + + def cache_dir + File.join(root, base_dir, 'tmp/cache') + end + def work_dir - # Default path set by CarrierWave: - # https://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L182 - CarrierWave.tmp_path + File.join(root, base_dir, 'tmp/work') end def filename @@ -67,6 +65,13 @@ class GitlabUploader < CarrierWave::Uploader::Base private + # Designed to be overridden by child uploaders that have a dynamic path + # segment -- that is, a path that changes based on mutable attributes of its + # associated model + def dynamic_segment + raise(NotImplementedError) + end + # To prevent files from moving across filesystems, override the default # implementation: # http://github.com/carrierwaveuploader/carrierwave/blob/v1.0.0/lib/carrierwave/uploader/cache.rb#L181-L183 @@ -74,6 +79,6 @@ class GitlabUploader < CarrierWave::Uploader::Base # To be safe, keep this directory outside of the the cache directory # because calling CarrierWave.clean_cache_files! will remove any files in # the cache directory. - File.join(work_dir, @cache_id, version_name.to_s, for_file) + File.join(work_dir, cache_id, version_name.to_s, for_file) end end diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb index 15dfb5a5763..0abb462ab7d 100644 --- a/app/uploaders/job_artifact_uploader.rb +++ b/app/uploaders/job_artifact_uploader.rb @@ -1,13 +1,7 @@ class JobArtifactUploader < GitlabUploader - storage :file + extend Workhorse::UploadPath - def self.local_store_path - Gitlab.config.artifacts.path - end - - def self.artifacts_upload_path - File.join(self.local_store_path, 'tmp/uploads/') - end + storage_options Gitlab.config.artifacts def size return super if model.size.nil? @@ -16,24 +10,12 @@ class JobArtifactUploader < GitlabUploader end def store_dir - default_local_path - end - - def cache_dir - File.join(self.class.local_store_path, 'tmp/cache') - end - - def work_dir - File.join(self.class.local_store_path, 'tmp/work') + dynamic_segment end private - def default_local_path - File.join(self.class.local_store_path, default_path) - end - - def default_path + def dynamic_segment creation_date = model.created_at.utc.strftime('%Y_%m_%d') File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, diff --git a/app/uploaders/legacy_artifact_uploader.rb b/app/uploaders/legacy_artifact_uploader.rb index 4f7f8a63108..28c458d3ff1 100644 --- a/app/uploaders/legacy_artifact_uploader.rb +++ b/app/uploaders/legacy_artifact_uploader.rb @@ -1,33 +1,15 @@ class LegacyArtifactUploader < GitlabUploader - storage :file + extend Workhorse::UploadPath - def self.local_store_path - Gitlab.config.artifacts.path - end - - def self.artifacts_upload_path - File.join(self.local_store_path, 'tmp/uploads/') - end + storage_options Gitlab.config.artifacts def store_dir - default_local_path - end - - def cache_dir - File.join(self.class.local_store_path, 'tmp/cache') - end - - def work_dir - File.join(self.class.local_store_path, 'tmp/work') + dynamic_segment end private - def default_local_path - File.join(self.class.local_store_path, default_path) - end - - def default_path + def dynamic_segment File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s) end end diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb index d11ebf0f9ca..e04c97ce179 100644 --- a/app/uploaders/lfs_object_uploader.rb +++ b/app/uploaders/lfs_object_uploader.rb @@ -1,19 +1,24 @@ class LfsObjectUploader < GitlabUploader - storage :file + extend Workhorse::UploadPath - def store_dir - "#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}" + # LfsObject are in `tmp/upload` instead of `tmp/uploads` + def self.workhorse_upload_path + File.join(root, 'tmp/upload') end - def cache_dir - "#{Gitlab.config.lfs.storage_path}/tmp/cache" - end + storage_options Gitlab.config.lfs def filename model.oid[4..-1] end - def work_dir - File.join(Gitlab.config.lfs.storage_path, 'tmp', 'work') + def store_dir + dynamic_segment + end + + private + + def dynamic_segment + File.join(model.oid[0, 2], model.oid[2, 2]) end end diff --git a/app/uploaders/namespace_file_uploader.rb b/app/uploaders/namespace_file_uploader.rb index 672126e9ec2..993e85fbc13 100644 --- a/app/uploaders/namespace_file_uploader.rb +++ b/app/uploaders/namespace_file_uploader.rb @@ -1,15 +1,19 @@ class NamespaceFileUploader < FileUploader - def self.base_dir - File.join(root_dir, '-', 'system', 'namespace') + # Re-Override + def self.root + options.storage_path end - def self.dynamic_path_segment(model) - dynamic_path_builder(model.id.to_s) + def self.base_dir(model) + File.join(options.base_dir, 'namespace', model_path_segment(model)) end - private + def self.model_path_segment(model) + File.join(model.id.to_s) + end - def secure_url - File.join('/uploads', @secret, file.filename) + # Re-Override + def store_dir + File.join(base_dir, dynamic_segment) end end diff --git a/app/uploaders/personal_file_uploader.rb b/app/uploaders/personal_file_uploader.rb index 3298ad104ec..e7d9ecd3222 100644 --- a/app/uploaders/personal_file_uploader.rb +++ b/app/uploaders/personal_file_uploader.rb @@ -1,23 +1,27 @@ class PersonalFileUploader < FileUploader - def self.dynamic_path_segment(model) - File.join(CarrierWave.root, model_path(model)) + # Re-Override + def self.root + options.storage_path end - def self.base_dir - File.join(root_dir, '-', 'system') + def self.base_dir(model) + File.join(options.base_dir, model_path_segment(model)) end - private + def self.model_path_segment(model) + return 'temp/' unless model - def secure_url - File.join(self.class.model_path(model), secret, file.filename) + File.join(model.class.to_s.underscore, model.id.to_s) + end + + # Revert-Override + def store_dir + File.join(base_dir, dynamic_segment) end - def self.model_path(model) - if model - File.join("/#{base_dir}", model.class.to_s.underscore, model.id.to_s) - else - File.join("/#{base_dir}", 'temp') - end + private + + def secure_url + File.join('/', base_dir, secret, file.filename) end end diff --git a/app/uploaders/records_uploads.rb b/app/uploaders/records_uploads.rb index feb4f04d7b7..dfb8dccec57 100644 --- a/app/uploaders/records_uploads.rb +++ b/app/uploaders/records_uploads.rb @@ -1,35 +1,61 @@ module RecordsUploads - extend ActiveSupport::Concern + module Concern + extend ActiveSupport::Concern - included do - after :store, :record_upload - before :remove, :destroy_upload - end + attr_accessor :upload - # After storing an attachment, create a corresponding Upload record - # - # NOTE: We're ignoring the argument passed to this callback because we want - # the `SanitizedFile` object from `CarrierWave::Uploader::Base#file`, not the - # `Tempfile` object the callback gets. - # - # Called `after :store` - def record_upload(_tempfile = nil) - return unless model - return unless file_storage? - return unless file.exists? - - Upload.record(self) - end + included do + after :store, :record_upload + before :remove, :destroy_upload + end + + # After storing an attachment, create a corresponding Upload record + # + # NOTE: We're ignoring the argument passed to this callback because we want + # the `SanitizedFile` object from `CarrierWave::Uploader::Base#file`, not the + # `Tempfile` object the callback gets. + # + # Called `after :store` + def record_upload(_tempfile = nil) + return unless model + return unless file && file.exists? + + Upload.transaction do + uploads.where(path: upload_path).delete_all + upload.destroy! if upload + + self.upload = build_upload_from_uploader(self) + upload.save! + end + end + + def upload_path + File.join(store_dir, filename.to_s) + end + + private + + def uploads + Upload.order(id: :desc).where(uploader: self.class.to_s) + end - private + def build_upload_from_uploader(uploader) + Upload.new( + size: uploader.file.size, + path: uploader.upload_path, + model: uploader.model, + uploader: uploader.class.to_s + ) + end - # Before removing an attachment, destroy any Upload records at the same path - # - # Called `before :remove` - def destroy_upload(*args) - return unless file_storage? - return unless file + # Before removing an attachment, destroy any Upload records at the same path + # + # Called `before :remove` + def destroy_upload(*args) + return unless file && file.exists? - Upload.remove_path(relative_path) + self.upload = nil + uploads.where(path: upload_path).delete_all + end end end diff --git a/app/uploaders/uploader_helper.rb b/app/uploaders/uploader_helper.rb index 7635c20ab3a..fd446d31092 100644 --- a/app/uploaders/uploader_helper.rb +++ b/app/uploaders/uploader_helper.rb @@ -32,14 +32,7 @@ module UploaderHelper def extension_match?(extensions) return false unless file - extension = - if file.respond_to?(:extension) - file.extension - else - # Not all CarrierWave storages respond to :extension - File.extname(file.path).delete('.') - end - + extension = file.try(:extension) || File.extname(file.path).delete('.') extensions.include?(extension.downcase) end end diff --git a/app/uploaders/workhorse.rb b/app/uploaders/workhorse.rb new file mode 100644 index 00000000000..782032cf516 --- /dev/null +++ b/app/uploaders/workhorse.rb @@ -0,0 +1,7 @@ +module Workhorse + module UploadPath + def workhorse_upload_path + File.join(root, base_dir, 'tmp/uploads') + end + end +end diff --git a/app/workers/upload_checksum_worker.rb b/app/workers/upload_checksum_worker.rb index 9222760c031..65d40336f18 100644 --- a/app/workers/upload_checksum_worker.rb +++ b/app/workers/upload_checksum_worker.rb @@ -3,7 +3,7 @@ class UploadChecksumWorker def perform(upload_id) upload = Upload.find(upload_id) - upload.calculate_checksum + upload.calculate_checksum! upload.save! rescue ActiveRecord::RecordNotFound Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping") |