diff options
Diffstat (limited to 'app')
25 files changed, 144 insertions, 60 deletions
diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb new file mode 100644 index 00000000000..d4de4cf1fda --- /dev/null +++ b/app/controllers/concerns/send_file_upload.rb @@ -0,0 +1,14 @@ +module SendFileUpload + def send_upload(file_upload, send_params: {}, redirect_params: {}, attachment: nil) + if attachment + redirect_params[:query] = { "response-content-disposition" => "attachment;filename=#{attachment.inspect}" } + send_params.merge!(filename: attachment, disposition: 'attachment') + end + + if file_upload.file_storage? + send_file file_upload.path, send_params + else + redirect_to file_upload.url(**redirect_params) + end + end +end diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb index 0837451cc49..abc283d7aa9 100644 --- a/app/controllers/projects/artifacts_controller.rb +++ b/app/controllers/projects/artifacts_controller.rb @@ -1,6 +1,7 @@ class Projects::ArtifactsController < Projects::ApplicationController include ExtractsPath include RendersBlob + include SendFileUpload layout 'project' before_action :authorize_read_build! @@ -10,11 +11,7 @@ class Projects::ArtifactsController < Projects::ApplicationController before_action :entry, only: [:file] def download - if artifacts_file.file_storage? - send_file artifacts_file.path, disposition: 'attachment' - else - redirect_to artifacts_file.url - end + send_upload(artifacts_file, attachment: artifacts_file.filename) end def browse @@ -45,8 +42,7 @@ class Projects::ArtifactsController < Projects::ApplicationController end def raw - path = Gitlab::Ci::Build::Artifacts::Path - .new(params[:path]) + path = Gitlab::Ci::Build::Artifacts::Path.new(params[:path]) send_artifacts_entry(build, path) end @@ -75,7 +71,7 @@ class Projects::ArtifactsController < Projects::ApplicationController end def validate_artifacts! - render_404 unless build && build.artifacts? + render_404 unless build&.artifacts? end def build diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb index 8b54ba3ad7c..8041bdbb0bd 100644 --- a/app/controllers/projects/jobs_controller.rb +++ b/app/controllers/projects/jobs_controller.rb @@ -1,4 +1,6 @@ class Projects::JobsController < Projects::ApplicationController + prepend EE::Projects::JobsController + before_action :build, except: [:index, :cancel_all] before_action :authorize_read_build!, diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb index 941638db427..88fc373945a 100644 --- a/app/controllers/projects/lfs_storage_controller.rb +++ b/app/controllers/projects/lfs_storage_controller.rb @@ -1,6 +1,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController include LfsRequest include WorkhorseRequest + include SendFileUpload skip_before_action :verify_workhorse_api!, only: [:download, :upload_finalize] @@ -11,7 +12,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController return end - send_file lfs_object.file.path, content_type: "application/octet-stream" + send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" }) end def upload_authorize diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index a02cc477e08..9bc774b7636 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -2,6 +2,7 @@ class Projects::RawController < Projects::ApplicationController include ExtractsPath include BlobHelper + include SendFileUpload before_action :require_non_empty_project before_action :assign_ref_vars @@ -31,7 +32,7 @@ class Projects::RawController < Projects::ApplicationController lfs_object = find_lfs_object if lfs_object && lfs_object.project_allowed_access?(@project) - send_file lfs_object.file.path, filename: @blob.name, disposition: 'attachment' + send_upload(lfs_object.file, attachment: @blob.name) else render_404 end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b230b7f47ef..53a0d787988 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -47,6 +47,7 @@ module Ci end scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } + scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } @@ -362,13 +363,19 @@ module Ci project.running_or_pending_build_count(force: true) end + def browsable_artifacts? + artifacts_metadata? + end + def artifacts_metadata_entry(path, **options) - metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( - artifacts_metadata.path, - path, - **options) + artifacts_metadata.use_file do |metadata_path| + metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( + metadata_path, + path, + **options) - metadata.to_entry + metadata.to_entry + end end def erase_artifacts! diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 0a599f72bc7..2dfd8d4ef58 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -1,5 +1,6 @@ module Ci class JobArtifact < ActiveRecord::Base + include AfterCommitQueue extend Gitlab::Ci::Model belongs_to :project @@ -9,6 +10,12 @@ module Ci mount_uploader :file, JobArtifactUploader + after_save if: :file_changed?, on: [:create, :update] do + run_after_commit do + file.schedule_migration_to_object_storage + end + end + delegate :open, :exists?, to: :file enum file_type: { diff --git a/app/models/group.rb b/app/models/group.rb index 75bf013ecd2..f1fe1366935 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -186,12 +186,6 @@ class Group < Namespace owners.include?(user) && owners.size == 1 end - def avatar_type - unless self.avatar.image? - self.errors.add :avatar, "only images allowed" - end - end - def post_create_hook Gitlab::AppLogger.info("Group \"#{name}\" was created") diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index fc586fa216e..65c157d61ca 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -1,11 +1,22 @@ class LfsObject < ActiveRecord::Base + prepend EE::LfsObject + include AfterCommitQueue + has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :lfs_objects_projects validates :oid, presence: true, uniqueness: true + scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } + mount_uploader :file, LfsObjectUploader + after_save if: :file_changed?, on: [:create, :update] do + run_after_commit do + file.schedule_migration_to_object_storage + end + end + def project_allowed_access?(project) projects.exists?(project.lfs_storage_project.id) end diff --git a/app/models/project.rb b/app/models/project.rb index ba278a49688..23144506f63 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -286,6 +286,7 @@ class Project < ActiveRecord::Base scope :non_archived, -> { where(archived: false) } scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } + scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } scope :with_statistics, -> { includes(:statistics) } scope :with_shared_runners, -> { where(shared_runners_enabled: true) } diff --git a/app/models/upload.rb b/app/models/upload.rb index 99ad37dc892..3aca452616c 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -21,6 +21,7 @@ class Upload < ActiveRecord::Base end def absolute_path + raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local? return path unless relative_path? uploader_class.absolute_path(self) @@ -30,7 +31,7 @@ class Upload < ActiveRecord::Base self.checksum = nil return unless checksummable? - self.checksum = self.class.hexdigest(absolute_path) + self.checksum = Digest::SHA256.file(absolute_path).hexdigest end def build_uploader @@ -62,7 +63,9 @@ class Upload < ActiveRecord::Base end def local? - true + return true if store.nil? + + store == ObjectStorage::Store::LOCAL end def foreground_checksummable? diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index c760bd3b626..4edc272863a 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -67,9 +67,9 @@ module Projects end def extract_archive!(temp_path) - if artifacts.ends_with?('.tar.gz') || artifacts.ends_with?('.tgz') + if artifacts_filename.ends_with?('.tar.gz') || artifacts_filename.ends_with?('.tgz') extract_tar_archive!(temp_path) - elsif artifacts.ends_with?('.zip') + elsif artifacts_filename.ends_with?('.zip') extract_zip_archive!(temp_path) else raise 'unsupported artifacts format' @@ -77,11 +77,13 @@ module Projects end def extract_tar_archive!(temp_path) - results = Open3.pipeline(%W(gunzip -c #{artifacts}), - %W(dd bs=#{BLOCK_SIZE} count=#{blocks}), - %W(tar -x -C #{temp_path} #{SITE_PATH}), - err: '/dev/null') - raise 'pages failed to extract' unless results.compact.all?(&:success?) + build.artifacts_file.use_file do |artifacts_path| + results = Open3.pipeline(%W(gunzip -c #{artifacts_path}), + %W(dd bs=#{BLOCK_SIZE} count=#{blocks}), + %W(tar -x -C #{temp_path} #{SITE_PATH}), + err: '/dev/null') + raise 'pages failed to extract' unless results.compact.all?(&:success?) + end end def extract_zip_archive!(temp_path) @@ -99,8 +101,10 @@ module Projects # -n never overwrite existing files # We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories site_path = File.join(SITE_PATH, '*') - unless system(*%W(unzip -qq -n #{artifacts} #{site_path} -d #{temp_path})) - raise 'pages failed to extract' + build.artifacts_file.use_file do |artifacts_path| + unless system(*%W(unzip -n #{artifacts_path} #{site_path} -d #{temp_path})) + raise 'pages failed to extract' + end end end @@ -131,6 +135,10 @@ module Projects 1 + max_size / BLOCK_SIZE end + def artifacts_filename + build.artifacts_file.filename + end + def max_size max_pages_size = Gitlab::CurrentSettings.max_pages_size.megabytes @@ -159,10 +167,6 @@ module Projects build.ref end - def artifacts - build.artifacts_file.path - end - def latest_sha project.commit(build.ref).try(:sha).to_s end diff --git a/app/uploaders/attachment_uploader.rb b/app/uploaders/attachment_uploader.rb index 4930fb2fca7..cd819dc9bff 100644 --- a/app/uploaders/attachment_uploader.rb +++ b/app/uploaders/attachment_uploader.rb @@ -1,8 +1,8 @@ class AttachmentUploader < GitlabUploader - include UploaderHelper include RecordsUploads::Concern - - storage :file + include ObjectStorage::Concern + prepend ObjectStorage::Extension::RecordsUploads + include UploaderHelper private diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb index 5c8e1cea62e..5848e6c6994 100644 --- a/app/uploaders/avatar_uploader.rb +++ b/app/uploaders/avatar_uploader.rb @@ -1,18 +1,18 @@ class AvatarUploader < GitlabUploader include UploaderHelper include RecordsUploads::Concern - - storage :file + include ObjectStorage::Concern + prepend ObjectStorage::Extension::RecordsUploads def exists? model.avatar.file && model.avatar.file.present? end - def move_to_cache + def move_to_store false end - def move_to_store + def move_to_cache false end diff --git a/app/uploaders/file_mover.rb b/app/uploaders/file_mover.rb index 8f56f09c9f7..93913d2fcce 100644 --- a/app/uploaders/file_mover.rb +++ b/app/uploaders/file_mover.rb @@ -24,11 +24,8 @@ class FileMover updated_text = model.read_attribute(update_field) .gsub(temp_file_uploader.markdown_link, uploader.markdown_link) model.update_attribute(update_field, updated_text) - - true rescue revert - false end diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb index bde1161dfa8..051f1b19938 100644 --- a/app/uploaders/file_uploader.rb +++ b/app/uploaders/file_uploader.rb @@ -9,11 +9,13 @@ class FileUploader < GitlabUploader include UploaderHelper include RecordsUploads::Concern + include ObjectStorage::Concern + prepend ObjectStorage::Extension::RecordsUploads MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)} DYNAMIC_PATH_PATTERN = %r{(?<secret>\h{32})/(?<identifier>.*)} - storage :file + attr_accessor :model after :remove, :prune_store_dir diff --git a/app/uploaders/gitlab_uploader.rb b/app/uploaders/gitlab_uploader.rb index 010100f2da1..f12f0466a1d 100644 --- a/app/uploaders/gitlab_uploader.rb +++ b/app/uploaders/gitlab_uploader.rb @@ -37,12 +37,10 @@ class GitlabUploader < CarrierWave::Uploader::Base cache_storage.is_a?(CarrierWave::Storage::File) end - # Reduce disk IO def move_to_cache file_storage? end - # Reduce disk IO def move_to_store file_storage? end @@ -51,10 +49,6 @@ class GitlabUploader < CarrierWave::Uploader::Base file.present? end - def store_dir - File.join(base_dir, dynamic_segment) - end - def cache_dir File.join(root, base_dir, 'tmp/cache') end @@ -76,6 +70,10 @@ class GitlabUploader < CarrierWave::Uploader::Base # Designed to be overridden by child uploaders that have a dynamic path # segment -- that is, a path that changes based on mutable attributes of its # associated model + # + # For example, `FileUploader` builds the storage path based on the associated + # project model's `path_with_namespace` value, which can change when the + # project or its containing namespace is moved or renamed. def dynamic_segment raise(NotImplementedError) end diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb index ad5385f45a4..d80221144b4 100644 --- a/app/uploaders/job_artifact_uploader.rb +++ b/app/uploaders/job_artifact_uploader.rb @@ -1,5 +1,7 @@ class JobArtifactUploader < GitlabUploader + prepend EE::JobArtifactUploader extend Workhorse::UploadPath + include ObjectStorage::Concern storage_options Gitlab.config.artifacts diff --git a/app/uploaders/legacy_artifact_uploader.rb b/app/uploaders/legacy_artifact_uploader.rb index 28c458d3ff1..b726b053493 100644 --- a/app/uploaders/legacy_artifact_uploader.rb +++ b/app/uploaders/legacy_artifact_uploader.rb @@ -1,5 +1,6 @@ class LegacyArtifactUploader < GitlabUploader extend Workhorse::UploadPath + include ObjectStorage::Concern storage_options Gitlab.config.artifacts diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb index e04c97ce179..e7cce1bbb0a 100644 --- a/app/uploaders/lfs_object_uploader.rb +++ b/app/uploaders/lfs_object_uploader.rb @@ -1,5 +1,6 @@ class LfsObjectUploader < GitlabUploader extend Workhorse::UploadPath + include ObjectStorage::Concern # LfsObject are in `tmp/upload` instead of `tmp/uploads` def self.workhorse_upload_path diff --git a/app/uploaders/namespace_file_uploader.rb b/app/uploaders/namespace_file_uploader.rb index 993e85fbc13..269415b1926 100644 --- a/app/uploaders/namespace_file_uploader.rb +++ b/app/uploaders/namespace_file_uploader.rb @@ -14,6 +14,13 @@ class NamespaceFileUploader < FileUploader # Re-Override def store_dir - File.join(base_dir, dynamic_segment) + store_dirs[object_store] + end + + def store_dirs + { + Store::LOCAL => File.join(base_dir, dynamic_segment), + Store::REMOTE => File.join('namespace', model_path_segment, dynamic_segment) + } end end diff --git a/app/uploaders/personal_file_uploader.rb b/app/uploaders/personal_file_uploader.rb index f2ad0badd53..d225002ad2e 100644 --- a/app/uploaders/personal_file_uploader.rb +++ b/app/uploaders/personal_file_uploader.rb @@ -14,6 +14,12 @@ class PersonalFileUploader < FileUploader File.join(model.class.to_s.underscore, model.id.to_s) end + def object_store + return Store::LOCAL unless model + + super + end + # model_path_segment does not require a model to be passed, so we can always # generate a path, even when there's no model. def model_valid? @@ -22,7 +28,14 @@ class PersonalFileUploader < FileUploader # Revert-Override def store_dir - File.join(base_dir, dynamic_segment) + store_dirs[object_store] + end + + def store_dirs + { + Store::LOCAL => File.join(base_dir, dynamic_segment), + Store::REMOTE => File.join(model_path_segment, dynamic_segment) + } end private diff --git a/app/uploaders/records_uploads.rb b/app/uploaders/records_uploads.rb index 458928bc067..89c74a78835 100644 --- a/app/uploaders/records_uploads.rb +++ b/app/uploaders/records_uploads.rb @@ -24,8 +24,7 @@ module RecordsUploads uploads.where(path: upload_path).delete_all upload.destroy! if upload - self.upload = build_upload - upload.save! + self.upload = build_upload.tap(&:save!) end end diff --git a/app/views/projects/jobs/_sidebar.html.haml b/app/views/projects/jobs/_sidebar.html.haml index e779473c239..2a341569fb5 100644 --- a/app/views/projects/jobs/_sidebar.html.haml +++ b/app/views/projects/jobs/_sidebar.html.haml @@ -35,9 +35,9 @@ = link_to download_project_job_artifacts_path(@project, @build), rel: 'nofollow', download: '', class: 'btn btn-sm btn-default' do Download - - if @build.artifacts_metadata? - = link_to browse_project_job_artifacts_path(@project, @build), class: 'btn btn-sm btn-default' do - Browse + - if @build.browsable_artifacts? + = link_to browse_project_job_artifacts_path(@project, @build), class: 'btn btn-sm btn-default' do + Browse - if @build.trigger_request .build-widget.block diff --git a/app/workers/object_storage_upload_worker.rb b/app/workers/object_storage_upload_worker.rb new file mode 100644 index 00000000000..e087261770f --- /dev/null +++ b/app/workers/object_storage_upload_worker.rb @@ -0,0 +1,23 @@ +class ObjectStorageUploadWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + sidekiq_options retry: 5 + + def perform(uploader_class_name, subject_class_name, file_field, subject_id) + uploader_class = uploader_class_name.constantize + subject_class = subject_class_name.constantize + + return unless uploader_class < ObjectStorage::Concern + return unless uploader_class.object_store_enabled? + return unless uploader_class.licensed? + return unless uploader_class.background_upload_enabled? + + subject = subject_class.find(subject_id) + uploader = subject.public_send(file_field) # rubocop:disable GitlabSecurity/PublicSend + uploader.migrate!(ObjectStorage::Store::REMOTE) + rescue RecordNotFound + # does not retry when the record do not exists + Rails.logger.warn("Cannot find subject #{subject_class} with id=#{subject_id}.") + end +end |