Add `direct_upload` setting for artifacts

6 files changed, 71 insertions, 41 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 62ffebeacb0..073471b4c4d 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -78,6 +78,14 @@ module API
       rack_response({ 'message' => '404 Not found' }.to_json, 404)
     end
 
+    rescue_from UploadedFile::InvalidPathError do |e|
+      rack_response({ 'message' => e.message }.to_json, 400)
+    end
+
+    rescue_from ObjectStorage::RemoteStoreError do |e|
+      rack_response({ 'message' => e.message }.to_json, 500)
+    end
+
     # Retain 405 error rather than a 500 error for Grape 0.15.0+.
     # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes
     rescue_from Grape::Exceptions::MethodNotAllowed do |e|
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index e59e8a45908..65370ad5e56 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -388,28 +388,6 @@ module API
 
     # file helpers
 
-    def uploaded_file(field, uploads_path)
-      if params[field]
-        bad_request!("#{field} is not a file") unless params[field][:filename]
-        return params[field]
-      end
-
-      return nil unless params["#{field}.path"] && params["#{field}.name"]
-
-      # sanitize file paths
-      # this requires all paths to exist
-      required_attributes! %W(#{field}.path)
-      uploads_path = File.realpath(uploads_path)
-      file_path = File.realpath(params["#{field}.path"])
-      bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
-
-      UploadedFile.new(
-        file_path,
-        params["#{field}.name"],
-        params["#{field}.type"] || 'application/octet-stream'
-      )
-    end
-
     def present_disk_file!(path, filename, content_type = 'application/octet-stream')
       filename ||= File.basename(path)
       header['Content-Disposition'] = "attachment; filename=#{filename}"
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 834253d8e94..60aeb69e10a 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -186,7 +186,7 @@ module API
 
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-        Gitlab::Workhorse.artifact_upload_ok
+        JobArtifactUploader.workhorse_authorize
       end
 
       desc 'Upload artifacts for job' do
@@ -201,14 +201,15 @@ module API
         requires :id, type: Integer, desc: %q(Job's ID)
         optional :token, type: String, desc: %q(Job's authentication token)
         optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
-        optional :file, type: File, desc: %q(Artifact's file)
         optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
         optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
-        optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file)
+        optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse))
+        optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse))
         optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
-        optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of the file)
+        optional 'metadata.size', type: Integer, desc: %q(real size of metadata (generated by Workhorse))
+        optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of metadata (generated by Workhorse))
       end
       post '/:id/artifacts' do
         not_allowed! unless Gitlab.config.artifacts.enabled
@@ -217,21 +218,34 @@ module API
         job = authenticate_job!
         forbidden!('Job is not running!') unless job.running?
 
-        workhorse_upload_path = JobArtifactUploader.workhorse_upload_path
-        artifacts = uploaded_file(:file, workhorse_upload_path)
-        metadata = uploaded_file(:metadata, workhorse_upload_path)
+        artifacts = UploadedFile.from_params(params, :file, JobArtifactUploader.workhorse_local_upload_path)
+        metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path)
 
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
 
+        bad_request!("Already uploaded") if job.job_artifacts_archive
+
         expire_in = params['expire_in'] ||
           Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
 
-        job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, file_sha256: params['file.sha256'], expire_in: expire_in)
-        job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, file_sha256: params['metadata.sha256'], expire_in: expire_in) if metadata
-        job.artifacts_expire_in = expire_in
+        job.build_job_artifacts_archive(
+          project: job.project,
+          file: artifacts,
+          file_type: :archive,
+          file_sha256: artifacts.sha256,
+          expire_in: expire_in)
+
+        if metadata
+          job.build_job_artifacts_metadata(
+            project: job.project,
+            file: metadata,
+            file_type: :metadata,
+            file_sha256: metadata.sha256,
+            expire_in: expire_in)
+        end
 
-        if job.save
+        if job.update(artifacts_expire_in: expire_in)
           present job, with: Entities::JobRequest::Response
         else
           render_validation_error!(job)
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index d4c54049b74..a5f5d719cc1 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -82,7 +82,7 @@ module Gitlab
         end
 
         def open_file(path, name)
-          ::UploadedFile.new(path, name || File.basename(path), 'application/octet-stream')
+          ::UploadedFile.new(path, filename: name || File.basename(path), content_type: 'application/octet-stream')
         end
       end
 
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index b102812ec12..2faeaf16d55 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -36,10 +36,6 @@ module Gitlab
         }
       end
 
-      def artifact_upload_ok
-        { TempPath: JobArtifactUploader.workhorse_upload_path }
-      end
-
       def send_git_blob(repository, blob)
         params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
                    {
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
index 4a3c40f88eb..5dc85b2baea 100644
--- a/lib/uploaded_file.rb
+++ b/lib/uploaded_file.rb
@@ -1,8 +1,10 @@
 require "tempfile"
+require "tmpdir"
 require "fileutils"
 
-# Taken from: Rack::Test::UploadedFile
 class UploadedFile
+  InvalidPathError = Class.new(StandardError)
+
   # The filename, *not* including the path, of the "uploaded" file
   attr_reader :original_filename
 
@@ -12,14 +14,46 @@ class UploadedFile
   # The content type of the "uploaded" file
   attr_accessor :content_type
 
-  def initialize(path, filename, content_type = "text/plain")
-    raise "#{path} file does not exist" unless ::File.exist?(path)
+  attr_reader :remote_id
+  attr_reader :sha256
+
+  def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil)
+    raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path)
 
     @content_type = content_type
     @original_filename = filename || ::File.basename(path)
+    @content_type = content_type
+    @sha256 = sha256
+    @remote_id = remote_id
     @tempfile = File.new(path, 'rb')
   end
 
+  def self.from_params(params, field, upload_path)
+    unless params["#{field}.path"]
+      raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"]
+
+      return
+    end
+
+    file_path = File.realpath(params["#{field}.path"])
+
+    unless self.allowed_path?(file_path, [upload_path, Dir.tmpdir].compact)
+      raise InvalidPathError, "insecure path used '#{file_path}'"
+    end
+
+    UploadedFile.new(file_path,
+      filename: params["#{field}.name"],
+      content_type: params["#{field}.type"] || 'application/octet-stream',
+      sha256: params["#{field}.sha256"],
+      remote_id: params["#{field}.remote_id"])
+  end
+
+  def self.allowed_path?(file_path, paths)
+    paths.any? do |path|
+      File.exist?(path) && file_path.start_with?(File.realpath(path))
+    end
+  end
+
   def path
     @tempfile.path
   end