diff options
author | Jacob Vosmaer <jacob@gitlab.com> | 2016-08-18 17:31:44 +0300 |
---|---|---|
committer | Jacob Vosmaer <jacob@gitlab.com> | 2016-12-15 14:26:36 +0300 |
commit | 6731ab5d76c34462f0b4424ff03c9646ad916b76 (patch) | |
tree | 0fc3f62485f5dfdf36d02e6407bba7640ea87aef /lib/gitlab | |
parent | 01ffcceb81f1a556cdce15ec89c15be12dba9732 (diff) |
Add Gitlab::Middleware::Multipart
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/gfm/uploads_rewriter.rb | 19 | ||||
-rw-r--r-- | lib/gitlab/middleware/multipart.rb | 99 | ||||
-rw-r--r-- | lib/gitlab/workhorse.rb | 6 |
3 files changed, 122 insertions, 2 deletions
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index abc8c8c55e6..8fab5489616 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -1,3 +1,5 @@ +require 'fileutils' + module Gitlab module Gfm ## @@ -22,7 +24,9 @@ module Gitlab return markdown unless file.try(:exists?) new_uploader = FileUploader.new(target_project) - new_uploader.store!(file) + with_link_in_tmp_dir(file.file) do |open_tmp_file| + new_uploader.store!(open_tmp_file) + end new_uploader.to_markdown end end @@ -46,6 +50,19 @@ module Gitlab uploader.retrieve_from_store!(file) uploader.file end + + # Because the uploaders use 'move_to_store' we must have a temporary + # file that is allowed to be (re)moved. + def with_link_in_tmp_dir(file) + dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file)) + # The filename matters to Carrierwave so we make sure to preserve it + tmp_file = File.join(dir, File.basename(file)) + File.link(file, tmp_file) + # Open the file to placate Carrierwave + File.open(tmp_file) { |open_file| yield open_file } + ensure + FileUtils.rm_rf(dir) + end end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb new file mode 100644 index 00000000000..65713e73a59 --- /dev/null +++ b/lib/gitlab/middleware/multipart.rb @@ -0,0 +1,99 @@ +# Gitlab::Middleware::Multipart - a Rack::Multipart replacement +# +# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby +# process time to copy files around. This alternative solution uses +# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a +# location where copying should not be needed. +# +# When gitlab-workhorse finds files in a multipart MIME body it sends +# a signed message via a request header. This message lists the names of +# the multipart entries that gitlab-workhorse filtered out of the +# multipart structure and saved to tempfiles. Workhorse adds new entries +# in the multipart structure with paths to the tempfiles. +# +# The job of this Rack middleware is to detect and decode the message +# from workhorse. If present, it walks the Rack 'params' hash for the +# current request, opens the respective tempfiles, and inserts the open +# Ruby File objects in the params hash where Rack::Multipart would have +# put them. The goal is that application code deeper down can keep +# working the way it did with Rack::Multipart without changes. +# +# CAVEAT: the code that modifies the params hash is a bit complex. It is +# conceivable that certain Rack params structures will not be modified +# correctly. We are not aware of such bugs at this time though. +# + +module Gitlab + module Middleware + class Multipart + RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' + + class Handler + def initialize(env, message) + @request = Rack::Request.new(env) + @rewritten_fields = message['rewritten_fields'] + @open_files = [] + end + + def with_open_files + @rewritten_fields.each do |field, tmp_path| + parsed_field = Rack::Utils.parse_nested_query(field) + raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 + + key, value = parsed_field.first + if value.nil? + value = File.open(tmp_path) + @open_files << value + else + value = decorate_params_value(value, @request.params[key], tmp_path) + end + @request.update_param(key, value) + end + + yield + ensure + @open_files.each(&:close) + end + + # This function calls itself recursively + def decorate_params_value(path_hash, value_hash, tmp_path) + unless path_hash.is_a?(Hash) && path_hash.count == 1 + raise "invalid path: #{path_hash.inspect}" + end + path_key, path_value = path_hash.first + + unless value_hash.is_a?(Hash) && value_hash[path_key] + raise "invalid value hash: #{value_hash.inspect}" + end + + case path_value + when nil + value_hash[path_key] = File.open(tmp_path) + @open_files << value_hash[path_key] + value_hash + when Hash + decorate_params_value(path_value, value_hash[path_key], tmp_path) + value_hash + else + raise "unexpected path value: #{path_value.inspect}" + end + end + end + + def initialize(app) + @app = app + end + + def call(env) + encoded_message = env.delete(RACK_ENV_KEY) + return @app.call(env) if encoded_message.blank? + + message = Gitlab::Workhorse.decode_jwt(encoded_message)[0] + + Handler.new(env, message).with_open_files do + @app.call(env) + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 594439a5d4b..aeb1a26e1ba 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -117,8 +117,12 @@ module Gitlab end def verify_api_request!(request_headers) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) + end + + def decode_jwt(encoded_message) JWT.decode( - request_headers[INTERNAL_API_REQUEST_HEADER], + encoded_message, secret, true, { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, |