diff options
author | Marin Jankovski <maxlazio@gmail.com> | 2015-10-12 17:42:14 +0300 |
---|---|---|
committer | Marin Jankovski <maxlazio@gmail.com> | 2015-11-16 14:39:13 +0300 |
commit | 14032d8eb1a60ae5920286249c1044be2fa27278 (patch) | |
tree | 78d27cfb47590f5c30b1ae75de60368c5e269cc9 /lib | |
parent | 9179fcece813d0fe54abb5d7a28874cfa799f9c5 (diff) |
Add support for git lfs.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/backend/grack_auth.rb | 5 | ||||
-rw-r--r-- | lib/gitlab/git_access.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/lfs/response.rb | 308 | ||||
-rw-r--r-- | lib/gitlab/lfs/router.rb | 95 | ||||
-rw-r--r-- | lib/support/nginx/gitlab | 9 | ||||
-rw-r--r-- | lib/support/nginx/gitlab-ssl | 9 |
6 files changed, 426 insertions, 6 deletions
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 440ef5a3cb3..0d156047ff0 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -33,6 +33,9 @@ module Grack auth! + lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call + return lfs_response unless lfs_response.nil? + if project && authorized_request? # Tell gitlab-workhorse the request is OK, and what the GL_ID is render_grack_auth_ok @@ -72,7 +75,7 @@ module Grack matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) if project && matched_login.present? && git_cmd == 'git-upload-pack' - underscored_service = matched_login['s'].underscore + underscored_service = matched_login['s'].underscore if Service.available_services_names.include?(underscored_service) service_method = "#{underscored_service}_service" diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c90184d31cf..3ed1eec517c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -13,7 +13,7 @@ module Gitlab def user return @user if defined?(@user) - @user = + @user = case actor when User actor @@ -125,7 +125,7 @@ module Gitlab def change_access_check(change) oldrev, newrev, ref = change.split(' ') - action = + action = if project.protected_branch?(branch_name(ref)) protected_branch_action(oldrev, newrev, branch_name(ref)) elsif protected_tag?(tag_name(ref)) @@ -148,7 +148,7 @@ module Gitlab build_status_object(false, "You are not allowed to change existing tags on this project.") else # :push_code build_status_object(false, "You are not allowed to push code to this project.") - end + end return status end diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb new file mode 100644 index 00000000000..4202c786466 --- /dev/null +++ b/lib/gitlab/lfs/response.rb @@ -0,0 +1,308 @@ +module Gitlab + module Lfs + class Response + + def initialize(project, user, request) + @origin_project = project + @project = storage_project(project) + @user = user + @env = request.env + @request = request + end + + # Return a response for a download request + # Can be a response to: + # Request from a user to get the file + # Request from gitlab-workhorse which file to serve to the user + def render_download_hypermedia_response(oid) + render_response_to_download do + if check_download_accept_header? + render_lfs_download_hypermedia(oid) + else + render_not_found + end + end + end + + def render_download_object_response(oid) + render_response_to_download do + if check_download_sendfile_header? && check_download_accept_header? + render_lfs_sendfile(oid) + else + render_not_found + end + end + end + + def render_lfs_api_auth + render_response_to_push do + request_body = JSON.parse(@request.body.read) + return render_not_found if request_body.empty? || request_body['objects'].empty? + + response = build_response(request_body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_storage_upload_authorize_response(oid, size) + render_response_to_push do + [ + 200, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload", + 'LfsOid' => oid, + 'LfsSize' => size + })] + ] + end + end + + def render_storage_upload_store_response(oid, size, tmp_file_name) + render_response_to_push do + render_lfs_upload_ok(oid, size, tmp_file_name) + end + end + + private + + def render_not_enabled + [ + 501, + { + "Content-Type" => "application/vnd.git-lfs+json", + }, + [JSON.dump({ + 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_unauthorized + [ + 401, + { + 'Content-Type' => 'text/plain' + }, + ['Unauthorized'] + ] + end + + def render_not_found + [ + 404, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Not found.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_forbidden + [ + 403, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Access forbidden. Check your access level.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_lfs_sendfile(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + + if lfs_object && lfs_object.file.exists? + [ + 200, + { + # GitLab-workhorse will forward Content-Type header + "Content-Type" => "application/octet-stream", + "X-Sendfile" => lfs_object.file.path + }, + [] + ] + else + render_not_found + end + end + + def render_lfs_download_hypermedia(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + if lfs_object + [ + 200, + { "Content-Type" => "application/vnd.git-lfs+json" }, + [JSON.dump(download_hypermedia(oid))] + ] + else + render_not_found + end + end + + def render_lfs_upload_ok(oid, size, tmp_file) + if store_file(oid, size, tmp_file) + [ + 200, + { + 'Content-Type' => 'text/plain', + 'Content-Length' => 0 + }, + [] + ] + else + [ + 422, + { 'Content-Type' => 'text/plain' }, + ["Unprocessable entity"] + ] + end + end + + def render_response_to_download + return render_not_enabled unless Gitlab.config.lfs.enabled + + unless @project.public? + return render_unauthorized unless @user + return render_forbidden unless user_can_fetch? + end + + yield + end + + def render_response_to_push + return render_not_enabled unless Gitlab.config.lfs.enabled + return render_unauthorized unless @user + return render_forbidden unless user_can_push? + + yield + end + + def check_download_sendfile_header? + @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" + end + + def check_download_accept_header? + @env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8" + end + + def user_can_fetch? + # Check user access against the project they used to initiate the pull + @user.can?(:download_code, @origin_project) + end + + def user_can_push? + # Check user access against the project they used to initiate the push + @user.can?(:push_code, @origin_project) + end + + def storage_project(project) + if project.forked? + project.forked_from_project + else + project + end + end + + def store_file(oid, size, tmp_file) + tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) + + object = LfsObject.find_or_create_by(oid: oid, size: size) + if object.file.exists? + success = true + else + success = move_tmp_file_to_storage(object, tmp_file_path) + end + + if success + success = link_to_project(object) + end + + success + ensure + # Ensure that the tmp file is removed + FileUtils.rm_f(tmp_file_path) + end + + def object_for_download(oid) + @project.lfs_objects.find_by(oid: oid) + end + + def move_tmp_file_to_storage(object, path) + File.open(path) do |f| + object.file = f + end + + object.file.store! + object.save + end + + def link_to_project(object) + if object && !object.projects.exists?(@project) + object.projects << @project + object.save + end + end + + def select_existing_objects(objects) + objects_oids = objects.map { |o| o['oid'] } + @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set + end + + def build_response(objects) + selected_objects = select_existing_objects(objects) + + upload_hypermedia(objects, selected_objects) + end + + def download_hypermedia(oid) + { + '_links' => { + 'download' => + { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}", + 'header' => { + 'Accept' => "application/vnd.git-lfs+json; charset=utf-8", + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + } + end + + def upload_hypermedia(all_objects, existing_objects) + all_objects.each do |object| + object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid']) + end + + { 'objects' => all_objects } + end + + def hypermedia_links(object) + { + "upload" => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", + 'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] } + }.compact + } + end + end + end +end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb new file mode 100644 index 00000000000..4809e834984 --- /dev/null +++ b/lib/gitlab/lfs/router.rb @@ -0,0 +1,95 @@ +module Gitlab + module Lfs + class Router + def initialize(project, user, request) + @project = project + @user = user + @env = request.env + @request = request + end + + def try_call + return unless @request && @request.path.present? + + case @request.request_method + when 'GET' + get_response + when 'POST' + post_response + when 'PUT' + put_response + else + nil + end + end + + private + + def get_response + path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/) + return nil unless path_match + + oid = path_match[2] + return nil unless oid + + case path_match[1] + when "info/lfs" + lfs.render_download_hypermedia_response(oid) + when "gitlab-lfs" + lfs.render_download_object_response(oid) + else + nil + end + end + + def post_response + post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/) + return nil unless post_path + + # Check for Batch API + if post_path[0].ends_with?("/info/lfs/objects/batch") + lfs.render_lfs_api_auth + else + nil + end + end + + def put_response + object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/) + return nil if object_match.nil? + + oid = object_match[1] + size = object_match[2].try(:to_i) + return nil if oid.nil? || size.nil? + + # GitLab-workhorse requests + # 1. Try to authorize the request + # 2. send a request with a header containing the name of the temporary file + if object_match[3] && object_match[3] == '/authorize' + lfs.render_storage_upload_authorize_response(oid, size) + else + tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) + return nil unless tmp_file_name + + lfs.render_storage_upload_store_response(oid, size, tmp_file_name) + end + end + + def lfs + return unless @project + + Gitlab::Lfs::Response.new(@project, @user, @request) + end + + def sanitize_tmp_filename(name) + if name.present? + name.gsub!(/^.*(\\|\/)/, '') + name = name.match(/[0-9a-f]{73}/) + name[0] if name + else + nil + end + end + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 0a7a4118077..93f2ad07aeb 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -44,7 +44,7 @@ upstream gitlab-workhorse { ## Normal HTTP host server { - ## Either remove "default_server" from the listen line below, + ## Either remove "default_server" from the listen line below, ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab ## to be served if you visit any address that your server responds to, eg. ## the ip address of the server (http://x.x.x.x/)n 0.0.0.0:80 default_server; @@ -113,6 +113,13 @@ server { proxy_pass http://gitlab; } + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index b463d5b6aa9..90749947fa4 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -48,7 +48,7 @@ upstream gitlab-workhorse { ## Redirects all HTTP traffic to the HTTPS host server { - ## Either remove "default_server" from the listen line below, + ## Either remove "default_server" from the listen line below, ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab ## to be served if you visit any address that your server responds to, eg. ## the ip address of the server (http://x.x.x.x/) @@ -160,6 +160,13 @@ server { proxy_pass http://gitlab; } + location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects { + client_max_body_size 0; + # 'Error' 418 is a hack to re-use the @gitlab-workhorse block + error_page 418 = @gitlab-workhorse; + return 418; + } + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { # 'Error' 418 is a hack to re-use the @gitlab-workhorse block error_page 418 = @gitlab-workhorse; |