diff options
Diffstat (limited to 'lib/gitlab')
25 files changed, 1143 insertions, 97 deletions
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 85a2d1a93a7..0d156047ff0 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -33,8 +33,11 @@ module Grack auth! + lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call + return lfs_response unless lfs_response.nil? + if project && authorized_request? - # Tell gitlab-git-http-server the request is OK, and what the GL_ID is + # Tell gitlab-workhorse the request is OK, and what the GL_ID is render_grack_auth_ok elsif @user.nil? && !@ci unauthorized @@ -72,7 +75,7 @@ module Grack matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) if project && matched_login.present? && git_cmd == 'git-upload-pack' - underscored_service = matched_login['s'].underscore + underscored_service = matched_login['s'].underscore if Service.available_services_names.include?(underscored_service) service_method = "#{underscored_service}_service" diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb index d72391dade5..0d696a1ee28 100644 --- a/lib/gitlab/compare_result.rb +++ b/lib/gitlab/compare_result.rb @@ -2,8 +2,8 @@ module Gitlab class CompareResult attr_reader :commits, :diffs - def initialize(compare) - @commits, @diffs = compare.commits, compare.diffs + def initialize(compare, diff_options = {}) + @commits, @diffs = compare.commits, compare.diffs(nil, diff_options) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 0ea1b6a2f6f..2d3e32d9539 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -23,7 +23,9 @@ module Gitlab restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'] + import_sources: Settings.gitlab['import_sources'], + shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], + max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'], ) end diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb index fdb6a35c78d..93c6a5bb7f5 100644 --- a/lib/gitlab/force_push_check.rb +++ b/lib/gitlab/force_push_check.rb @@ -7,7 +7,7 @@ module Gitlab if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_refs, _ = Gitlab::Popen.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) + missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) missed_refs.split("\n").size > 0 end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c90184d31cf..3ed1eec517c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -13,7 +13,7 @@ module Gitlab def user return @user if defined?(@user) - @user = + @user = case actor when User actor @@ -125,7 +125,7 @@ module Gitlab def change_access_check(change) oldrev, newrev, ref = change.split(' ') - action = + action = if project.protected_branch?(branch_name(ref)) protected_branch_action(oldrev, newrev, branch_name(ref)) elsif protected_tag?(tag_name(ref)) @@ -148,7 +148,7 @@ module Gitlab build_status_object(false, "You are not allowed to change existing tags on this project.") else # :push_code build_status_object(false, "You are not allowed to push code to this project.") - end + end return status end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 39d17def930..4d83d8e72a8 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -6,7 +6,7 @@ module Gitlab # Returns true for a valid reference name, false otherwise def validate(ref_name) Gitlab::Utils.system_silent( - %W(git check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 03c410726a5..87fee28dc01 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -30,7 +30,7 @@ module Gitlab def user_map @user_map ||= begin - user_map = Hash.new do |hash, user| + user_map = Hash.new do |hash, user| # Replace ... by \.\.\., so `johnsm...@gmail.com` isn't autolinked. Client.mask_email(user).sub("...", "\\.\\.\\.") end @@ -76,18 +76,7 @@ module Gitlab attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"]) body = format_issue_body(author, date, content, attachments) - - labels = [] - raw_issue["labels"].each do |label| - name = nice_label_name(label) - labels << name - - unless @known_labels.include?(name) - create_label(name) - @known_labels << name - end - end - labels << nice_status_name(raw_issue["status"]) + labels = import_issue_labels(raw_issue) assignee_id = nil if raw_issue.has_key?("owner") @@ -110,6 +99,7 @@ module Gitlab assignee_id: assignee_id, state: raw_issue["state"] == "closed" ? "closed" : "opened" ) + issue.add_labels_by_names(labels) if issue.iid != raw_issue["id"] @@ -120,6 +110,23 @@ module Gitlab end end + def import_issue_labels(raw_issue) + labels = [] + + raw_issue["labels"].each do |label| + name = nice_label_name(label) + labels << name + + unless @known_labels.include?(name) + create_label(name) + @known_labels << name + end + end + + labels << nice_status_name(raw_issue["status"]) + labels + end + def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift @@ -172,7 +179,7 @@ module Gitlab "#5cb85c" when "Status: Started" "#8e44ad" - + when "Priority: Critical" "#ffcfcf" when "Priority: High" @@ -181,7 +188,7 @@ module Gitlab "#fff5cc" when "Priority: Low" "#cfe9ff" - + when "Type: Defect" "#d9534f" when "Type: Enhancement" @@ -249,8 +256,8 @@ module Gitlab end if raw_updates.has_key?("cc") - cc = raw_updates["cc"].map do |l| - deleted = l.start_with?("-") + cc = raw_updates["cc"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = user_map[l] l = "~~#{l}~~" if deleted @@ -261,8 +268,8 @@ module Gitlab end if raw_updates.has_key?("labels") - labels = raw_updates["labels"].map do |l| - deleted = l.start_with?("-") + labels = raw_updates["labels"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = nice_label_name(l) l = "~~#{l}~~" if deleted @@ -278,45 +285,39 @@ module Gitlab if raw_updates.has_key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocked on: #{blocked_ons.join(", ")}*" end if raw_updates.has_key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocking: #{blockings.join(", ")}*" end updates end + def format_blocking_updates(raw_blocked_on) + name, id = raw_blocked_on.split(":", 2) + + deleted = name.start_with?("-") + name = name[1..-1] if deleted + + text = + if name == project.import_source + "##{id}" + else + "#{project.namespace.path}/#{name}##{id}" + end + text = "~~#{text}~~" if deleted + text + end + def format_attachments(issue_id, comment_id, raw_attachments) return [] unless raw_attachments @@ -325,7 +326,7 @@ module Gitlab filename = attachment["fileName"] link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}" - + text = "[#{filename}](#{link})" text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/i text diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb index 99e7b529ba9..44507bde25d 100644 --- a/lib/gitlab/inline_diff.rb +++ b/lib/gitlab/inline_diff.rb @@ -11,48 +11,71 @@ module Gitlab indexes.each do |index| first_line = diff_arr[index+1] second_line = diff_arr[index+2] - max_length = [first_line.size, second_line.size].max # Skip inline diff if empty line was replaced with content next if first_line == "-\n" - first_the_same_symbols = 0 - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - if first_line[i] != second_line[i] && i > 0 - break - end - end + first_token = find_first_token(first_line, second_line) + apply_first_token(diff_arr, index, first_token) + + last_token = find_last_token(first_line, second_line, first_token) + apply_last_token(diff_arr, index, last_token) + end + + diff_arr + end + + def apply_first_token(diff_arr, index, first_token) + start = first_token + START + + if first_token.empty? + # In case if we remove string of spaces in commit + diff_arr[index+1].sub!("-", "-" => "-#{START}") + diff_arr[index+2].sub!("+", "+" => "+#{START}") + else + diff_arr[index+1].sub!(first_token, first_token => start) + diff_arr[index+2].sub!(first_token, first_token => start) + end + end - first_token = first_line[0..first_the_same_symbols][1..-1] - start = first_token + START + def apply_last_token(diff_arr, index, last_token) + # This is tricky: escape backslashes so that `sub` doesn't interpret them + # as backreferences. Regexp.escape does NOT do the right thing. + replace_token = FINISH + last_token.gsub(/\\/, '\&\&') + diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + end + + def find_first_token(first_line, second_line) + max_length = [first_line.size, second_line.size].max + first_the_same_symbols = 0 + + (0..max_length + 1).each do |i| + first_the_same_symbols = i - 1 - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) + if first_line[i] != second_line[i] && i > 0 + break end + end + + first_line[0..first_the_same_symbols][1..-1] + end + + def find_last_token(first_line, second_line, first_token) + max_length = [first_line.size, second_line.size].max + last_the_same_symbols = 0 + + (1..max_length + 1).each do |i| + last_the_same_symbols = -i + shortest_line = second_line.size > first_line.size ? first_line : second_line - last_the_same_symbols = 0 - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - if ( first_line[-i] != second_line[-i] ) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end + if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size + break end - last_the_same_symbols += 1 - last_token = first_line[last_the_same_symbols..-1] - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) end - diff_arr + + last_the_same_symbols += 1 + first_line[last_the_same_symbols..-1] end def _indexes_of_changed_lines(diff_arr) diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb new file mode 100644 index 00000000000..4202c786466 --- /dev/null +++ b/lib/gitlab/lfs/response.rb @@ -0,0 +1,308 @@ +module Gitlab + module Lfs + class Response + + def initialize(project, user, request) + @origin_project = project + @project = storage_project(project) + @user = user + @env = request.env + @request = request + end + + # Return a response for a download request + # Can be a response to: + # Request from a user to get the file + # Request from gitlab-workhorse which file to serve to the user + def render_download_hypermedia_response(oid) + render_response_to_download do + if check_download_accept_header? + render_lfs_download_hypermedia(oid) + else + render_not_found + end + end + end + + def render_download_object_response(oid) + render_response_to_download do + if check_download_sendfile_header? && check_download_accept_header? + render_lfs_sendfile(oid) + else + render_not_found + end + end + end + + def render_lfs_api_auth + render_response_to_push do + request_body = JSON.parse(@request.body.read) + return render_not_found if request_body.empty? || request_body['objects'].empty? + + response = build_response(request_body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_storage_upload_authorize_response(oid, size) + render_response_to_push do + [ + 200, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload", + 'LfsOid' => oid, + 'LfsSize' => size + })] + ] + end + end + + def render_storage_upload_store_response(oid, size, tmp_file_name) + render_response_to_push do + render_lfs_upload_ok(oid, size, tmp_file_name) + end + end + + private + + def render_not_enabled + [ + 501, + { + "Content-Type" => "application/vnd.git-lfs+json", + }, + [JSON.dump({ + 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_unauthorized + [ + 401, + { + 'Content-Type' => 'text/plain' + }, + ['Unauthorized'] + ] + end + + def render_not_found + [ + 404, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Not found.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_forbidden + [ + 403, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Access forbidden. Check your access level.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_lfs_sendfile(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + + if lfs_object && lfs_object.file.exists? + [ + 200, + { + # GitLab-workhorse will forward Content-Type header + "Content-Type" => "application/octet-stream", + "X-Sendfile" => lfs_object.file.path + }, + [] + ] + else + render_not_found + end + end + + def render_lfs_download_hypermedia(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + if lfs_object + [ + 200, + { "Content-Type" => "application/vnd.git-lfs+json" }, + [JSON.dump(download_hypermedia(oid))] + ] + else + render_not_found + end + end + + def render_lfs_upload_ok(oid, size, tmp_file) + if store_file(oid, size, tmp_file) + [ + 200, + { + 'Content-Type' => 'text/plain', + 'Content-Length' => 0 + }, + [] + ] + else + [ + 422, + { 'Content-Type' => 'text/plain' }, + ["Unprocessable entity"] + ] + end + end + + def render_response_to_download + return render_not_enabled unless Gitlab.config.lfs.enabled + + unless @project.public? + return render_unauthorized unless @user + return render_forbidden unless user_can_fetch? + end + + yield + end + + def render_response_to_push + return render_not_enabled unless Gitlab.config.lfs.enabled + return render_unauthorized unless @user + return render_forbidden unless user_can_push? + + yield + end + + def check_download_sendfile_header? + @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" + end + + def check_download_accept_header? + @env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8" + end + + def user_can_fetch? + # Check user access against the project they used to initiate the pull + @user.can?(:download_code, @origin_project) + end + + def user_can_push? + # Check user access against the project they used to initiate the push + @user.can?(:push_code, @origin_project) + end + + def storage_project(project) + if project.forked? + project.forked_from_project + else + project + end + end + + def store_file(oid, size, tmp_file) + tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) + + object = LfsObject.find_or_create_by(oid: oid, size: size) + if object.file.exists? + success = true + else + success = move_tmp_file_to_storage(object, tmp_file_path) + end + + if success + success = link_to_project(object) + end + + success + ensure + # Ensure that the tmp file is removed + FileUtils.rm_f(tmp_file_path) + end + + def object_for_download(oid) + @project.lfs_objects.find_by(oid: oid) + end + + def move_tmp_file_to_storage(object, path) + File.open(path) do |f| + object.file = f + end + + object.file.store! + object.save + end + + def link_to_project(object) + if object && !object.projects.exists?(@project) + object.projects << @project + object.save + end + end + + def select_existing_objects(objects) + objects_oids = objects.map { |o| o['oid'] } + @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set + end + + def build_response(objects) + selected_objects = select_existing_objects(objects) + + upload_hypermedia(objects, selected_objects) + end + + def download_hypermedia(oid) + { + '_links' => { + 'download' => + { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}", + 'header' => { + 'Accept' => "application/vnd.git-lfs+json; charset=utf-8", + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + } + end + + def upload_hypermedia(all_objects, existing_objects) + all_objects.each do |object| + object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid']) + end + + { 'objects' => all_objects } + end + + def hypermedia_links(object) + { + "upload" => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", + 'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] } + }.compact + } + end + end + end +end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb new file mode 100644 index 00000000000..4809e834984 --- /dev/null +++ b/lib/gitlab/lfs/router.rb @@ -0,0 +1,95 @@ +module Gitlab + module Lfs + class Router + def initialize(project, user, request) + @project = project + @user = user + @env = request.env + @request = request + end + + def try_call + return unless @request && @request.path.present? + + case @request.request_method + when 'GET' + get_response + when 'POST' + post_response + when 'PUT' + put_response + else + nil + end + end + + private + + def get_response + path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/) + return nil unless path_match + + oid = path_match[2] + return nil unless oid + + case path_match[1] + when "info/lfs" + lfs.render_download_hypermedia_response(oid) + when "gitlab-lfs" + lfs.render_download_object_response(oid) + else + nil + end + end + + def post_response + post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/) + return nil unless post_path + + # Check for Batch API + if post_path[0].ends_with?("/info/lfs/objects/batch") + lfs.render_lfs_api_auth + else + nil + end + end + + def put_response + object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/) + return nil if object_match.nil? + + oid = object_match[1] + size = object_match[2].try(:to_i) + return nil if oid.nil? || size.nil? + + # GitLab-workhorse requests + # 1. Try to authorize the request + # 2. send a request with a header containing the name of the temporary file + if object_match[3] && object_match[3] == '/authorize' + lfs.render_storage_upload_authorize_response(oid, size) + else + tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) + return nil unless tmp_file_name + + lfs.render_storage_upload_store_response(oid, size, tmp_file_name) + end + end + + def lfs + return unless @project + + Gitlab::Lfs::Response.new(@project, @user, @request) + end + + def sanitize_tmp_filename(name) + if name.present? + name.gsub!(/^.*(\\|\/)/, '') + name = name.match(/[0-9a-f]{73}/) + name[0] if name + else + nil + end + end + end + end +end diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb index 6ee3d1ce039..632be4d7542 100644 --- a/lib/gitlab/markdown/relative_link_filter.rb +++ b/lib/gitlab/markdown/relative_link_filter.rb @@ -51,7 +51,7 @@ module Gitlab relative_url_root, context[:project].path_with_namespace, path_type(file_path), - ref || 'master', # assume that if no ref exists we can point to master + ref || context[:project].default_branch, # if no ref exists, point to the default branch file_path ].compact.join('/').squeeze('/').chomp('/') diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb index e368de7d848..ffb9dc33b64 100644 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ b/lib/gitlab/markdown/sanitization_filter.rb @@ -48,6 +48,12 @@ module Gitlab # Allow span elements whitelist[:elements].push('span') + # Allow any protocol in `a` elements... + whitelist[:protocols].delete('a') + + # ...but then remove links with the `javascript` protocol + whitelist[:transformers].push(remove_javascript_links) + # Remove `rel` attribute from `a` elements whitelist[:transformers].push(remove_rel) @@ -57,6 +63,19 @@ module Gitlab whitelist end + def remove_javascript_links + lambda do |env| + node = env[:node] + + return unless node.name == 'a' + return unless node.has_attribute?('href') + + if node['href'].start_with?('javascript', ':javascript') + node.remove_attribute('href') + end + end + end + def remove_rel lambda do |env| if env[:node_name] == 'a' diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 90c3fe8da33..9ad7a38d505 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -1,6 +1,12 @@ module Gitlab module OAuth class Provider + LABELS = { + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google" + }.freeze + def self.providers Devise.omniauth_providers end @@ -23,8 +29,9 @@ module Gitlab end def self.label_for(name) + name = name.to_s config = config_for(name) - (config && config['label']) || name.to_s.titleize + (config && config['label']) || LABELS[name] || name.titleize end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0a2be605af9..70de6a74e76 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -20,6 +20,8 @@ module Gitlab Kaminari.paginate_array(blobs).page(page).per(per_page) when 'wiki_blobs' Kaminari.paginate_array(wiki_blobs).page(page).per(per_page) + when 'commits' + Kaminari.paginate_array(commits).page(page).per(per_page) else super end @@ -27,7 +29,7 @@ module Gitlab def total_count @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + notes_count + wiki_blobs_count + commits_count end def blobs_count @@ -42,6 +44,10 @@ module Gitlab @wiki_blobs_count ||= wiki_blobs.count end + def commits_count + @commits_count ||= commits.count + end + private def blobs @@ -70,6 +76,14 @@ module Gitlab Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC') end + def commits + if project.empty_repo? || query.blank? + [] + else + project.repository.find_commits_by_message(query).compact + end + end + def limit_project_ids [project.id] end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index d010ade704e..fa068d50763 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -18,7 +18,10 @@ module Gitlab # homepage: String, # }, # commits: Array, - # total_commits_count: Fixnum + # total_commits_count: Fixnum, + # added: ["CHANGELOG"], + # modified: [], + # removed: ["tmp/file.txt"] # } # def build(project, user, oldrev, newrev, ref, commits = [], message = nil) @@ -33,6 +36,8 @@ module Gitlab commit_attrs = commits_limited.map(&:hook_attrs) type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" + + repo_changes = repo_changes(project, newrev, oldrev) # Hash to be passed as post_receive_data data = { object_kind: type, @@ -55,7 +60,10 @@ module Gitlab visibility_level: project.visibility_level }, commits: commit_attrs, - total_commits_count: commits_count + total_commits_count: commits_count, + added: repo_changes[:added], + modified: repo_changes[:modified], + removed: repo_changes[:removed] } data @@ -86,6 +94,27 @@ module Gitlab newrev end end + + def repo_changes(project, newrev, oldrev) + changes = { added: [], modified: [], removed: [] } + compare_result = CompareService.new. + execute(project, newrev, project, oldrev) + + if compare_result + compare_result.diffs.each do |diff| + case true + when diff.deleted_file + changes[:removed] << diff.old_path + when diff.renamed_file, diff.new_file + changes[:added] << diff.new_path + else + changes[:modified] << diff.new_path + end + end + end + + changes + end end end end diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb new file mode 100644 index 00000000000..6360527a7aa --- /dev/null +++ b/lib/gitlab/sherlock.rb @@ -0,0 +1,19 @@ +require 'securerandom' + +module Gitlab + module Sherlock + @collection = Collection.new + + class << self + attr_reader :collection + end + + def self.enabled? + Rails.env.development? && !!ENV['ENABLE_SHERLOCK'] + end + + def self.enable_line_profiler? + RUBY_ENGINE == 'ruby' + end + end +end diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb new file mode 100644 index 00000000000..66bd6258521 --- /dev/null +++ b/lib/gitlab/sherlock/collection.rb @@ -0,0 +1,49 @@ +module Gitlab + module Sherlock + # A collection of transactions recorded by Sherlock. + # + # Method calls for this class are synchronized using a mutex to allow + # sharing of a single Collection instance between threads (e.g. when using + # Puma as a webserver). + class Collection + include Enumerable + + def initialize + @transactions = [] + @mutex = Mutex.new + end + + def add(transaction) + synchronize { @transactions << transaction } + end + + alias_method :<<, :add + + def each(&block) + synchronize { @transactions.each(&block) } + end + + def clear + synchronize { @transactions.clear } + end + + def empty? + synchronize { @transactions.empty? } + end + + def find_transaction(id) + find { |trans| trans.id == id } + end + + def newest_first + sort { |a, b| b.finished_at <=> a.finished_at } + end + + private + + def synchronize(&block) + @mutex.synchronize(&block) + end + end + end +end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb new file mode 100644 index 00000000000..8a3e1a5e5bf --- /dev/null +++ b/lib/gitlab/sherlock/file_sample.rb @@ -0,0 +1,31 @@ +module Gitlab + module Sherlock + class FileSample + attr_reader :id, :file, :line_samples, :events, :duration + + # file - The full path to the file this sample belongs to. + # line_samples - An array of LineSample objects. + # duration - The total execution time in milliseconds. + # events - The total amount of events. + def initialize(file, line_samples, duration, events) + @id = SecureRandom.uuid + @file = file + @line_samples = line_samples + @duration = duration + @events = events + end + + def relative_path + @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + end + + def to_param + @id + end + + def source + @source ||= File.read(@file) + end + end + end +end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb new file mode 100644 index 00000000000..aa1468bff6b --- /dev/null +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -0,0 +1,98 @@ +module Gitlab + module Sherlock + # Class for profiling code on a per line basis. + # + # The LineProfiler class can be used to profile code on per line basis + # without littering your code with Ruby implementation specific profiling + # methods. + # + # This profiler only includes samples taking longer than a given threshold + # and those that occur in the actual application (e.g. files from Gems are + # ignored). + class LineProfiler + # The minimum amount of time that has to be spent in a file for it to be + # included in a list of samples. + MINIMUM_DURATION = 10.0 + + # Profiles the given block. + # + # Example: + # + # profiler = LineProfiler.new + # + # retval, samples = profiler.profile do + # "cats are amazing" + # end + # + # retval # => "cats are amazing" + # samples # => [#<Gitlab::Sherlock::FileSample ...>, ...] + # + # Returns an Array containing the block's return value and an Array of + # FileSample objects. + def profile(&block) + if mri? + profile_mri(&block) + else + raise NotImplementedError, + 'Line profiling is not supported on this platform' + end + end + + # Profiles the given block using rblineprof (MRI only). + def profile_mri + require 'rblineprof' + + retval = nil + samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } + + file_samples = aggregate_rblineprof(samples) + + [retval, file_samples] + end + + # Returns an Array of file samples based on the output of rblineprof. + # + # lineprof_stats - A Hash containing rblineprof statistics on a per file + # basis. + # + # Returns an Array of FileSample objects. + def aggregate_rblineprof(lineprof_stats) + samples = [] + + lineprof_stats.each do |(file, stats)| + source_lines = File.read(file).each_line.to_a + line_samples = [] + + total_duration = microsec_to_millisec(stats[0][0]) + total_events = stats[0][2] + + next if total_duration <= MINIMUM_DURATION + + stats[1..-1].each_with_index do |data, index| + next unless source_lines[index] + + duration = microsec_to_millisec(data[0]) + events = data[2] + + line_samples << LineSample.new(duration, events) + end + + samples << FileSample. + new(file, line_samples, total_duration, total_events) + end + + samples + end + + private + + def microsec_to_millisec(microsec) + microsec / 1000.0 + end + + def mri? + RUBY_ENGINE == 'ruby' + end + end + end +end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb new file mode 100644 index 00000000000..eb1948eb6d6 --- /dev/null +++ b/lib/gitlab/sherlock/line_sample.rb @@ -0,0 +1,36 @@ +module Gitlab + module Sherlock + class LineSample + attr_reader :duration, :events + + # duration - The execution time in milliseconds. + # events - The amount of events. + def initialize(duration, events) + @duration = duration + @events = events + end + + # Returns the sample duration percentage relative to the given duration. + # + # Example: + # + # sample.duration # => 150 + # sample.percentage_of(1500) # => 10.0 + # + # total_duration - The total duration to compare with. + # + # Returns a float + def percentage_of(total_duration) + (duration.to_f / total_duration) * 100.0 + end + + # Returns true if the current sample takes up the majority of the given + # duration. + # + # total_duration - The total duration to compare with. + def majority_of?(total_duration) + percentage_of(total_duration) >= 30 + end + end + end +end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb new file mode 100644 index 00000000000..5ac265618ad --- /dev/null +++ b/lib/gitlab/sherlock/location.rb @@ -0,0 +1,26 @@ +module Gitlab + module Sherlock + class Location + attr_reader :path, :line + + SHERLOCK_DIR = File.dirname(__FILE__) + + # Creates a new Location from a `Thread::Backtrace::Location`. + def self.from_ruby_location(location) + new(location.path, location.lineno) + end + + # path - The full path of the frame as a String. + # line - The line number of the frame as a Fixnum. + def initialize(path, line) + @path = path + @line = line + end + + # Returns true if the current frame originated from the application. + def application? + @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) + end + end + end +end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb new file mode 100644 index 00000000000..687332fc5fc --- /dev/null +++ b/lib/gitlab/sherlock/middleware.rb @@ -0,0 +1,41 @@ +module Gitlab + module Sherlock + # Rack middleware used for tracking request metrics. + class Middleware + CONTENT_TYPES = /text\/html|application\/json/i + + IGNORE_PATHS = %r{^/sherlock} + + def initialize(app) + @app = app + end + + # env - A Hash containing Rack environment details. + def call(env) + if instrument?(env) + call_with_instrumentation(env) + else + @app.call(env) + end + end + + def call_with_instrumentation(env) + trans = transaction_from_env(env) + retval = trans.run { @app.call(env) } + + Sherlock.collection.add(trans) + + retval + end + + def instrument?(env) + !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && + env['REQUEST_URI'] !~ IGNORE_PATHS) + end + + def transaction_from_env(env) + Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) + end + end + end +end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb new file mode 100644 index 00000000000..4917c4ae2ac --- /dev/null +++ b/lib/gitlab/sherlock/query.rb @@ -0,0 +1,114 @@ +module Gitlab + module Sherlock + class Query + attr_reader :id, :query, :started_at, :finished_at, :backtrace + + # SQL identifiers that should be prefixed with newlines. + PREFIX_NEWLINE = / + \s+(FROM + |(LEFT|RIGHT)?INNER\s+JOIN + |(LEFT|RIGHT)?OUTER\s+JOIN + |WHERE + |AND + |GROUP\s+BY + |ORDER\s+BY + |LIMIT + |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + + # Creates a new Query using a String and a separate Array of bindings. + # + # query - A String containing a SQL query, optionally with numeric + # placeholders (`$1`, `$2`, etc). + # + # bindings - An Array of ActiveRecord columns and their values. + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + # + # Returns a new Query object. + def self.new_with_bindings(query, bindings, started_at, finished_at) + bindings.each_with_index do |(_, value), index| + quoted_value = ActiveRecord::Base.connection.quote(value) + + query = query.gsub("$#{index + 1}", quoted_value) + end + + new(query, started_at, finished_at) + end + + # query - The SQL query as a String (without placeholders). + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + def initialize(query, started_at, finished_at) + @id = SecureRandom.uuid + @query = query + @started_at = started_at + @finished_at = finished_at + @backtrace = caller_locations.map do |loc| + Location.from_ruby_location(loc) + end + + unless @query.end_with?(';') + @query += ';' + end + end + + # Returns the query duration in milliseconds. + def duration + @duration ||= (@finished_at - @started_at) * 1000.0 + end + + def to_param + @id + end + + # Returns a human readable version of the query. + def formatted_query + @formatted_query ||= format_sql(@query) + end + + # Returns the last application frame of the backtrace. + def last_application_frame + @last_application_frame ||= @backtrace.find(&:application?) + end + + # Returns an Array of application frames (excluding Gems and the likes). + def application_backtrace + @application_backtrace ||= @backtrace.select(&:application?) + end + + # Returns the query plan as a String. + def explain + unless @explain + ActiveRecord::Base.connection.transaction do + @explain = raw_explain(@query).values.flatten.join("\n") + + # Roll back any queries that mutate data so we don't mess up + # anything when running explain on an INSERT, UPDATE, DELETE, etc. + raise ActiveRecord::Rollback + end + end + + @explain + end + + private + + def raw_explain(query) + if Gitlab::Database.postgresql? + explain = "EXPLAIN ANALYZE #{query};" + else + explain = "EXPLAIN #{query};" + end + + ActiveRecord::Base.connection.execute(explain) + end + + def format_sql(query) + query.each_line. + map { |line| line.strip }. + join("\n"). + gsub(PREFIX_NEWLINE) { "\n#{$1} " } + end + end + end +end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb new file mode 100644 index 00000000000..d87a4c9bb4a --- /dev/null +++ b/lib/gitlab/sherlock/transaction.rb @@ -0,0 +1,131 @@ +module Gitlab + module Sherlock + class Transaction + attr_reader :id, :type, :path, :queries, :file_samples, :started_at, + :finished_at, :view_counts + + # type - The type of transaction (e.g. "GET", "POST", etc) + # path - The path of the transaction (e.g. the HTTP request path) + def initialize(type, path) + @id = SecureRandom.uuid + @type = type + @path = path + @queries = [] + @file_samples = [] + @started_at = nil + @finished_at = nil + @thread = Thread.current + @view_counts = Hash.new(0) + end + + # Runs the transaction and returns the block's return value. + def run + @started_at = Time.now + + retval = with_subscriptions do + profile_lines { yield } + end + + @finished_at = Time.now + + retval + end + + # Returns the duration in seconds. + def duration + @duration ||= started_at && finished_at ? finished_at - started_at : 0 + end + + def to_param + @id + end + + # Returns the queries sorted in descending order by their durations. + def sorted_queries + @queries.sort { |a, b| b.duration <=> a.duration } + end + + # Returns the file samples sorted in descending order by their durations. + def sorted_file_samples + @file_samples.sort { |a, b| b.duration <=> a.duration } + end + + # Finds a query by the given ID. + # + # id - The query ID as a String. + # + # Returns a Query object if one could be found, nil otherwise. + def find_query(id) + @queries.find { |query| query.id == id } + end + + # Finds a file sample by the given ID. + # + # id - The query ID as a String. + # + # Returns a FileSample object if one could be found, nil otherwise. + def find_file_sample(id) + @file_samples.find { |sample| sample.id == id } + end + + def profile_lines + retval = nil + + if Sherlock.enable_line_profiler? + retval, @file_samples = LineProfiler.new.profile { yield } + else + retval = yield + end + + retval + end + + def subscribe_to_active_record + ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| + next unless same_thread? + + track_query(data[:sql].strip, data[:binds], start, finish) + end + end + + def subscribe_to_action_view + regex = /render_(template|partial)\.action_view/ + + ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data| + next unless same_thread? + + track_view(data[:identifier]) + end + end + + private + + def track_query(query, bindings, start, finish) + @queries << Query.new_with_bindings(query, bindings, start, finish) + end + + def track_view(path) + @view_counts[path] += 1 + end + + def with_subscriptions + ar_subscriber = subscribe_to_active_record + av_subscriber = subscribe_to_action_view + + retval = yield + + ActiveSupport::Notifications.unsubscribe(ar_subscriber) + ActiveSupport::Notifications.unsubscribe(av_subscriber) + + retval + end + + # In case somebody uses a multi-threaded server locally (e.g. Puma) we + # _only_ want to track notifications that originate from the transaction + # thread. + def same_thread? + Thread.current == @thread + end + end + end +end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index cf040971c6e..f3567f3ef85 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -50,15 +50,15 @@ module Gitlab end def fetch_git_tags - remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) + remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) remote_tags.split("\n").grep(/tags\/v#{current_version.major}/) end def update_commands { - "Stash changed files" => %W(git stash), - "Get latest code" => %W(git fetch), - "Switch to new version" => %W(git checkout v#{latest_version}), + "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), + "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), + "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), "Install gems" => %W(bundle), "Migrate DB" => %W(bundle exec rake db:migrate), "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), |