diff options
author | Jeroen Nijhof <jeroen@jeroennijhof.nl> | 2016-01-06 16:55:44 +0300 |
---|---|---|
committer | Jeroen Nijhof <jeroen@jeroennijhof.nl> | 2016-01-06 16:55:44 +0300 |
commit | 9b28220f8874c7ab342286e74f0b21895a2dd777 (patch) | |
tree | 0b2ec2d97a95796893778623adabb975e0224b64 /lib/gitlab | |
parent | d4690af8bc283c402e49cb8b3056c7de9d57e886 (diff) | |
parent | 8b39b8cd54bb73b485ee6ea7fc5d3bbfbe07cd5d (diff) |
Merge gitlab.com:gitlab-org/gitlab-ce
Diffstat (limited to 'lib/gitlab')
92 files changed, 2528 insertions, 2034 deletions
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index bf33e5b1b1e..b203b9d70e4 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,14 +1,10 @@ require 'asciidoctor' -require 'html/pipeline' module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - # Provide autoload paths for filters to prevent a circular dependency error - autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline' @@ -24,13 +20,11 @@ module Gitlab # :requested_path # :ref # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter - # html_opts - a Hash of options for HTML output: - # :xhtml - output XHTML instead of HTML # - def self.render(input, context, asciidoc_opts = {}, html_opts = {}) - asciidoc_opts = asciidoc_opts.reverse_merge( + def self.render(input, context, asciidoc_opts = {}) + asciidoc_opts.reverse_merge!( safe: :secure, - backend: html_opts[:xhtml] ? :xhtml5 : :html5, + backend: :html5, attributes: [] ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) @@ -38,23 +32,10 @@ module Gitlab html = ::Asciidoctor.convert(input, asciidoc_opts) if context[:project] - result = HTML::Pipeline.new(filters).call(html, context) - - save_opts = html_opts[:xhtml] ? - Nokogiri::XML::Node::SaveOptions::AS_XHTML : 0 - - html = result[:output].to_html(save_with: save_opts) + html = Banzai.render(html, context.merge(pipeline: :asciidoc)) end html.html_safe end - - private - - def self.filters - [ - Gitlab::Markdown::RelativeLinkFilter - ] - end end end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 6830a916bcb..cdcaae8094c 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -33,8 +33,11 @@ module Grack auth! + lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call + return lfs_response unless lfs_response.nil? + if project && authorized_request? - # Tell gitlab-git-http-server the request is OK, and what the GL_ID is + # Tell gitlab-workhorse the request is OK, and what the GL_ID is render_grack_auth_ok elsif @user.nil? && !@ci unauthorized @@ -72,9 +75,11 @@ module Grack matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) if project && matched_login.present? && git_cmd == 'git-upload-pack' - underscored_service = matched_login['s'].underscore + underscored_service = matched_login['s'].underscore - if Service.available_services_names.include?(underscored_service) + if underscored_service == 'gitlab_ci' + return project && project.valid_build_token?(password) + elsif Service.available_services_names.include?(underscored_service) service_method = "#{underscored_service}_service" service = project.send(service_method) @@ -193,12 +198,19 @@ module Grack end def render_grack_auth_ok + repo_path = + if @request.path_info =~ /^([\w\.\/-]+)\.wiki\.git/ + ProjectWiki.new(project).repository.path_to_repo + else + project.repository.path_to_repo + end + [ 200, { "Content-Type" => "application/json" }, [JSON.dump({ 'GL_ID' => Gitlab::ShellEnv.gl_id(@user), - 'RepoPath' => project.repository.path_to_repo, + 'RepoPath' => repo_path, })] ] end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 59f7a45b791..4c15d58d680 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,8 +1,8 @@ module Gitlab class Shell - class AccessDenied < StandardError; end + class Error < StandardError; end - class KeyAdder < Struct.new(:io) + KeyAdder = Struct.new(:io) do def add_key(id, key) key.gsub!(/[[:space:]]+/, ' ').strip! io.puts("#{id}\t#{key}") @@ -36,8 +36,9 @@ module Gitlab # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(name, url) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'import-project', - "#{name}.git", url, '240']) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240']) + raise Error, output unless status.zero? + true end # Move repository diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 35e34d033e0..03aac1a025a 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo["name"], path: repo["slug"], description: repo["description"], diff --git a/lib/gitlab/blacklist.rb b/lib/gitlab/blacklist.rb deleted file mode 100644 index 43145e0ee1b..00000000000 --- a/lib/gitlab/blacklist.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - module Blacklist - extend self - - def path - %w( - admin - dashboard - files - groups - help - profile - projects - search - public - assets - u - s - teams - merge_requests - issues - users - snippets - services - repository - hooks - notes - unsubscribes - all - ci - ) - end - end -end diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb new file mode 100644 index 00000000000..86bfa0a4378 --- /dev/null +++ b/lib/gitlab/build_data_builder.rb @@ -0,0 +1,64 @@ +module Gitlab + class BuildDataBuilder + class << self + def build(build) + project = build.project + commit = build.commit + user = build.user + + data = { + object_kind: 'build', + + ref: build.ref, + tag: build.tag, + before_sha: build.before_sha, + sha: build.sha, + + # TODO: should this be not prefixed with build_? + # Leaving this way to have backward compatibility + build_id: build.id, + build_name: build.name, + build_stage: build.stage, + build_status: build.status, + build_started_at: build.started_at, + build_finished_at: build.finished_at, + build_duration: build.duration, + + # TODO: do we still need it? + project_id: project.id, + project_name: project.name_with_namespace, + + user: { + id: user.try(:id), + name: user.try(:name), + email: user.try(:email), + }, + + commit: { + id: commit.id, + sha: commit.sha, + message: commit.git_commit_message, + author_name: commit.git_author_name, + author_email: commit.git_author_email, + status: commit.status, + duration: commit.duration, + started_at: commit.started_at, + finished_at: commit.finished_at, + }, + + repository: { + name: project.name, + url: project.url_to_repo, + description: project.description, + homepage: project.web_url, + git_http_url: project.http_url_to_repo, + git_ssh_url: project.ssh_url_to_repo, + visibility_level: project.visibility_level + }, + } + + data + end + end + end +end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index aeec595782c..9bef9037ad6 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -1,6 +1,12 @@ module Gitlab class ClosingIssueExtractor - ISSUE_CLOSING_REGEX = Regexp.new(Gitlab.config.gitlab.issue_closing_pattern) + ISSUE_CLOSING_REGEX = begin + link_pattern = URI.regexp(%w(http https)) + + pattern = Gitlab.config.gitlab.issue_closing_pattern + pattern = pattern.sub('%{issue_ref}', "(?:(?:#{link_pattern})|(?:#{Issue.reference_pattern}))") + Regexp.new(pattern).freeze + end def initialize(project, current_user = nil) @extractor = Gitlab::ReferenceExtractor.new(project, current_user) @@ -9,10 +15,12 @@ module Gitlab def closed_by_message(message) return [] if message.nil? - closing_statements = message.scan(ISSUE_CLOSING_REGEX). - map { |ref| ref[0] }.join(" ") + closing_statements = [] + message.scan(ISSUE_CLOSING_REGEX) do + closing_statements << Regexp.last_match[0] + end - @extractor.analyze(closing_statements) + @extractor.analyze(closing_statements.join(" ")) @extractor.issues end diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb index d72391dade5..0d696a1ee28 100644 --- a/lib/gitlab/compare_result.rb +++ b/lib/gitlab/compare_result.rb @@ -2,8 +2,8 @@ module Gitlab class CompareResult attr_reader :commits, :diffs - def initialize(compare) - @commits, @diffs = compare.commits, compare.diffs + def initialize(compare, diff_options = {}) + @commits, @diffs = compare.commits, compare.diffs(nil, diff_options) end end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 8a7f8dc5003..85583dce9ee 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -45,11 +45,11 @@ module Gitlab end def starting_year - (Time.now - 1.year).strftime("%Y") + 1.year.ago.year end def starting_month - Date.today.strftime("%m").to_i + Date.today.month end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 0ea1b6a2f6f..7a86c09158e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -23,7 +23,9 @@ module Gitlab restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'] + import_sources: Settings.gitlab['import_sources'], + shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], + max_artifacts_size: Settings.artifacts['max_size'], ) end @@ -36,7 +38,9 @@ module Gitlab true end - use_db && ActiveRecord::Base.connection.active? && ActiveRecord::Base.connection.table_exists?('application_settings') + use_db && ActiveRecord::Base.connection.active? && + !ActiveRecord::Migrator.needs_migration? && + ActiveRecord::Base.connection.table_exists?('application_settings') end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 741a52714ac..de77a6fbff1 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,11 +1,29 @@ module Gitlab module Database def self.mysql? - ActiveRecord::Base.connection.adapter_name.downcase == 'mysql' + ActiveRecord::Base.connection.adapter_name.downcase == 'mysql2' end def self.postgresql? ActiveRecord::Base.connection.adapter_name.downcase == 'postgresql' end + + def true_value + case ActiveRecord::Base.connection.adapter_name.downcase + when 'postgresql' + "'t'" + else + 1 + end + end + + def false_value + case ActiveRecord::Base.connection.adapter_name.downcase + when 'postgresql' + "'f'" + else + 0 + end + end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 142058aa69d..79061cd0141 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -46,11 +46,11 @@ module Gitlab end def added_lines - diff_lines.select(&:added?).size + diff_lines.count(&:added?) end def removed_lines - diff_lines.select(&:removed?).size + diff_lines.count(&:removed?) end end end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb new file mode 100644 index 00000000000..a2eb7a70bd2 --- /dev/null +++ b/lib/gitlab/email/message/repository_push.rb @@ -0,0 +1,137 @@ +module Gitlab + module Email + module Message + class RepositoryPush + attr_accessor :recipient + attr_reader :author_id, :ref, :action + + include Gitlab::Application.routes.url_helpers + + delegate :namespace, :name_with_namespace, to: :project, prefix: :project + delegate :name, to: :author, prefix: :author + + def initialize(notify, project_id, recipient, opts = {}) + raise ArgumentError, 'Missing options: author_id, ref, action' unless + opts[:author_id] && opts[:ref] && opts[:action] + + @notify = notify + @project_id = project_id + @recipient = recipient + @opts = opts.dup + + @author_id = @opts.delete(:author_id) + @ref = @opts.delete(:ref) + @action = @opts.delete(:action) + end + + def project + @project ||= Project.find(@project_id) + end + + def author + @author ||= User.find(@author_id) + end + + def commits + @commits ||= (Commit.decorate(compare.commits, project) if compare) + end + + def diffs + @diffs ||= (compare.diffs if compare) + end + + def diffs_count + diffs.count if diffs + end + + def compare + @opts[:compare] + end + + def compare_timeout + compare.timeout if compare + end + + def reverse_compare? + @opts[:reverse_compare] || false + end + + def disable_diffs? + @opts[:disable_diffs] || false + end + + def send_from_committer_email? + @opts[:send_from_committer_email] || false + end + + def action_name + @action_name ||= + case @action + when :create + 'pushed new' + when :delete + 'deleted' + else + 'pushed to' + end + end + + def ref_name + @ref_name ||= Gitlab::Git.ref_name(@ref) + end + + def ref_type + @ref_type ||= Gitlab::Git.tag_ref?(@ref) ? 'tag' : 'branch' + end + + def target_url + if @action == :push && commits + if commits.length > 1 + namespace_project_compare_url(project_namespace, + project, + from: Commit.new(compare.base, project), + to: Commit.new(compare.head, project)) + else + namespace_project_commit_url(project_namespace, + project, commits.first) + end + else + unless @action == :delete + namespace_project_tree_url(project_namespace, + project, ref_name) + end + end + end + + def reply_to + if send_from_committer_email? && @notify.can_send_from_user_email?(author) + author.email + else + Gitlab.config.gitlab.email_reply_to + end + end + + def subject + subject_text = '[Git]' + subject_text << "[#{project.path_with_namespace}]" + subject_text << "[#{ref_name}]" if @action == :push + subject_text << ' ' + + if @action == :push && commits + if commits.length > 1 + subject_text << "Deleted " if reverse_compare? + subject_text << "#{commits.length} commits: #{commits.first.title}" + else + subject_text << "Deleted 1 commit: " if reverse_compare? + subject_text << commits.first.title + end + else + subject_action = action_name.dup + subject_action[0] = subject_action[0].capitalize + subject_text << "#{subject_action} #{ref_type} #{ref_name}" + end + end + end + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 496256700b8..403ebeec474 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -199,7 +199,7 @@ module Gitlab s = s.gsub(/^#/, "\\#") s = s.gsub(/^-/, "\\-") s = s.gsub("`", "\\~") - s = s.gsub("\r", "") + s = s.delete("\r") s = s.gsub("\n", " \n") s end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 8b1b6f48ed5..e0163499e30 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -12,7 +12,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo.safe_name, path: repo.path, namespace: namespace, diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb index fdb6a35c78d..93c6a5bb7f5 100644 --- a/lib/gitlab/force_push_check.rb +++ b/lib/gitlab/force_push_check.rb @@ -7,7 +7,7 @@ module Gitlab if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_refs, _ = Gitlab::Popen.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) + missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) missed_refs.split("\n").size > 0 end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 0c350d7c675..f065cc5e9e9 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -20,6 +20,10 @@ module Gitlab def blank_ref?(ref) ref == BLANK_SHA end + + def version + Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first) + end end end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index dd393fe09d2..07b856ca64c 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -16,6 +16,17 @@ module Gitlab def trigger(gl_id, oldrev, newrev, ref) return true unless exists? + case name + when "pre-receive", "post-receive" + call_receive_hook(gl_id, oldrev, newrev, ref) + when "update" + call_update_hook(gl_id, oldrev, newrev, ref) + end + end + + private + + def call_receive_hook(gl_id, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") # function will return true if succesful @@ -54,6 +65,12 @@ module Gitlab exit_status end + + def call_update_hook(gl_id, oldrev, newrev, ref) + Dir.chdir(repo_path) do + system({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + end + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c90184d31cf..3ed1eec517c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -13,7 +13,7 @@ module Gitlab def user return @user if defined?(@user) - @user = + @user = case actor when User actor @@ -125,7 +125,7 @@ module Gitlab def change_access_check(change) oldrev, newrev, ref = change.split(' ') - action = + action = if project.protected_branch?(branch_name(ref)) protected_branch_action(oldrev, newrev, branch_name(ref)) elsif protected_tag?(tag_name(ref)) @@ -148,7 +148,7 @@ module Gitlab build_status_object(false, "You are not allowed to change existing tags on this project.") else # :push_code build_status_object(false, "You are not allowed to push code to this project.") - end + end return status end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 39d17def930..4d83d8e72a8 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -6,7 +6,7 @@ module Gitlab # Returns true for a valid reference name, false otherwise def validate(ref_name) Gitlab::Utils.system_silent( - %W(git check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) end end end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb new file mode 100644 index 00000000000..202263c6742 --- /dev/null +++ b/lib/gitlab/github_import/base_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module GithubImport + class BaseFormatter + attr_reader :formatter, :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + private + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 270cbcd9ccd..74d1529e1ff 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -46,7 +46,7 @@ module Gitlab end def github_options - OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + OmniAuth::Strategies::GitHub.default_options[:client_options].to_h.symbolize_keys end end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb new file mode 100644 index 00000000000..7d58e53991a --- /dev/null +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module GithubImport + class CommentFormatter < BaseFormatter + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + if on_diff? + Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) + end + end + + def on_diff? + raw_data.path && raw_data.position + end + + def note + formatter.author_line(author) + body + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index bd7340a80f1..2b0afbc7b39 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -12,39 +12,59 @@ module Gitlab end def execute - #Issues && Comments + import_issues + import_pull_requests + + true + end + + private + + def import_issues client.list_issues(project.import_source, state: :all, sort: :created, - direction: :asc).each do |issue| - if issue.pull_request.nil? - - body = @formatter.author_line(issue.user.login) - body += issue.body + direction: :asc).each do |raw_data| + gh_issue = IssueFormatter.new(project, raw_data) - if issue.comments > 0 - body += @formatter.comments_header + if gh_issue.valid? + issue = Issue.create!(gh_issue.attributes) - client.issue_comments(project.import_source, issue.number).each do |c| - body += @formatter.comment(c.user.login, c.created_at, c.body) - end + if gh_issue.has_comments? + import_comments(gh_issue.number, issue) end + end + end + end + + def import_pull_requests + client.pull_requests(project.import_source, state: :all, + sort: :created, + direction: :asc).each do |raw_data| + pull_request = PullRequestFormatter.new(project, raw_data) - project.issues.create!( - description: body, - title: issue.title, - state: issue.state == 'closed' ? 'closed' : 'opened', - author_id: gl_user_id(project, issue.user.id) - ) + if !pull_request.cross_project? && pull_request.valid? + merge_request = MergeRequest.create!(pull_request.attributes) + import_comments(pull_request.number, merge_request) + import_comments_on_diff(pull_request.number, merge_request) end end end - private + def import_comments(issue_number, noteable) + comments = client.issue_comments(project.import_source, issue_number) + create_comments(comments, noteable) + end - def gl_user_id(project, github_id) - user = User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s) - (user && user.id) || project.creator_id + def import_comments_on_diff(pull_request_number, merge_request) + comments = client.pull_request_comments(project.import_source, pull_request_number) + create_comments(comments, merge_request) + end + + def create_comments(comments, noteable) + comments.each do |raw_data| + comment = CommentFormatter.new(project, raw_data) + noteable.notes.create!(comment.attributes) + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb new file mode 100644 index 00000000000..1e3ba44f27c --- /dev/null +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -0,0 +1,66 @@ +module Gitlab + module GithubImport + class IssueFormatter < BaseFormatter + def attributes + { + project: project, + title: raw_data.title, + description: description, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def has_comments? + raw_data.comments > 0 + end + + def number + raw_data.number + end + + def valid? + raw_data.pull_request.nil? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + @formatter.author_line(author) + body + end + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb new file mode 100644 index 00000000000..b7c47958cc7 --- /dev/null +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -0,0 +1,101 @@ +module Gitlab + module GithubImport + class PullRequestFormatter < BaseFormatter + def attributes + { + title: raw_data.title, + description: description, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def cross_project? + source_repo.fork == true + end + + def number + raw_data.number + end + + def valid? + source_branch.present? && target_branch.present? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + formatter.author_line(author) + body + end + + def source_project + project + end + + def source_repo + raw_data.head.repo + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + + def state + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + when raw_data.state == 'closed' + 'closed' + else + 'opened' + end + end + + def updated_at + case state + when 'merged' then raw_data.merged_at + when 'closed' then raw_data.closed_at + else + raw_data.updated_at + end + end + end + end +end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 9c00896c913..86fb6c51765 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -75,7 +75,7 @@ module Gitlab end def gitlab_options - OmniAuth::Strategies::GitLab.default_options[:client_options].symbolize_keys + OmniAuth::Strategies::GitLab.default_options[:client_options].to_h.symbolize_keys end end end diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index d9452de6a50..7baaadb813c 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo["name"], path: repo["path"], description: repo["description"], diff --git a/lib/gitlab/gitorious_import/project_creator.rb b/lib/gitlab/gitorious_import/project_creator.rb index cc9a91c91f4..8e22aa9286d 100644 --- a/lib/gitlab/gitorious_import/project_creator.rb +++ b/lib/gitlab/gitorious_import/project_creator.rb @@ -10,7 +10,8 @@ module Gitlab end def execute - ::Projects::CreateService.new(current_user, + ::Projects::CreateService.new( + current_user, name: repo.name, path: repo.path, description: repo.description, diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 03c410726a5..62da327931f 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -30,7 +30,7 @@ module Gitlab def user_map @user_map ||= begin - user_map = Hash.new do |hash, user| + user_map = Hash.new do |hash, user| # Replace ... by \.\.\., so `johnsm...@gmail.com` isn't autolinked. Client.mask_email(user).sub("...", "\\.\\.\\.") end @@ -76,18 +76,7 @@ module Gitlab attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"]) body = format_issue_body(author, date, content, attachments) - - labels = [] - raw_issue["labels"].each do |label| - name = nice_label_name(label) - labels << name - - unless @known_labels.include?(name) - create_label(name) - @known_labels << name - end - end - labels << nice_status_name(raw_issue["status"]) + labels = import_issue_labels(raw_issue) assignee_id = nil if raw_issue.has_key?("owner") @@ -110,6 +99,7 @@ module Gitlab assignee_id: assignee_id, state: raw_issue["state"] == "closed" ? "closed" : "opened" ) + issue.add_labels_by_names(labels) if issue.iid != raw_issue["id"] @@ -120,6 +110,23 @@ module Gitlab end end + def import_issue_labels(raw_issue) + labels = [] + + raw_issue["labels"].each do |label| + name = nice_label_name(label) + labels << name + + unless @known_labels.include?(name) + create_label(name) + @known_labels << name + end + end + + labels << nice_status_name(raw_issue["status"]) + labels + end + def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift @@ -164,15 +171,13 @@ module Gitlab when /\AMilestone:/ "#fee3ff" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" when "Status: New" "#428bca" when "Status: Accepted" "#5cb85c" when "Status: Started" "#8e44ad" - + when "Priority: Critical" "#ffcfcf" when "Priority: High" @@ -181,7 +186,7 @@ module Gitlab "#fff5cc" when "Priority: Low" "#cfe9ff" - + when "Type: Defect" "#d9534f" when "Type: Enhancement" @@ -192,6 +197,8 @@ module Gitlab "#8e44ad" when "Type: Other" "#7f8c8d" + when *@closed_statuses.map { |s| nice_status_name(s) } + "#cfcfcf" else "#e2e2e2" end @@ -220,7 +227,7 @@ module Gitlab s = s.gsub("`", "\\`") # Carriage returns make me sad - s = s.gsub("\r", "") + s = s.delete("\r") # Markdown ignores single newlines, but we need them as <br />. s = s.gsub("\n", " \n") @@ -249,8 +256,8 @@ module Gitlab end if raw_updates.has_key?("cc") - cc = raw_updates["cc"].map do |l| - deleted = l.start_with?("-") + cc = raw_updates["cc"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = user_map[l] l = "~~#{l}~~" if deleted @@ -261,8 +268,8 @@ module Gitlab end if raw_updates.has_key?("labels") - labels = raw_updates["labels"].map do |l| - deleted = l.start_with?("-") + labels = raw_updates["labels"].map do |l| + deleted = l.start_with?("-") l = l[1..-1] if deleted l = nice_label_name(l) l = "~~#{l}~~" if deleted @@ -278,45 +285,39 @@ module Gitlab if raw_updates.has_key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocked on: #{blocked_ons.join(", ")}*" end if raw_updates.has_key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| - name, id = raw_blocked_on.split(":", 2) - - deleted = name.start_with?("-") - name = name[1..-1] if deleted - - text = - if name == project.import_source - "##{id}" - else - "#{project.namespace.path}/#{name}##{id}" - end - text = "~~#{text}~~" if deleted - text + format_blocking_updates(raw_blocked_on) end + updates << "*Blocking: #{blockings.join(", ")}*" end updates end + def format_blocking_updates(raw_blocked_on) + name, id = raw_blocked_on.split(":", 2) + + deleted = name.start_with?("-") + name = name[1..-1] if deleted + + text = + if name == project.import_source + "##{id}" + else + "#{project.namespace.path}/#{name}##{id}" + end + text = "~~#{text}~~" if deleted + text + end + def format_attachments(issue_id, comment_id, raw_attachments) return [] unless raw_attachments @@ -325,7 +326,7 @@ module Gitlab filename = attachment["fileName"] link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}" - + text = "[#{filename}](#{link})" text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/i text diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 1cb7d16aeb3..87821c23460 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -11,7 +11,8 @@ module Gitlab end def execute - project = ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo.name, path: repo.name, description: repo.summary, diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb index 99e7b529ba9..44507bde25d 100644 --- a/lib/gitlab/inline_diff.rb +++ b/lib/gitlab/inline_diff.rb @@ -11,48 +11,71 @@ module Gitlab indexes.each do |index| first_line = diff_arr[index+1] second_line = diff_arr[index+2] - max_length = [first_line.size, second_line.size].max # Skip inline diff if empty line was replaced with content next if first_line == "-\n" - first_the_same_symbols = 0 - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - if first_line[i] != second_line[i] && i > 0 - break - end - end + first_token = find_first_token(first_line, second_line) + apply_first_token(diff_arr, index, first_token) + + last_token = find_last_token(first_line, second_line, first_token) + apply_last_token(diff_arr, index, last_token) + end + + diff_arr + end + + def apply_first_token(diff_arr, index, first_token) + start = first_token + START + + if first_token.empty? + # In case if we remove string of spaces in commit + diff_arr[index+1].sub!("-", "-" => "-#{START}") + diff_arr[index+2].sub!("+", "+" => "+#{START}") + else + diff_arr[index+1].sub!(first_token, first_token => start) + diff_arr[index+2].sub!(first_token, first_token => start) + end + end - first_token = first_line[0..first_the_same_symbols][1..-1] - start = first_token + START + def apply_last_token(diff_arr, index, last_token) + # This is tricky: escape backslashes so that `sub` doesn't interpret them + # as backreferences. Regexp.escape does NOT do the right thing. + replace_token = FINISH + last_token.gsub(/\\/, '\&\&') + diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) + end + + def find_first_token(first_line, second_line) + max_length = [first_line.size, second_line.size].max + first_the_same_symbols = 0 + + (0..max_length + 1).each do |i| + first_the_same_symbols = i - 1 - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) + if first_line[i] != second_line[i] && i > 0 + break end + end + + first_line[0..first_the_same_symbols][1..-1] + end + + def find_last_token(first_line, second_line, first_token) + max_length = [first_line.size, second_line.size].max + last_the_same_symbols = 0 + + (1..max_length + 1).each do |i| + last_the_same_symbols = -i + shortest_line = second_line.size > first_line.size ? first_line : second_line - last_the_same_symbols = 0 - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - if ( first_line[-i] != second_line[-i] ) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end + if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size + break end - last_the_same_symbols += 1 - last_token = first_line[last_the_same_symbols..-1] - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) end - diff_arr + + last_the_same_symbols += 1 + first_line[last_the_same_symbols..-1] end def _indexes_of_changed_lines(diff_arr) diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 16ff03c38d4..c438a3d167b 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -37,13 +37,15 @@ module Gitlab # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.block unless user.blocked? + user.block false else user.activate if user.blocked? && !ldap_config.block_auto_created_users true end else + # Block the user if they no longer exist in LDAP/AD + user.block false end rescue diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 4be99dd88c2..aef08c97d1d 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -14,7 +14,7 @@ module Gitlab # LDAP distinguished name is case-insensitive identity = ::Identity. where(provider: provider). - where('lower(extern_uid) = ?', uid.mb_chars.downcase.to_s).last + iwhere(extern_uid: uid).last identity && identity.user end end @@ -31,7 +31,7 @@ module Gitlab def find_by_uid_and_provider self.class.find_by_uid_and_provider( - auth_hash.uid.downcase, auth_hash.provider) + auth_hash.uid, auth_hash.provider) end def find_by_email @@ -47,7 +47,7 @@ module Gitlab # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } identity ||= gl_user.identities.build(provider: auth_hash.provider) - + # For a new user set extern_uid to the LDAP DN # For an existing user with matching email but changed DN, update the DN. # For an existing user with no change in DN, this line changes nothing. diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb new file mode 100644 index 00000000000..9d9617761b3 --- /dev/null +++ b/lib/gitlab/lfs/response.rb @@ -0,0 +1,327 @@ +module Gitlab + module Lfs + class Response + + def initialize(project, user, request) + @origin_project = project + @project = storage_project(project) + @user = user + @env = request.env + @request = request + end + + def render_download_object_response(oid) + render_response_to_download do + if check_download_sendfile_header? + render_lfs_sendfile(oid) + else + render_not_found + end + end + end + + def render_batch_operation_response + request_body = JSON.parse(@request.body.read) + case request_body["operation"] + when "download" + render_batch_download(request_body) + when "upload" + render_batch_upload(request_body) + else + render_not_found + end + end + + def render_storage_upload_authorize_response(oid, size) + render_response_to_push do + [ + 200, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload", + 'LfsOid' => oid, + 'LfsSize' => size + })] + ] + end + end + + def render_storage_upload_store_response(oid, size, tmp_file_name) + render_response_to_push do + render_lfs_upload_ok(oid, size, tmp_file_name) + end + end + + def render_unsupported_deprecated_api + [ + 501, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'message' => 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + private + + def render_not_enabled + [ + 501, + { + "Content-Type" => "application/json; charset=utf-8", + }, + [JSON.dump({ + 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_unauthorized + [ + 401, + { + 'Content-Type' => 'text/plain' + }, + ['Unauthorized'] + ] + end + + def render_not_found + [ + 404, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Not found.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_forbidden + [ + 403, + { + "Content-Type" => "application/vnd.git-lfs+json" + }, + [JSON.dump({ + 'message' => 'Access forbidden. Check your access level.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + + def render_lfs_sendfile(oid) + return render_not_found unless oid.present? + + lfs_object = object_for_download(oid) + + if lfs_object && lfs_object.file.exists? + [ + 200, + { + # GitLab-workhorse will forward Content-Type header + "Content-Type" => "application/octet-stream", + "X-Sendfile" => lfs_object.file.path + }, + [] + ] + else + render_not_found + end + end + + def render_batch_upload(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_push do + response = build_upload_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_batch_download(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_download do + response = build_download_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_lfs_upload_ok(oid, size, tmp_file) + if store_file(oid, size, tmp_file) + [ + 200, + { + 'Content-Type' => 'text/plain', + 'Content-Length' => 0 + }, + [] + ] + else + [ + 422, + { 'Content-Type' => 'text/plain' }, + ["Unprocessable entity"] + ] + end + end + + def render_response_to_download + return render_not_enabled unless Gitlab.config.lfs.enabled + + unless @project.public? + return render_unauthorized unless @user + return render_forbidden unless user_can_fetch? + end + + yield + end + + def render_response_to_push + return render_not_enabled unless Gitlab.config.lfs.enabled + return render_unauthorized unless @user + return render_forbidden unless user_can_push? + + yield + end + + def check_download_sendfile_header? + @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" + end + + def user_can_fetch? + # Check user access against the project they used to initiate the pull + @user.can?(:download_code, @origin_project) + end + + def user_can_push? + # Check user access against the project they used to initiate the push + @user.can?(:push_code, @origin_project) + end + + def storage_project(project) + if project.forked? + storage_project(project.forked_from_project) + else + project + end + end + + def store_file(oid, size, tmp_file) + tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file) + + object = LfsObject.find_or_create_by(oid: oid, size: size) + if object.file.exists? + success = true + else + success = move_tmp_file_to_storage(object, tmp_file_path) + end + + if success + success = link_to_project(object) + end + + success + ensure + # Ensure that the tmp file is removed + FileUtils.rm_f(tmp_file_path) + end + + def object_for_download(oid) + @project.lfs_objects.find_by(oid: oid) + end + + def move_tmp_file_to_storage(object, path) + File.open(path) do |f| + object.file = f + end + + object.file.store! + object.save + end + + def link_to_project(object) + if object && !object.projects.exists?(@project.id) + object.projects << @project + object.save + end + end + + def select_existing_objects(objects) + objects_oids = objects.map { |o| o['oid'] } + @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set + end + + def build_upload_batch_response(objects) + selected_objects = select_existing_objects(objects) + + upload_hypermedia_links(objects, selected_objects) + end + + def build_download_batch_response(objects) + selected_objects = select_existing_objects(objects) + + download_hypermedia_links(objects, selected_objects) + end + + def download_hypermedia_links(all_objects, existing_objects) + all_objects.each do |object| + if existing_objects.include?(object['oid']) + object['actions'] = { + 'download' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + else + object['error'] = { + 'code' => 404, + 'message' => "Object does not exist on the server or you don't have permissions to access it", + } + end + end + + { 'objects' => all_objects } + end + + def upload_hypermedia_links(all_objects, existing_objects) + all_objects.each do |object| + # generate actions only for non-existing objects + next if existing_objects.include?(object['oid']) + + object['actions'] = { + 'upload' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + end + + { 'objects' => all_objects } + end + end + end +end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb new file mode 100644 index 00000000000..78d02891102 --- /dev/null +++ b/lib/gitlab/lfs/router.rb @@ -0,0 +1,97 @@ +module Gitlab + module Lfs + class Router + def initialize(project, user, request) + @project = project + @user = user + @env = request.env + @request = request + end + + def try_call + return unless @request && @request.path.present? + + case @request.request_method + when 'GET' + get_response + when 'POST' + post_response + when 'PUT' + put_response + else + nil + end + end + + private + + def get_response + path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/) + return nil unless path_match + + oid = path_match[2] + return nil unless oid + + case path_match[1] + when "info/lfs" + lfs.render_unsupported_deprecated_api + when "gitlab-lfs" + lfs.render_download_object_response(oid) + else + nil + end + end + + def post_response + post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/) + return nil unless post_path + + # Check for Batch API + if post_path[0].ends_with?("/info/lfs/objects/batch") + lfs.render_batch_operation_response + elsif post_path[0].ends_with?("/info/lfs/objects") + lfs.render_unsupported_deprecated_api + else + nil + end + end + + def put_response + object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/) + return nil if object_match.nil? + + oid = object_match[1] + size = object_match[2].try(:to_i) + return nil if oid.nil? || size.nil? + + # GitLab-workhorse requests + # 1. Try to authorize the request + # 2. send a request with a header containing the name of the temporary file + if object_match[3] && object_match[3] == '/authorize' + lfs.render_storage_upload_authorize_response(oid, size) + else + tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) + return nil unless tmp_file_name + + lfs.render_storage_upload_store_response(oid, size, tmp_file_name) + end + end + + def lfs + return unless @project + + Gitlab::Lfs::Response.new(@project, @user, @request) + end + + def sanitize_tmp_filename(name) + if name.present? + name.gsub!(/^.*(\\|\/)/, '') + name = name.match(/[0-9a-f]{73}/) + name[0] if name + else + nil + end + end + end + end +end diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb deleted file mode 100644 index b082bfc434b..00000000000 --- a/lib/gitlab/markdown.rb +++ /dev/null @@ -1,200 +0,0 @@ -require 'html/pipeline' - -module Gitlab - # Custom parser for GitLab-flavored Markdown - # - # See the files in `lib/gitlab/markdown/` for specific processing information. - module Markdown - # Convert a Markdown String into an HTML-safe String of HTML - # - # Note that while the returned HTML will have been sanitized of dangerous - # HTML, it may post a risk of information leakage if it's not also passed - # through `post_process`. - # - # Also note that the returned String is always HTML, not XHTML. Views - # requiring XHTML, such as Atom feeds, need to call `post_process` on the - # result, providing the appropriate `pipeline` option. - # - # markdown - Markdown String - # context - Hash of context options passed to our HTML Pipeline - # - # Returns an HTML-safe String - def self.render(markdown, context = {}) - html = renderer.render(markdown) - html = gfm(html, context) - - html.html_safe - end - - # Convert a Markdown String into HTML without going through the HTML - # Pipeline. - # - # Note that because the pipeline is skipped, SanitizationFilter is as well. - # Do not output the result of this method to the user. - # - # markdown - Markdown String - # - # Returns a String - def self.render_without_gfm(markdown) - renderer.render(markdown) - end - - # Perform post-processing on an HTML String - # - # This method is used to perform state-dependent changes to a String of - # HTML, such as removing references that the current user doesn't have - # permission to make (`RedactorFilter`). - # - # html - String to process - # options - Hash of options to customize output - # :pipeline - Symbol pipeline type - # :project - Project - # :user - User object - # - # Returns an HTML-safe String - def self.post_process(html, options) - context = { - project: options[:project], - current_user: options[:user] - } - doc = post_processor.to_document(html, context) - - if options[:pipeline] == :atom - doc.to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML) - else - doc.to_html - end.html_safe - end - - # Provide autoload paths for filters to prevent a circular dependency error - autoload :AutolinkFilter, 'gitlab/markdown/autolink_filter' - autoload :CommitRangeReferenceFilter, 'gitlab/markdown/commit_range_reference_filter' - autoload :CommitReferenceFilter, 'gitlab/markdown/commit_reference_filter' - autoload :EmojiFilter, 'gitlab/markdown/emoji_filter' - autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/external_issue_reference_filter' - autoload :ExternalLinkFilter, 'gitlab/markdown/external_link_filter' - autoload :IssueReferenceFilter, 'gitlab/markdown/issue_reference_filter' - autoload :LabelReferenceFilter, 'gitlab/markdown/label_reference_filter' - autoload :MergeRequestReferenceFilter, 'gitlab/markdown/merge_request_reference_filter' - autoload :RedactorFilter, 'gitlab/markdown/redactor_filter' - autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' - autoload :SanitizationFilter, 'gitlab/markdown/sanitization_filter' - autoload :SnippetReferenceFilter, 'gitlab/markdown/snippet_reference_filter' - autoload :SyntaxHighlightFilter, 'gitlab/markdown/syntax_highlight_filter' - autoload :TableOfContentsFilter, 'gitlab/markdown/table_of_contents_filter' - autoload :TaskListFilter, 'gitlab/markdown/task_list_filter' - autoload :UserReferenceFilter, 'gitlab/markdown/user_reference_filter' - autoload :UploadLinkFilter, 'gitlab/markdown/upload_link_filter' - - # Public: Parse the provided HTML with GitLab-Flavored Markdown - # - # html - HTML String - # options - A Hash of options used to customize output (default: {}) - # :no_header_anchors - Disable header anchors in TableOfContentsFilter - # :path - Current path String - # :pipeline - Symbol pipeline type - # :project - Current Project object - # :project_wiki - Current ProjectWiki object - # :ref - Current ref String - # - # Returns an HTML-safe String - def self.gfm(html, options = {}) - return '' unless html.present? - - @pipeline ||= HTML::Pipeline.new(filters) - - context = { - # SanitizationFilter - pipeline: options[:pipeline], - - # EmojiFilter - asset_host: Gitlab::Application.config.asset_host, - asset_root: Gitlab.config.gitlab.base_url, - - # ReferenceFilter - only_path: only_path_pipeline?(options[:pipeline]), - project: options[:project], - - # RelativeLinkFilter - project_wiki: options[:project_wiki], - ref: options[:ref], - requested_path: options[:path], - - # TableOfContentsFilter - no_header_anchors: options[:no_header_anchors] - } - - @pipeline.to_html(html, context).html_safe - end - - private - - # Check if a pipeline enables the `only_path` context option - # - # Returns Boolean - def self.only_path_pipeline?(pipeline) - case pipeline - when :atom, :email - false - else - true - end - end - - def self.redcarpet_options - # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use - @redcarpet_options ||= { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - end - - def self.renderer - @markdown ||= begin - renderer = Redcarpet::Render::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) - end - end - - def self.post_processor - @post_processor ||= HTML::Pipeline.new([Gitlab::Markdown::RedactorFilter]) - end - - # Filters used in our pipeline - # - # SanitizationFilter should come first so that all generated reference HTML - # goes through untouched. - # - # See https://github.com/jch/html-pipeline#filters for more filters. - def self.filters - [ - Gitlab::Markdown::SyntaxHighlightFilter, - Gitlab::Markdown::SanitizationFilter, - - Gitlab::Markdown::UploadLinkFilter, - Gitlab::Markdown::RelativeLinkFilter, - Gitlab::Markdown::EmojiFilter, - Gitlab::Markdown::TableOfContentsFilter, - Gitlab::Markdown::AutolinkFilter, - Gitlab::Markdown::ExternalLinkFilter, - - Gitlab::Markdown::UserReferenceFilter, - Gitlab::Markdown::IssueReferenceFilter, - Gitlab::Markdown::ExternalIssueReferenceFilter, - Gitlab::Markdown::MergeRequestReferenceFilter, - Gitlab::Markdown::SnippetReferenceFilter, - Gitlab::Markdown::CommitRangeReferenceFilter, - Gitlab::Markdown::CommitReferenceFilter, - Gitlab::Markdown::LabelReferenceFilter, - - Gitlab::Markdown::TaskListFilter - ] - end - end -end diff --git a/lib/gitlab/markdown/autolink_filter.rb b/lib/gitlab/markdown/autolink_filter.rb deleted file mode 100644 index c37c3bc55bf..00000000000 --- a/lib/gitlab/markdown/autolink_filter.rb +++ /dev/null @@ -1,107 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' -require 'uri' - -module Gitlab - module Markdown - # HTML Filter for auto-linking URLs in HTML. - # - # Based on HTML::Pipeline::AutolinkFilter - # - # Context options: - # :autolink - Boolean, skips all processing done by this filter when false - # :link_attr - Hash of attributes for the generated links - # - class AutolinkFilter < HTML::Pipeline::Filter - include ActionView::Helpers::TagHelper - - # Pattern to match text that should be autolinked. - # - # A URI scheme begins with a letter and may contain letters, numbers, - # plus, period and hyphen. Schemes are case-insensitive but we're being - # picky here and allowing only lowercase for autolinks. - # - # See http://en.wikipedia.org/wiki/URI_scheme - # - # The negative lookbehind ensures that users can paste a URL followed by a - # period or comma for punctuation without those characters being included - # in the generated link. - # - # Rubular: http://rubular.com/r/cxjPyZc7Sb - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://\S+)(?<!,|\.)} - - # Text matching LINK_PATTERN inside these elements will not be linked - IGNORE_PARENTS = %w(a code kbd pre script style).to_set - - def call - return doc if context[:autolink] == false - - rinku_parse - text_parse - end - - private - - # Run the text through Rinku as a first pass - # - # This will quickly autolink http(s) and ftp links. - # - # `@doc` will be re-parsed with the HTML String from Rinku. - def rinku_parse - # Convert the options from a Hash to a String that Rinku expects - options = tag_options(link_options) - - # NOTE: We don't parse email links because it will erroneously match - # external Commit and CommitRange references. - # - # The final argument tells Rinku to link short URLs that don't include a - # period (e.g., http://localhost:3000/) - rinku = Rinku.auto_link(html, :urls, options, IGNORE_PARENTS.to_a, 1) - - # Rinku returns a String, so parse it back to a Nokogiri::XML::Document - # for further processing. - @doc = parse_html(rinku) - end - - # Autolinks any text matching LINK_PATTERN that Rinku didn't already - # replace - def text_parse - search_text_nodes(doc).each do |node| - content = node.to_html - - next if has_ancestor?(node, IGNORE_PARENTS) - next unless content.match(LINK_PATTERN) - - # If Rinku didn't link this, there's probably a good reason, so we'll - # skip it too - next if content.start_with?(*%w(http https ftp)) - - html = autolink_filter(content) - - next if html == content - - node.replace(html) - end - - doc - end - - def autolink_filter(text) - text.gsub(LINK_PATTERN) do |match| - # Remove any trailing HTML entities and store them for appending - # outside the link element. The entity must be marked HTML safe in - # order to be output literally rather than escaped. - match.gsub!(/((?:&[\w#]+;)+)\z/, '') - dropped = ($1 || '').html_safe - - options = link_options.merge(href: match) - content_tag(:a, match, options) + dropped - end - end - - def link_options - @link_options ||= context[:link_attr] || {} - end - end - end -end diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/commit_range_reference_filter.rb deleted file mode 100644 index e070edae0a4..00000000000 --- a/lib/gitlab/markdown/commit_range_reference_filter.rb +++ /dev/null @@ -1,92 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces commit range references with links. - # - # This filter supports cross-project references. - class CommitRangeReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find commit range references in text - # - # CommitRangeReferenceFilter.references_in(text) do |match, commit_range, project_ref| - # "<a href=...>#{commit_range}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the String commit range, and an optional String - # of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(CommitRange.reference_pattern) do |match| - yield match, $~[:commit_range], $~[:project] - end - end - - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit-range") - range = CommitRange.new(id, project) - - return unless range.valid_commits? - - { commit_range: range } - end - - def initialize(*args) - super - - @commit_map = {} - end - - def call - replace_text_nodes_matching(CommitRange.reference_pattern) do |content| - commit_range_link_filter(content) - end - end - - # Replace commit range references in text with links to compare the commit - # ranges. - # - # text - String text to replace references in. - # - # Returns a String with commit range references replaced with links. All - # links have `gfm` and `gfm-commit_range` class names attached for - # styling. - def commit_range_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - range = CommitRange.new(id, project) - - if range.valid_commits? - url = url_for_commit_range(project, range) - - title = range.reference_title - klass = reference_class(:commit_range) - data = data_attribute(project: project.id, commit_range: id) - - project_ref += '@' if project_ref - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{project_ref}#{range}</a>) - else - match - end - end - end - - def url_for_commit_range(project, range) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_compare_url(project.namespace, project, - range.to_param.merge(only_path: context[:only_path])) - end - end - end -end diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/commit_reference_filter.rb deleted file mode 100644 index 8cdbeb1f9cf..00000000000 --- a/lib/gitlab/markdown/commit_reference_filter.rb +++ /dev/null @@ -1,88 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces commit references with links. - # - # This filter supports cross-project references. - class CommitReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find commit references in text - # - # CommitReferenceFilter.references_in(text) do |match, commit, project_ref| - # "<a href=...>#{commit}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the String commit identifier, and an optional - # String of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Commit.reference_pattern) do |match| - yield match, $~[:commit], $~[:project] - end - end - - def self.referenced_by(node) - project = Project.find(node.attr("data-project")) rescue nil - return unless project - - id = node.attr("data-commit") - commit = commit_from_ref(project, id) - - return unless commit - - { commit: commit } - end - - def call - replace_text_nodes_matching(Commit.reference_pattern) do |content| - commit_link_filter(content) - end - end - - # Replace commit references in text with links to the commit specified. - # - # text - String text to replace references in. - # - # Returns a String with commit references replaced with links. All links - # have `gfm` and `gfm-commit` class names attached for styling. - def commit_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if commit = self.class.commit_from_ref(project, id) - url = url_for_commit(project, commit) - - title = escape_once(commit.link_title) - klass = reference_class(:commit) - data = data_attribute(project: project.id, commit: id) - - project_ref += '@' if project_ref - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{project_ref}#{commit.short_id}</a>) - else - match - end - end - end - - def self.commit_from_ref(project, id) - if project && project.valid_repo? - project.commit(id) - end - end - - def url_for_commit(project, commit) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_commit_url(project.namespace, project, commit, - only_path: context[:only_path]) - end - end - end -end diff --git a/lib/gitlab/markdown/cross_project_reference.rb b/lib/gitlab/markdown/cross_project_reference.rb deleted file mode 100644 index 6ab04a584b0..00000000000 --- a/lib/gitlab/markdown/cross_project_reference.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # Common methods for ReferenceFilters that support an optional cross-project - # reference. - module CrossProjectReference - # Given a cross-project reference string, get the Project record - # - # Defaults to value of `context[:project]` if: - # * No reference is given OR - # * Reference given doesn't exist - # - # ref - String reference. - # - # Returns a Project, or nil if the reference can't be found - def project_from_ref(ref) - return context[:project] unless ref - - Project.find_with_namespace(ref) - end - end - end -end diff --git a/lib/gitlab/markdown/emoji_filter.rb b/lib/gitlab/markdown/emoji_filter.rb deleted file mode 100644 index da10e4d3760..00000000000 --- a/lib/gitlab/markdown/emoji_filter.rb +++ /dev/null @@ -1,80 +0,0 @@ -require 'action_controller' -require 'gitlab/markdown' -require 'gitlab_emoji' -require 'html/pipeline/filter' - -module Gitlab - module Markdown - # HTML filter that replaces :emoji: with images. - # - # Based on HTML::Pipeline::EmojiFilter - # - # Context options: - # :asset_root - # :asset_host - class EmojiFilter < HTML::Pipeline::Filter - IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set - - def call - search_text_nodes(doc).each do |node| - content = node.to_html - next unless content.include?(':') - next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) - - html = emoji_image_filter(content) - - next if html == content - - node.replace(html) - end - - doc - end - - # Replace :emoji: with corresponding images. - # - # text - String text to replace :emoji: in. - # - # Returns a String with :emoji: replaced with images. - def emoji_image_filter(text) - text.gsub(emoji_pattern) do |match| - name = $1 - "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{emoji_url(name)}' height='20' width='20' align='absmiddle' />" - end - end - - private - - def emoji_url(name) - emoji_path = "emoji/#{emoji_filename(name)}" - if context[:asset_host] - # Asset host is specified. - url_to_image(emoji_path) - elsif context[:asset_root] - # Gitlab url is specified - File.join(context[:asset_root], url_to_image(emoji_path)) - else - # All other cases - url_to_image(emoji_path) - end - end - - def url_to_image(image) - ActionController::Base.helpers.url_to_image(image) - end - - # Build a regexp that matches all valid :emoji: names. - def self.emoji_pattern - @emoji_pattern ||= /:(#{Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ - end - - def emoji_pattern - self.class.emoji_pattern - end - - def emoji_filename(name) - "#{Emoji.emoji_filename(name)}.png" - end - end - end -end diff --git a/lib/gitlab/markdown/external_issue_reference_filter.rb b/lib/gitlab/markdown/external_issue_reference_filter.rb deleted file mode 100644 index 8f86f13976a..00000000000 --- a/lib/gitlab/markdown/external_issue_reference_filter.rb +++ /dev/null @@ -1,63 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces external issue tracker references with links. - # References are ignored if the project doesn't use an external issue - # tracker. - class ExternalIssueReferenceFilter < ReferenceFilter - # Public: Find `JIRA-123` issue references in text - # - # ExternalIssueReferenceFilter.references_in(text) do |match, issue| - # "<a href=...>##{issue}</a>" - # end - # - # text - String text to search. - # - # Yields the String match and the String issue reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(ExternalIssue.reference_pattern) do |match| - yield match, $~[:issue] - end - end - - def call - # Early return if the project isn't using an external tracker - return doc if project.nil? || project.default_issues_tracker? - - replace_text_nodes_matching(ExternalIssue.reference_pattern) do |content| - issue_link_filter(content) - end - end - - # Replace `JIRA-123` issue references in text with links to the referenced - # issue's details page. - # - # text - String text to replace references in. - # - # Returns a String with `JIRA-123` references replaced with links. All - # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text) - project = context[:project] - - self.class.references_in(text) do |match, issue| - url = url_for_issue(issue, project, only_path: context[:only_path]) - - title = escape_once("Issue in #{project.external_issue_tracker.title}") - klass = reference_class(:issue) - data = data_attribute(project: project.id) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - end - end - - def url_for_issue(*args) - IssuesHelper.url_for_issue(*args) - end - end - end -end diff --git a/lib/gitlab/markdown/external_link_filter.rb b/lib/gitlab/markdown/external_link_filter.rb deleted file mode 100644 index 29e51b6ade6..00000000000 --- a/lib/gitlab/markdown/external_link_filter.rb +++ /dev/null @@ -1,34 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' - -module Gitlab - module Markdown - # HTML Filter to add a `rel="nofollow"` attribute to external links - # - class ExternalLinkFilter < HTML::Pipeline::Filter - def call - doc.search('a').each do |node| - next unless node.has_attribute?('href') - - link = node.attribute('href').value - - # Skip non-HTTP(S) links - next unless link.start_with?('http') - - # Skip internal links - next if link.start_with?(internal_url) - - node.set_attribute('rel', 'nofollow') - end - - doc - end - - private - - def internal_url - @internal_url ||= Gitlab.config.gitlab.url - end - end - end -end diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb deleted file mode 100644 index 481d282f7b1..00000000000 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ /dev/null @@ -1,72 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces issue references with links. References to - # issues that do not exist are ignored. - # - # This filter supports cross-project references. - class IssueReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `#123` issue references in text - # - # IssueReferenceFilter.references_in(text) do |match, issue, project_ref| - # "<a href=...>##{issue}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer issue ID, and an optional String of - # the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Issue.reference_pattern) do |match| - yield match, $~[:issue].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { issue: LazyReference.new(Issue, node.attr("data-issue")) } - end - - def call - replace_text_nodes_matching(Issue.reference_pattern) do |content| - issue_link_filter(content) - end - end - - # Replace `#123` issue references in text with links to the referenced - # issue's details page. - # - # text - String text to replace references in. - # - # Returns a String with `#123` references replaced with links. All links - # have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && issue = project.get_issue(id) - url = url_for_issue(id, project, only_path: context[:only_path]) - - title = escape_once("Issue: #{issue.title}") - klass = reference_class(:issue) - data = data_attribute(project: project.id, issue: issue.id) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end - end - - def url_for_issue(*args) - IssuesHelper.url_for_issue(*args) - end - end - end -end diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/label_reference_filter.rb deleted file mode 100644 index 618acb7a578..00000000000 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ /dev/null @@ -1,87 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces label references with links. - class LabelReferenceFilter < ReferenceFilter - # Public: Find label references in text - # - # LabelReferenceFilter.references_in(text) do |match, id, name| - # "<a href=...>#{Label.find(id)}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, an optional Integer label ID, and an optional - # String label name. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Label.reference_pattern) do |match| - yield match, $~[:label_id].to_i, $~[:label_name] - end - end - - def self.referenced_by(node) - { label: LazyReference.new(Label, node.attr("data-label")) } - end - - def call - replace_text_nodes_matching(Label.reference_pattern) do |content| - label_link_filter(content) - end - end - - # Replace label references in text with links to the label specified. - # - # text - String text to replace references in. - # - # Returns a String with label references replaced with links. All links - # have `gfm` and `gfm-label` class names attached for styling. - def label_link_filter(text) - project = context[:project] - - self.class.references_in(text) do |match, id, name| - params = label_params(id, name) - - if label = project.labels.find_by(params) - url = url_for_label(project, label) - klass = reference_class(:label) - data = data_attribute(project: project.id, label: label.id) - - %(<a href="#{url}" #{data} - class="#{klass}">#{render_colored_label(label)}</a>) - else - match - end - end - end - - def url_for_label(project, label) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_issues_path(project.namespace, project, - label_name: label.name, - only_path: context[:only_path]) - end - - def render_colored_label(label) - LabelsHelper.render_colored_label(label) - end - - # Parameters to pass to `Label.find_by` based on the given arguments - # - # id - Integer ID to pass. If present, returns {id: id} - # name - String name to pass. If `id` is absent, finds by name without - # surrounding quotes. - # - # Returns a Hash. - def label_params(id, name) - if name - { name: name.tr('"', '') } - else - { id: id } - end - end - end - end -end diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb deleted file mode 100644 index 5bc63269808..00000000000 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ /dev/null @@ -1,74 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces merge request references with links. References - # to merge requests that do not exist are ignored. - # - # This filter supports cross-project references. - class MergeRequestReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `!123` merge request references in text - # - # MergeRequestReferenceFilter.references_in(text) do |match, merge_request, project_ref| - # "<a href=...>##{merge_request}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer merge request ID, and an optional - # String of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(MergeRequest.reference_pattern) do |match| - yield match, $~[:merge_request].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { merge_request: LazyReference.new(MergeRequest, node.attr("data-merge-request")) } - end - - def call - replace_text_nodes_matching(MergeRequest.reference_pattern) do |content| - merge_request_link_filter(content) - end - end - - # Replace `!123` merge request references in text with links to the - # referenced merge request's details page. - # - # text - String text to replace references in. - # - # Returns a String with `!123` references replaced with links. All links - # have `gfm` and `gfm-merge_request` class names attached for styling. - def merge_request_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && merge_request = project.merge_requests.find_by(iid: id) - title = escape_once("Merge Request: #{merge_request.title}") - klass = reference_class(:merge_request) - data = data_attribute(project: project.id, merge_request: merge_request.id) - - url = url_for_merge_request(merge_request, project) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end - end - - def url_for_merge_request(mr, project) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_merge_request_url(project.namespace, project, mr, - only_path: context[:only_path]) - end - end - end -end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb new file mode 100644 index 00000000000..8f3f43c0e91 --- /dev/null +++ b/lib/gitlab/markdown/pipeline.rb @@ -0,0 +1,34 @@ +require 'banzai' + +module Gitlab + module Markdown + class Pipeline + def self.[](name) + name ||= :full + const_get("#{name.to_s.camelize}Pipeline") + end + + def self.filters + [] + end + + def self.transform_context(context) + context + end + + def self.html_pipeline + @html_pipeline ||= HTML::Pipeline.new(filters) + end + + class << self + %i(call to_document to_html).each do |meth| + define_method(meth) do |text, context| + context = transform_context(context) + + html_pipeline.send(meth, text, context) + end + end + end + end + end +end diff --git a/lib/gitlab/markdown/redactor_filter.rb b/lib/gitlab/markdown/redactor_filter.rb deleted file mode 100644 index a1f3a8a8ebf..00000000000 --- a/lib/gitlab/markdown/redactor_filter.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' - -module Gitlab - module Markdown - # HTML filter that removes references to records that the current user does - # not have permission to view. - # - # Expected to be run in its own post-processing pipeline. - # - class RedactorFilter < HTML::Pipeline::Filter - def call - doc.css('a.gfm').each do |node| - unless user_can_reference?(node) - node.replace(node.text) - end - end - - doc - end - - private - - def user_can_reference?(node) - if node.has_attribute?('data-reference-filter') - reference_type = node.attr('data-reference-filter') - reference_filter = reference_type.constantize - - reference_filter.user_can_reference?(current_user, node, context) - else - true - end - end - - def current_user - context[:current_user] - end - end - end -end diff --git a/lib/gitlab/markdown/reference_filter.rb b/lib/gitlab/markdown/reference_filter.rb deleted file mode 100644 index a4c560f578c..00000000000 --- a/lib/gitlab/markdown/reference_filter.rb +++ /dev/null @@ -1,133 +0,0 @@ -require 'active_support/core_ext/string/output_safety' -require 'gitlab/markdown' -require 'html/pipeline/filter' - -module Gitlab - module Markdown - # Base class for GitLab Flavored Markdown reference filters. - # - # References within <pre>, <code>, <a>, and <style> elements are ignored. - # - # Context options: - # :project (required) - Current project, ignored if reference is cross-project. - # :only_path - Generate path-only links. - class ReferenceFilter < HTML::Pipeline::Filter - LazyReference = Struct.new(:klass, :ids) do - def self.load(refs) - lazy_references, values = refs.partition { |ref| ref.is_a?(self) } - - lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| - ids = refs.flat_map(&:ids) - klass.where(id: ids) - end - - values + lazy_values - end - - def load - self.klass.where(id: self.ids) - end - end - - def self.user_can_reference?(user, node, context) - if node.has_attribute?('data-project') - project_id = node.attr('data-project').to_i - return true if project_id == context[:project].try(:id) - - project = Project.find(project_id) rescue nil - Ability.abilities.allowed?(user, :read_project, project) - else - true - end - end - - def self.referenced_by(node) - raise NotImplementedError, "#{self} does not implement #{__method__}" - end - - # Returns a data attribute String to attach to a reference link - # - # attributes - Hash, where the key becomes the data attribute name and the - # value is the data attribute value - # - # Examples: - # - # data_attribute(project: 1, issue: 2) - # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" - # - # data_attribute(project: 3, merge_request: 4) - # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" - # - # Returns a String - def data_attribute(attributes = {}) - attributes[:reference_filter] = self.class.name - attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ") - end - - def escape_once(html) - ERB::Util.html_escape_once(html) - end - - def ignore_parents - @ignore_parents ||= begin - # Don't look for references in text nodes that are children of these - # elements. - parents = %w(pre code a style) - parents << 'blockquote' if context[:ignore_blockquotes] - parents.to_set - end - end - - def ignored_ancestry?(node) - has_ancestor?(node, ignore_parents) - end - - def project - context[:project] - end - - def reference_class(type) - "gfm gfm-#{type}" - end - - # Iterate through the document's text nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's content matches `pattern` AND - # * The node is not an ancestor of an ignored node type - # - # pattern - Regex pattern against which to match the node's content - # - # Yields the current node's String contents. The result of the block will - # replace the node's existing content and update the current document. - # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_text_nodes_matching(pattern) - return doc if project.nil? - - search_text_nodes(doc).each do |node| - next if ignored_ancestry?(node) - next unless node.text =~ pattern - - content = node.to_html - - html = yield content - - next if html == content - - node.replace(html) - end - - doc - end - - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project - end - end - end -end diff --git a/lib/gitlab/markdown/reference_gatherer_filter.rb b/lib/gitlab/markdown/reference_gatherer_filter.rb deleted file mode 100644 index 00f983675e6..00000000000 --- a/lib/gitlab/markdown/reference_gatherer_filter.rb +++ /dev/null @@ -1,63 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' - -module Gitlab - module Markdown - # HTML filter that gathers all referenced records that the current user has - # permission to view. - # - # Expected to be run in its own post-processing pipeline. - # - class ReferenceGathererFilter < HTML::Pipeline::Filter - def initialize(*) - super - - result[:references] ||= Hash.new { |hash, type| hash[type] = [] } - end - - def call - doc.css('a.gfm').each do |node| - gather_references(node) - end - - load_lazy_references unless context[:load_lazy_references] == false - - doc - end - - private - - def gather_references(node) - return unless node.has_attribute?('data-reference-filter') - - reference_type = node.attr('data-reference-filter') - reference_filter = reference_type.constantize - - return if context[:reference_filter] && reference_filter != context[:reference_filter] - - return unless reference_filter.user_can_reference?(current_user, node, context) - - references = reference_filter.referenced_by(node) - return unless references - - references.each do |type, values| - Array.wrap(values).each do |value| - result[:references][type] << value - end - end - end - - # Will load all references of one type using one query. - def load_lazy_references - refs = result[:references] - refs.each do |type, values| - refs[type] = ReferenceFilter::LazyReference.load(values) - end - end - - def current_user - context[:current_user] - end - end - end -end diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb deleted file mode 100644 index 6ee3d1ce039..00000000000 --- a/lib/gitlab/markdown/relative_link_filter.rb +++ /dev/null @@ -1,157 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' -require 'uri' - -module Gitlab - module Markdown - # HTML filter that "fixes" relative links to files in a repository. - # - # Context options: - # :commit - # :project - # :project_wiki - # :ref - # :requested_path - class RelativeLinkFilter < HTML::Pipeline::Filter - def call - return doc unless linkable_files? - - doc.search('a').each do |el| - process_link_attr el.attribute('href') - end - - doc.search('img').each do |el| - process_link_attr el.attribute('src') - end - - doc - end - - protected - - def linkable_files? - context[:project_wiki].nil? && repository.try(:exists?) && !repository.empty? - end - - def process_link_attr(html_attr) - return if html_attr.blank? - - uri = URI(html_attr.value) - if uri.relative? && uri.path.present? - html_attr.value = rebuild_relative_uri(uri).to_s - end - rescue URI::Error - # noop - end - - def rebuild_relative_uri(uri) - file_path = relative_file_path(uri.path) - - uri.path = [ - relative_url_root, - context[:project].path_with_namespace, - path_type(file_path), - ref || 'master', # assume that if no ref exists we can point to master - file_path - ].compact.join('/').squeeze('/').chomp('/') - - uri - end - - def relative_file_path(path) - nested_path = build_relative_path(path, context[:requested_path]) - file_exists?(nested_path) ? nested_path : path - end - - # Convert a relative path into its correct location based on the currently - # requested path - # - # path - Relative path String - # request_path - Currently-requested path String - # - # Examples: - # - # # File in the same directory as the current path - # build_relative_path("users.md", "doc/api/README.md") - # # => "doc/api/users.md" - # - # # File in the same directory, which is also the current path - # build_relative_path("users.md", "doc/api") - # # => "doc/api/users.md" - # - # # Going up one level to a different directory - # build_relative_path("../update/7.14-to-8.0.md", "doc/api/README.md") - # # => "doc/update/7.14-to-8.0.md" - # - # Returns a String - def build_relative_path(path, request_path) - return request_path if path.empty? - return path unless request_path - - parts = request_path.split('/') - parts.pop if path_type(request_path) != 'tree' - - while parts.length > 1 && path.start_with?('../') - parts.pop - path.sub!('../', '') - end - - parts.push(path).join('/') - end - - def file_exists?(path) - return false if path.nil? - repository.blob_at(current_sha, path).present? || - repository.tree(current_sha, path).entries.any? - end - - # Get the type of the given path - # - # path - String path to check - # - # Examples: - # - # path_type('doc/README.md') # => 'blob' - # path_type('doc/logo.png') # => 'raw' - # path_type('doc/api') # => 'tree' - # - # Returns a String - def path_type(path) - unescaped_path = Addressable::URI.unescape(path) - - if tree?(unescaped_path) - 'tree' - elsif image?(unescaped_path) - 'raw' - else - 'blob' - end - end - - def tree?(path) - repository.tree(current_sha, path).entries.any? - end - - def image?(path) - repository.blob_at(current_sha, path).try(:image?) - end - - def current_sha - context[:commit].try(:id) || - ref ? repository.commit(ref).try(:sha) : repository.head_commit.sha - end - - def relative_url_root - Gitlab.config.gitlab.relative_url_root.presence || '/' - end - - def ref - context[:ref] - end - - def repository - context[:project].try(:repository) - end - end - end -end diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb deleted file mode 100644 index e368de7d848..00000000000 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ /dev/null @@ -1,84 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' -require 'html/pipeline/sanitization_filter' - -module Gitlab - module Markdown - # Sanitize HTML - # - # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. - class SanitizationFilter < HTML::Pipeline::SanitizationFilter - def whitelist - # Descriptions are more heavily sanitized, allowing only a few elements. - # See http://git.io/vkuAN - if pipeline == :description - whitelist = LIMITED - whitelist[:elements] -= %w(pre code img ol ul li) - else - whitelist = super - end - - customize_whitelist(whitelist) - - whitelist - end - - private - - def pipeline - context[:pipeline] || :default - end - - def customized?(transformers) - transformers.last.source_location[0] == __FILE__ - end - - def customize_whitelist(whitelist) - # Only push these customizations once - return if customized?(whitelist[:transformers]) - - # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class) - whitelist[:attributes]['span'] = %w(class) - - # Allow table alignment - whitelist[:attributes]['th'] = %w(style) - whitelist[:attributes]['td'] = %w(style) - - # Allow span elements - whitelist[:elements].push('span') - - # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(remove_rel) - - # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(clean_spans) - - whitelist - end - - def remove_rel - lambda do |env| - if env[:node_name] == 'a' - env[:node].remove_attribute('rel') - end - end - end - - def clean_spans - lambda do |env| - node = env[:node] - - return unless node.name == 'span' - return unless node.has_attribute?('class') - - unless has_ancestor?(node, 'pre') - node.remove_attribute('class') - end - - { node_whitelist: [node] } - end - end - end - end -end diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb deleted file mode 100644 index f783f951711..00000000000 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ /dev/null @@ -1,74 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces snippet references with links. References to - # snippets that do not exist are ignored. - # - # This filter supports cross-project references. - class SnippetReferenceFilter < ReferenceFilter - include CrossProjectReference - - # Public: Find `$123` snippet references in text - # - # SnippetReferenceFilter.references_in(text) do |match, snippet| - # "<a href=...>$#{snippet}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer snippet ID, and an optional String - # of the external project reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Snippet.reference_pattern) do |match| - yield match, $~[:snippet].to_i, $~[:project] - end - end - - def self.referenced_by(node) - { snippet: LazyReference.new(Snippet, node.attr("data-snippet")) } - end - - def call - replace_text_nodes_matching(Snippet.reference_pattern) do |content| - snippet_link_filter(content) - end - end - - # Replace `$123` snippet references in text with links to the referenced - # snippets's details page. - # - # text - String text to replace references in. - # - # Returns a String with `$123` references replaced with links. All links - # have `gfm` and `gfm-snippet` class names attached for styling. - def snippet_link_filter(text) - self.class.references_in(text) do |match, id, project_ref| - project = self.project_from_ref(project_ref) - - if project && snippet = project.snippets.find_by(id: id) - title = escape_once("Snippet: #{snippet.title}") - klass = reference_class(:snippet) - data = data_attribute(project: project.id, snippet: snippet.id) - - url = url_for_snippet(snippet, project) - - %(<a href="#{url}" #{data} - title="#{title}" - class="#{klass}">#{match}</a>) - else - match - end - end - end - - def url_for_snippet(snippet, project) - h = Gitlab::Application.routes.url_helpers - h.namespace_project_snippet_url(project.namespace, project, snippet, - only_path: context[:only_path]) - end - end - end -end diff --git a/lib/gitlab/markdown/syntax_highlight_filter.rb b/lib/gitlab/markdown/syntax_highlight_filter.rb deleted file mode 100644 index 8597e02f0de..00000000000 --- a/lib/gitlab/markdown/syntax_highlight_filter.rb +++ /dev/null @@ -1,45 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' -require 'rouge/plugins/redcarpet' - -module Gitlab - module Markdown - # HTML Filter to highlight fenced code blocks - # - class SyntaxHighlightFilter < HTML::Pipeline::Filter - include Rouge::Plugins::Redcarpet - - def call - doc.search('pre > code').each do |node| - highlight_node(node) - end - - doc - end - - def highlight_node(node) - language = node.attr('class') - code = node.text - - begin - highlighted = block_code(code, language) - rescue - # Gracefully handle syntax highlighter bugs/errors to ensure - # users can still access an issue/comment/etc. - highlighted = "<pre>#{code}</pre>" - end - - # Replace the parent `pre` element with the entire highlighted block - node.parent.replace(highlighted) - end - - private - - # Override Rouge::Plugins::Redcarpet#rouge_formatter - def rouge_formatter(lexer) - Rouge::Formatters::HTMLGitlab.new( - cssclass: "code highlight js-syntax-highlight #{lexer.tag}") - end - end - end -end diff --git a/lib/gitlab/markdown/table_of_contents_filter.rb b/lib/gitlab/markdown/table_of_contents_filter.rb deleted file mode 100644 index bbb3bf7fc8b..00000000000 --- a/lib/gitlab/markdown/table_of_contents_filter.rb +++ /dev/null @@ -1,63 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' - -module Gitlab - module Markdown - # HTML filter that adds an anchor child element to all Headers in a - # document, so that they can be linked to. - # - # Generates the Table of Contents with links to each header. See Results. - # - # Based on HTML::Pipeline::TableOfContentsFilter. - # - # Context options: - # :no_header_anchors - Skips all processing done by this filter. - # - # Results: - # :toc - String containing Table of Contents data as a `ul` element with - # `li` child elements. - class TableOfContentsFilter < HTML::Pipeline::Filter - PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u - - def call - return doc if context[:no_header_anchors] - - result[:toc] = "" - - headers = Hash.new(0) - - doc.css('h1, h2, h3, h4, h5, h6').each do |node| - text = node.text - - id = text.downcase - id.gsub!(PUNCTUATION_REGEXP, '') # remove punctuation - id.gsub!(' ', '-') # replace spaces with dash - id.squeeze!('-') # replace multiple dashes with one - - uniq = (headers[id] > 0) ? "-#{headers[id]}" : '' - headers[id] += 1 - - if header_content = node.children.first - href = "#{id}#{uniq}" - push_toc(href, text) - header_content.add_previous_sibling(anchor_tag(href)) - end - end - - result[:toc] = %Q{<ul class="section-nav">\n#{result[:toc]}</ul>} unless result[:toc].empty? - - doc - end - - private - - def anchor_tag(href) - %Q{<a id="#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>} - end - - def push_toc(href, text) - result[:toc] << %Q{<li><a href="##{href}">#{text}</a></li>\n} - end - end - end -end diff --git a/lib/gitlab/markdown/task_list_filter.rb b/lib/gitlab/markdown/task_list_filter.rb deleted file mode 100644 index 2f133ae8500..00000000000 --- a/lib/gitlab/markdown/task_list_filter.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'gitlab/markdown' -require 'task_list/filter' - -module Gitlab - module Markdown - # Work around a bug in the default TaskList::Filter that adds a `task-list` - # class to every list element, regardless of whether or not it contains a - # task list. - # - # This is a (hopefully) temporary fix, pending a new release of the - # task_list gem. - # - # See https://github.com/github/task_list/pull/60 - class TaskListFilter < TaskList::Filter - def add_css_class(node, *new_class_names) - if new_class_names.include?('task-list') - super if node.children.any? { |c| c['class'] == 'task-list-item' } - else - super - end - end - end - end -end diff --git a/lib/gitlab/markdown/upload_link_filter.rb b/lib/gitlab/markdown/upload_link_filter.rb deleted file mode 100644 index fbada73ab86..00000000000 --- a/lib/gitlab/markdown/upload_link_filter.rb +++ /dev/null @@ -1,47 +0,0 @@ -require 'gitlab/markdown' -require 'html/pipeline/filter' -require 'uri' - -module Gitlab - module Markdown - # HTML filter that "fixes" relative upload links to files. - # Context options: - # :project (required) - Current project - # - class UploadLinkFilter < HTML::Pipeline::Filter - def call - doc.search('a').each do |el| - process_link_attr el.attribute('href') - end - - doc.search('img').each do |el| - process_link_attr el.attribute('src') - end - - doc - end - - protected - - def process_link_attr(html_attr) - return if html_attr.blank? - - uri = html_attr.value - if uri.starts_with?("/uploads/") - html_attr.value = build_url(uri).to_s - end - end - - def build_url(uri) - File.join(Gitlab.config.gitlab.url, context[:project].path_with_namespace, uri) - end - - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project - end - end - end -end diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb deleted file mode 100644 index 2a594e1662e..00000000000 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ /dev/null @@ -1,122 +0,0 @@ -require 'gitlab/markdown' - -module Gitlab - module Markdown - # HTML filter that replaces user or group references with links. - # - # A special `@all` reference is also supported. - class UserReferenceFilter < ReferenceFilter - # Public: Find `@user` user references in text - # - # UserReferenceFilter.references_in(text) do |match, username| - # "<a href=...>@#{user}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, and the String user name. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(User.reference_pattern) do |match| - yield match, $~[:user] - end - end - - def self.referenced_by(node) - if node.has_attribute?('data-group') - group = Group.find(node.attr('data-group')) rescue nil - return unless group - - { user: group.users } - elsif node.has_attribute?('data-user') - { user: LazyReference.new(User, node.attr('data-user')) } - elsif node.has_attribute?('data-project') - project = Project.find(node.attr('data-project')) rescue nil - return unless project - - { user: project.team.members.flatten } - end - end - - def self.user_can_reference?(user, node, context) - if node.has_attribute?('data-group') - group = Group.find(node.attr('data-group')) rescue nil - Ability.abilities.allowed?(user, :read_group, group) - else - super - end - end - - def call - replace_text_nodes_matching(User.reference_pattern) do |content| - user_link_filter(content) - end - end - - # Replace `@user` user references in text with links to the referenced - # user's profile page. - # - # text - String text to replace references in. - # - # Returns a String with `@user` references replaced with links. All links - # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text) - self.class.references_in(text) do |match, username| - if username == 'all' - link_to_all - elsif namespace = Namespace.find_by(path: username) - link_to_namespace(namespace) || match - else - match - end - end - end - - private - - def urls - Gitlab::Application.routes.url_helpers - end - - def link_class - reference_class(:project_member) - end - - def link_to_all - project = context[:project] - - url = urls.namespace_project_url(project.namespace, project, - only_path: context[:only_path]) - data = data_attribute(project: project.id) - - text = User.reference_prefix + 'all' - %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) - end - - def link_to_namespace(namespace) - if namespace.is_a?(Group) - link_to_group(namespace.path, namespace) - else - link_to_user(namespace.path, namespace) - end - end - - def link_to_group(group, namespace) - url = urls.group_url(group, only_path: context[:only_path]) - data = data_attribute(group: namespace.id) - - text = Group.reference_prefix + group - %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) - end - - def link_to_user(user, namespace) - url = urls.user_url(user, only_path: context[:only_path]) - data = data_attribute(user: namespace.owner_id) - - text = User.reference_prefix + user - %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) - end - end - end -end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb new file mode 100644 index 00000000000..ee88ab34d6c --- /dev/null +++ b/lib/gitlab/metrics.rb @@ -0,0 +1,102 @@ +module Gitlab + module Metrics + extend Gitlab::CurrentSettings + + RAILS_ROOT = Rails.root.to_s + METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s + PATH_REGEX = /^#{RAILS_ROOT}\/?/ + + def self.settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + username: current_application_settings[:metrics_username], + password: current_application_settings[:metrics_password], + port: current_application_settings[:metrics_port] + } + end + + def self.enabled? + settings[:enabled] || false + end + + def self.mri? + RUBY_ENGINE == 'ruby' + end + + def self.method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + @method_call_threshold ||= settings[:method_call_threshold] + end + + def self.pool + @pool + end + + # Returns a relative path and line number based on the last application call + # frame. + def self.last_relative_application_frame + frame = caller_locations.find do |l| + l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) + end + + if frame + return frame.path.sub(PATH_REGEX, ''), frame.lineno + else + return nil, nil + end + end + + def self.submit_metrics(metrics) + prepared = prepare_metrics(metrics) + + pool.with do |connection| + prepared.each do |metric| + begin + connection.write_points([metric]) + rescue StandardError + end + end + end + end + + def self.prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys + + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end + end + + new_hash + end + end + + def self.escape_value(value) + value.to_s.gsub('=', '\\=') + end + + # When enabled this should be set before being used as the usual pattern + # "@foo ||= bar" is _not_ thread-safe. + if enabled? + @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + user = settings[:username] + pw = settings[:password] + port = settings[:port] + + InfluxDB::Client. + new(udp: { host: host, port: port }, username: user, password: pw) + end + end + end +end diff --git a/lib/gitlab/metrics/delta.rb b/lib/gitlab/metrics/delta.rb new file mode 100644 index 00000000000..bcf28eed84d --- /dev/null +++ b/lib/gitlab/metrics/delta.rb @@ -0,0 +1,32 @@ +module Gitlab + module Metrics + # Class for calculating the difference between two numeric values. + # + # Every call to `compared_with` updates the internal value. This makes it + # possible to use a single Delta instance to calculate the delta over time + # of an ever increasing number. + # + # Example usage: + # + # delta = Delta.new(0) + # + # delta.compared_with(10) # => 10 + # delta.compared_with(15) # => 5 + # delta.compared_with(20) # => 5 + class Delta + def initialize(value = 0) + @value = value + end + + # new_value - The value to compare with as a Numeric. + # + # Returns a new Numeric (depending on the type of `new_value`). + def compared_with(new_value) + delta = new_value - @value + @value = new_value + + delta + end + end + end +end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb new file mode 100644 index 00000000000..d9fce2e6758 --- /dev/null +++ b/lib/gitlab/metrics/instrumentation.rb @@ -0,0 +1,148 @@ +module Gitlab + module Metrics + # Module for instrumenting methods. + # + # This module allows instrumenting of methods without having to actually + # alter the target code (e.g. by including modules). + # + # Example usage: + # + # Gitlab::Metrics::Instrumentation.instrument_method(User, :by_login) + module Instrumentation + SERIES = 'method_calls' + + def self.configure + yield self + end + + # Instruments a class method. + # + # mod - The module to instrument as a Module/Class. + # name - The name of the method to instrument. + def self.instrument_method(mod, name) + instrument(:class, mod, name) + end + + # Instruments an instance method. + # + # mod - The module to instrument as a Module/Class. + # name - The name of the method to instrument. + def self.instrument_instance_method(mod, name) + instrument(:instance, mod, name) + end + + # Recursively instruments all subclasses of the given root module. + # + # This can be used to for example instrument all ActiveRecord models (as + # these all inherit from ActiveRecord::Base). + # + # This method can optionally take a block to pass to `instrument_methods` + # and `instrument_instance_methods`. + # + # root - The root module for which to instrument subclasses. The root + # module itself is not instrumented. + def self.instrument_class_hierarchy(root, &block) + visit = root.subclasses + + until visit.empty? + klass = visit.pop + + instrument_methods(klass, &block) + instrument_instance_methods(klass, &block) + + klass.subclasses.each { |c| visit << c } + end + end + + # Instruments all public methods of a module. + # + # This method optionally takes a block that can be used to determine if a + # method should be instrumented or not. The block is passed the receiving + # module and an UnboundMethod. If the block returns a non truthy value the + # method is not instrumented. + # + # mod - The module to instrument. + def self.instrument_methods(mod) + mod.public_methods(false).each do |name| + method = mod.method(name) + + if method.owner == mod.singleton_class + if !block_given? || block_given? && yield(mod, method) + instrument_method(mod, name) + end + end + end + end + + # Instruments all public instance methods of a module. + # + # See `instrument_methods` for more information. + # + # mod - The module to instrument. + def self.instrument_instance_methods(mod) + mod.public_instance_methods(false).each do |name| + method = mod.instance_method(name) + + if method.owner == mod + if !block_given? || block_given? && yield(mod, method) + instrument_instance_method(mod, name) + end + end + end + end + + # Instruments a method. + # + # type - The type (:class or :instance) of method to instrument. + # mod - The module containing the method. + # name - The name of the method to instrument. + def self.instrument(type, mod, name) + return unless Metrics.enabled? + + name = name.to_sym + alias_name = :"_original_#{name}" + target = type == :instance ? mod : mod.singleton_class + + if type == :instance + target = mod + label = "#{mod.name}##{name}" + else + target = mod.singleton_class + label = "#{mod.name}.#{name}" + end + + target.class_eval <<-EOF, __FILE__, __LINE__ + 1 + alias_method #{alias_name.inspect}, #{name.inspect} + + def #{name}(*args, &block) + trans = Gitlab::Metrics::Instrumentation.transaction + + if trans + start = Time.now + retval = __send__(#{alias_name.inspect}, *args, &block) + duration = (Time.now - start) * 1000.0 + + if duration >= Gitlab::Metrics.method_call_threshold + trans.increment(:method_duration, duration) + + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, + { duration: duration }, + method: #{label.inspect}) + end + + retval + else + __send__(#{alias_name.inspect}, *args, &block) + end + end + EOF + end + + # Small layer of indirection to make it easier to stub out the current + # transaction. + def self.transaction + Transaction.current + end + end + end +end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb new file mode 100644 index 00000000000..7ea9555cc8c --- /dev/null +++ b/lib/gitlab/metrics/metric.rb @@ -0,0 +1,28 @@ +module Gitlab + module Metrics + # Class for storing details of a single metric (label, value, etc). + class Metric + attr_reader :series, :values, :tags, :created_at + + # series - The name of the series (as a String) to store the metric in. + # values - A Hash containing the values to store. + # tags - A Hash containing extra tags to add to the metrics. + def initialize(series, values, tags = {}) + @values = values + @series = series + @tags = tags + @created_at = Time.now.utc + end + + # Returns a Hash in a format that can be directly written to InfluxDB. + def to_hash + { + series: @series, + tags: @tags, + values: @values, + timestamp: @created_at.to_i * 1_000_000_000 + } + end + end + end +end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb new file mode 100644 index 00000000000..5c0587c4c51 --- /dev/null +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -0,0 +1,49 @@ +module Gitlab + module Metrics + # Rack middleware for tracking Rails requests. + class RackMiddleware + CONTROLLER_KEY = 'action_controller.instance' + + def initialize(app) + @app = app + end + + # env - A Hash containing Rack environment details. + def call(env) + trans = transaction_from_env(env) + retval = nil + + begin + retval = trans.run { @app.call(env) } + + # Even in the event of an error we want to submit any metrics we + # might've gathered up to this point. + ensure + if env[CONTROLLER_KEY] + tag_controller(trans, env) + end + + trans.finish + end + + retval + end + + def transaction_from_env(env) + trans = Transaction.new + + trans.add_tag(:request_method, env['REQUEST_METHOD']) + trans.add_tag(:request_uri, env['REQUEST_URI']) + + trans + end + + def tag_controller(trans, env) + controller = env[CONTROLLER_KEY] + label = "#{controller.class.name}##{controller.action_name}" + + trans.add_tag(:action, label) + end + end + end +end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb new file mode 100644 index 00000000000..1ea425bc904 --- /dev/null +++ b/lib/gitlab/metrics/sampler.rb @@ -0,0 +1,107 @@ +module Gitlab + module Metrics + # Class that sends certain metrics to InfluxDB at a specific interval. + # + # This class is used to gather statistics that can't be directly associated + # with a transaction such as system memory usage, garbage collection + # statistics, etc. + class Sampler + # interval - The sampling interval in seconds. + def initialize(interval = 15) + @interval = interval + @metrics = [] + + @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) + @last_major_gc = Delta.new(GC.stat[:major_gc_count]) + + if Gitlab::Metrics.mri? + require 'allocations' + + Allocations.start + end + end + + def start + Thread.new do + Thread.current.abort_on_exception = true + + loop do + sleep(@interval) + + sample + end + end + end + + def sample + sample_memory_usage + sample_file_descriptors + sample_objects + sample_gc + + flush + ensure + GC::Profiler.clear + @metrics.clear + end + + def flush + Metrics.submit_metrics(@metrics.map(&:to_hash)) + end + + def sample_memory_usage + add_metric('memory_usage', value: System.memory_usage) + end + + def sample_file_descriptors + add_metric('file_descriptors', value: System.file_descriptor_count) + end + + if Metrics.mri? + def sample_objects + sample = Allocations.to_hash + counts = sample.each_with_object({}) do |(klass, count), hash| + hash[klass.name] = count + end + + # Symbols aren't allocated so we'll need to add those manually. + counts['Symbol'] = Symbol.all_symbols.length + + counts.each do |name, count| + add_metric('object_counts', { count: count }, type: name) + end + end + else + def sample_objects + end + end + + def sample_gc + time = GC::Profiler.total_time * 1000.0 + stats = GC.stat.merge(total_time: time) + + # We want the difference of GC runs compared to the last sample, not the + # total amount since the process started. + stats[:minor_gc_count] = + @last_minor_gc.compared_with(stats[:minor_gc_count]) + + stats[:major_gc_count] = + @last_major_gc.compared_with(stats[:major_gc_count]) + + stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] + + add_metric('gc_statistics', stats) + end + + def add_metric(series, values, tags = {}) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def sidekiq? + Sidekiq.server? + end + end + end +end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb new file mode 100644 index 00000000000..ad441decfa2 --- /dev/null +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -0,0 +1,23 @@ +module Gitlab + module Metrics + # Sidekiq middleware for tracking jobs. + # + # This middleware is intended to be used as a server-side middleware. + class SidekiqMiddleware + def call(worker, message, queue) + trans = Transaction.new + + begin + trans.run { yield } + ensure + tag_worker(trans, worker) + trans.finish + end + end + + def tag_worker(trans, worker) + trans.add_tag(:action, "#{worker.class.name}#perform") + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb new file mode 100644 index 00000000000..7c0105d543a --- /dev/null +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -0,0 +1,54 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the rendering timings of views. + class ActionView < ActiveSupport::Subscriber + attach_to :action_view + + SERIES = 'views' + + def render_template(event) + track(event) if current_transaction + end + + alias_method :render_view, :render_template + + private + + def track(event) + values = values_for(event) + tags = tags_for(event) + + current_transaction.increment(:view_duration, event.duration) + current_transaction.add_metric(SERIES, values, tags) + end + + def relative_path(path) + path.gsub(/^#{Rails.root.to_s}\/?/, '') + end + + def values_for(event) + { duration: event.duration } + end + + def tags_for(event) + path = relative_path(event.payload[:identifier]) + tags = { view: path } + + file, line = Metrics.last_relative_application_frame + + if file and line + tags[:file] = file + tags[:line] = line + end + + tags + end + + def current_transaction + Transaction.current + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb new file mode 100644 index 00000000000..8008b3bc895 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -0,0 +1,22 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total query duration of a transaction. + class ActiveRecord < ActiveSupport::Subscriber + attach_to :active_record + + def sql(event) + return unless current_transaction + + current_transaction.increment(:sql_duration, event.duration) + end + + private + + def current_transaction + Transaction.current + end + end + end + end +end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb new file mode 100644 index 00000000000..83371265278 --- /dev/null +++ b/lib/gitlab/metrics/system.rb @@ -0,0 +1,35 @@ +module Gitlab + module Metrics + # Module for gathering system/process statistics such as the memory usage. + # + # This module relies on the /proc filesystem being available. If /proc is + # not available the methods of this module will be stubbed. + module System + if File.exist?('/proc') + # Returns the current process' memory usage in bytes. + def self.memory_usage + mem = 0 + match = File.read('/proc/self/status').match(/VmRSS:\s+(\d+)/) + + if match and match[1] + mem = match[1].to_f * 1024 + end + + mem + end + + def self.file_descriptor_count + Dir.glob('/proc/self/fd/*').length + end + else + def self.memory_usage + 0.0 + end + + def self.file_descriptor_count + 0 + end + end + end + end +end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb new file mode 100644 index 00000000000..68b86de0655 --- /dev/null +++ b/lib/gitlab/metrics/transaction.rb @@ -0,0 +1,79 @@ +module Gitlab + module Metrics + # Class for storing metrics information of a single transaction. + class Transaction + THREAD_KEY = :_gitlab_metrics_transaction + + attr_reader :uuid, :tags + + def self.current + Thread.current[THREAD_KEY] + end + + def initialize + @metrics = [] + @uuid = SecureRandom.uuid + + @started_at = nil + @finished_at = nil + + @values = Hash.new(0) + @tags = {} + end + + def duration + @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 + end + + def run + Thread.current[THREAD_KEY] = self + + @started_at = Time.now + + yield + ensure + @finished_at = Time.now + + Thread.current[THREAD_KEY] = nil + end + + def add_metric(series, values, tags = {}) + tags = tags.merge(transaction_id: @uuid) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def increment(name, value) + @values[name] += value + end + + def add_tag(key, value) + @tags[key] = value + end + + def finish + track_self + submit + end + + def track_self + values = { duration: duration } + + @values.each do |name, value| + values[name] = value + end + + add_metric('transactions', values, @tags) + end + + def submit + Metrics.submit_metrics(@metrics.map(&:to_hash)) + end + + def sidekiq? + Sidekiq.server? + end + end + end +end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index d94b104bbf8..ba31599432b 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -62,7 +62,7 @@ module Gitlab # Get the first part of the email address (before @) # In addtion in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].parameterize + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s end def generate_temporarily_email(username) diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 90c3fe8da33..9ad7a38d505 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -1,6 +1,12 @@ module Gitlab module OAuth class Provider + LABELS = { + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google" + }.freeze + def self.providers Devise.omniauth_providers end @@ -23,8 +29,9 @@ module Gitlab end def self.label_for(name) + name = name.to_s config = config_for(name) - (config && config['label']) || name.to_s.titleize + (config && config['label']) || LABELS[name] || name.titleize end end end diff --git a/lib/gitlab/o_auth/session.rb b/lib/gitlab/o_auth/session.rb new file mode 100644 index 00000000000..f33bfd0bd0e --- /dev/null +++ b/lib/gitlab/o_auth/session.rb @@ -0,0 +1,17 @@ +module Gitlab + module OAuth + module Session + def self.create(provider, ticket) + Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration) + end + + def self.destroy(provider, ticket) + Rails.cache.delete("gitlab:#{provider}:#{ticket}") + end + + def self.valid?(provider, ticket) + Rails.cache.read("gitlab:#{provider}:#{ticket}").present? + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 17ce4d4b174..f1a362f5303 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -64,7 +64,7 @@ module Gitlab # If a corresponding person exists with same uid in a LDAP server, # set up a Gitlab user with dual LDAP and Omniauth identities. - if user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn.downcase, ldap_person.provider) + if user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) # Case when a LDAP user already exists in Gitlab. Add the Omniauth identity to existing account. user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider) else diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0dab7bcfa4d..70de6a74e76 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -9,7 +9,7 @@ module Gitlab else nil end - @query = Shellwords.shellescape(query) if query.present? + @query = query end def objects(scope, page = nil) @@ -20,6 +20,8 @@ module Gitlab Kaminari.paginate_array(blobs).page(page).per(per_page) when 'wiki_blobs' Kaminari.paginate_array(wiki_blobs).page(page).per(per_page) + when 'commits' + Kaminari.paginate_array(commits).page(page).per(per_page) else super end @@ -27,7 +29,7 @@ module Gitlab def total_count @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + notes_count + wiki_blobs_count + commits_count end def blobs_count @@ -42,6 +44,10 @@ module Gitlab @wiki_blobs_count ||= wiki_blobs.count end + def commits_count + @commits_count ||= commits.count + end + private def blobs @@ -70,6 +76,14 @@ module Gitlab Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC') end + def commits + if project.empty_repo? || query.blank? + [] + else + project.repository.find_commits_by_message(query).compact + end + end + def limit_project_ids [project.id] end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index d010ade704e..4f9cdef3869 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -30,9 +30,12 @@ module Gitlab # For performance purposes maximum 20 latest commits # will be passed as post receive hook data. - commit_attrs = commits_limited.map(&:hook_attrs) + commit_attrs = commits_limited.map do |commit| + commit.hook_attrs(with_changed_files: true) + end type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" + # Hash to be passed as post_receive_data data = { object_kind: type, diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb new file mode 100644 index 00000000000..70e7f25d518 --- /dev/null +++ b/lib/gitlab/recaptcha.rb @@ -0,0 +1,14 @@ +module Gitlab + module Recaptcha + def self.load_configurations! + if current_application_settings.recaptcha_enabled + ::Recaptcha.configure do |config| + config.public_key = current_application_settings.recaptcha_site_key + config.private_key = current_application_settings.recaptcha_private_key + end + + true + end + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index da8df8a3025..be795649e59 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,73 +1,42 @@ -require 'gitlab/markdown' +require 'banzai' module Gitlab # Extract possible GFM references from an arbitrary String for further processing. - class ReferenceExtractor - attr_accessor :project, :current_user, :load_lazy_references + class ReferenceExtractor < Banzai::ReferenceExtractor + attr_accessor :project, :current_user, :author - def initialize(project, current_user = nil, load_lazy_references: true) + def initialize(project, current_user = nil, author = nil) @project = project @current_user = current_user - @load_lazy_references = load_lazy_references + @author = author + + @references = {} + + super() end - def analyze(text) - references.clear - @text = Gitlab::Markdown.render_without_gfm(text) + def analyze(text, context = {}) + super(text, context.merge(project: project)) end - %i(user label issue merge_request snippet commit commit_range).each do |type| + %i(user label merge_request snippet commit commit_range).each do |type| define_method("#{type}s") do - references[type] + @references[type] ||= references(type, reference_context) end end - private - - def references - @references ||= Hash.new do |references, type| - type = type.to_sym - next references[type] if references.has_key?(type) - - references[type] = pipeline_result(type) + def issues + if project && project.jira_tracker? + @references[:external_issue] ||= references(:external_issue, reference_context) + else + @references[:issue] ||= references(:issue, reference_context) end end - # Instantiate and call HTML::Pipeline with a single reference filter type, - # returning the result - # - # filter_type - Symbol reference type (e.g., :commit, :issue, etc.) - # - # Returns the results Array for the requested filter type - def pipeline_result(filter_type) - return [] if @text.blank? - - klass = "#{filter_type.to_s.camelize}ReferenceFilter" - filter = Gitlab::Markdown.const_get(klass) - - context = { - project: project, - current_user: current_user, - - # We don't actually care about the links generated - only_path: true, - ignore_blockquotes: true, - - # ReferenceGathererFilter - load_lazy_references: false, - reference_filter: filter - } - - pipeline = HTML::Pipeline.new([filter, Gitlab::Markdown::ReferenceGathererFilter], context) - result = pipeline.call(@text) - - values = result[:references][filter_type].uniq - - if @load_lazy_references - values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq - end + private - values + def reference_context + { project: project, current_user: current_user, author: author } end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 9f1adc860d1..53ab2686b43 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -51,6 +51,23 @@ module Gitlab "can contain only letters, digits, '_', '-' and '.'. " end + def file_path_regex + @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze + end + + def file_path_regex_message + "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. " + end + + + def directory_traversal_regex + @directory_traversal_regex ||= /\.{2}/.freeze + end + + def directory_traversal_regex_message + "cannot include directory traversal. " + end + def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 31aa3528c4c..2ef0e982256 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -14,7 +14,7 @@ module Gitlab def self.mute_mailer code = <<-eos -def Notify.delay +def Notify.deliver_later self end eos diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb new file mode 100644 index 00000000000..6360527a7aa --- /dev/null +++ b/lib/gitlab/sherlock.rb @@ -0,0 +1,19 @@ +require 'securerandom' + +module Gitlab + module Sherlock + @collection = Collection.new + + class << self + attr_reader :collection + end + + def self.enabled? + Rails.env.development? && !!ENV['ENABLE_SHERLOCK'] + end + + def self.enable_line_profiler? + RUBY_ENGINE == 'ruby' + end + end +end diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb new file mode 100644 index 00000000000..66bd6258521 --- /dev/null +++ b/lib/gitlab/sherlock/collection.rb @@ -0,0 +1,49 @@ +module Gitlab + module Sherlock + # A collection of transactions recorded by Sherlock. + # + # Method calls for this class are synchronized using a mutex to allow + # sharing of a single Collection instance between threads (e.g. when using + # Puma as a webserver). + class Collection + include Enumerable + + def initialize + @transactions = [] + @mutex = Mutex.new + end + + def add(transaction) + synchronize { @transactions << transaction } + end + + alias_method :<<, :add + + def each(&block) + synchronize { @transactions.each(&block) } + end + + def clear + synchronize { @transactions.clear } + end + + def empty? + synchronize { @transactions.empty? } + end + + def find_transaction(id) + find { |trans| trans.id == id } + end + + def newest_first + sort { |a, b| b.finished_at <=> a.finished_at } + end + + private + + def synchronize(&block) + @mutex.synchronize(&block) + end + end + end +end diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb new file mode 100644 index 00000000000..8a3e1a5e5bf --- /dev/null +++ b/lib/gitlab/sherlock/file_sample.rb @@ -0,0 +1,31 @@ +module Gitlab + module Sherlock + class FileSample + attr_reader :id, :file, :line_samples, :events, :duration + + # file - The full path to the file this sample belongs to. + # line_samples - An array of LineSample objects. + # duration - The total execution time in milliseconds. + # events - The total amount of events. + def initialize(file, line_samples, duration, events) + @id = SecureRandom.uuid + @file = file + @line_samples = line_samples + @duration = duration + @events = events + end + + def relative_path + @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + end + + def to_param + @id + end + + def source + @source ||= File.read(@file) + end + end + end +end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb new file mode 100644 index 00000000000..aa1468bff6b --- /dev/null +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -0,0 +1,98 @@ +module Gitlab + module Sherlock + # Class for profiling code on a per line basis. + # + # The LineProfiler class can be used to profile code on per line basis + # without littering your code with Ruby implementation specific profiling + # methods. + # + # This profiler only includes samples taking longer than a given threshold + # and those that occur in the actual application (e.g. files from Gems are + # ignored). + class LineProfiler + # The minimum amount of time that has to be spent in a file for it to be + # included in a list of samples. + MINIMUM_DURATION = 10.0 + + # Profiles the given block. + # + # Example: + # + # profiler = LineProfiler.new + # + # retval, samples = profiler.profile do + # "cats are amazing" + # end + # + # retval # => "cats are amazing" + # samples # => [#<Gitlab::Sherlock::FileSample ...>, ...] + # + # Returns an Array containing the block's return value and an Array of + # FileSample objects. + def profile(&block) + if mri? + profile_mri(&block) + else + raise NotImplementedError, + 'Line profiling is not supported on this platform' + end + end + + # Profiles the given block using rblineprof (MRI only). + def profile_mri + require 'rblineprof' + + retval = nil + samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield } + + file_samples = aggregate_rblineprof(samples) + + [retval, file_samples] + end + + # Returns an Array of file samples based on the output of rblineprof. + # + # lineprof_stats - A Hash containing rblineprof statistics on a per file + # basis. + # + # Returns an Array of FileSample objects. + def aggregate_rblineprof(lineprof_stats) + samples = [] + + lineprof_stats.each do |(file, stats)| + source_lines = File.read(file).each_line.to_a + line_samples = [] + + total_duration = microsec_to_millisec(stats[0][0]) + total_events = stats[0][2] + + next if total_duration <= MINIMUM_DURATION + + stats[1..-1].each_with_index do |data, index| + next unless source_lines[index] + + duration = microsec_to_millisec(data[0]) + events = data[2] + + line_samples << LineSample.new(duration, events) + end + + samples << FileSample. + new(file, line_samples, total_duration, total_events) + end + + samples + end + + private + + def microsec_to_millisec(microsec) + microsec / 1000.0 + end + + def mri? + RUBY_ENGINE == 'ruby' + end + end + end +end diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb new file mode 100644 index 00000000000..eb1948eb6d6 --- /dev/null +++ b/lib/gitlab/sherlock/line_sample.rb @@ -0,0 +1,36 @@ +module Gitlab + module Sherlock + class LineSample + attr_reader :duration, :events + + # duration - The execution time in milliseconds. + # events - The amount of events. + def initialize(duration, events) + @duration = duration + @events = events + end + + # Returns the sample duration percentage relative to the given duration. + # + # Example: + # + # sample.duration # => 150 + # sample.percentage_of(1500) # => 10.0 + # + # total_duration - The total duration to compare with. + # + # Returns a float + def percentage_of(total_duration) + (duration.to_f / total_duration) * 100.0 + end + + # Returns true if the current sample takes up the majority of the given + # duration. + # + # total_duration - The total duration to compare with. + def majority_of?(total_duration) + percentage_of(total_duration) >= 30 + end + end + end +end diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb new file mode 100644 index 00000000000..5ac265618ad --- /dev/null +++ b/lib/gitlab/sherlock/location.rb @@ -0,0 +1,26 @@ +module Gitlab + module Sherlock + class Location + attr_reader :path, :line + + SHERLOCK_DIR = File.dirname(__FILE__) + + # Creates a new Location from a `Thread::Backtrace::Location`. + def self.from_ruby_location(location) + new(location.path, location.lineno) + end + + # path - The full path of the frame as a String. + # line - The line number of the frame as a Fixnum. + def initialize(path, line) + @path = path + @line = line + end + + # Returns true if the current frame originated from the application. + def application? + @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR) + end + end + end +end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb new file mode 100644 index 00000000000..687332fc5fc --- /dev/null +++ b/lib/gitlab/sherlock/middleware.rb @@ -0,0 +1,41 @@ +module Gitlab + module Sherlock + # Rack middleware used for tracking request metrics. + class Middleware + CONTENT_TYPES = /text\/html|application\/json/i + + IGNORE_PATHS = %r{^/sherlock} + + def initialize(app) + @app = app + end + + # env - A Hash containing Rack environment details. + def call(env) + if instrument?(env) + call_with_instrumentation(env) + else + @app.call(env) + end + end + + def call_with_instrumentation(env) + trans = transaction_from_env(env) + retval = trans.run { @app.call(env) } + + Sherlock.collection.add(trans) + + retval + end + + def instrument?(env) + !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES && + env['REQUEST_URI'] !~ IGNORE_PATHS) + end + + def transaction_from_env(env) + Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI']) + end + end + end +end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb new file mode 100644 index 00000000000..4917c4ae2ac --- /dev/null +++ b/lib/gitlab/sherlock/query.rb @@ -0,0 +1,114 @@ +module Gitlab + module Sherlock + class Query + attr_reader :id, :query, :started_at, :finished_at, :backtrace + + # SQL identifiers that should be prefixed with newlines. + PREFIX_NEWLINE = / + \s+(FROM + |(LEFT|RIGHT)?INNER\s+JOIN + |(LEFT|RIGHT)?OUTER\s+JOIN + |WHERE + |AND + |GROUP\s+BY + |ORDER\s+BY + |LIMIT + |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + + # Creates a new Query using a String and a separate Array of bindings. + # + # query - A String containing a SQL query, optionally with numeric + # placeholders (`$1`, `$2`, etc). + # + # bindings - An Array of ActiveRecord columns and their values. + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + # + # Returns a new Query object. + def self.new_with_bindings(query, bindings, started_at, finished_at) + bindings.each_with_index do |(_, value), index| + quoted_value = ActiveRecord::Base.connection.quote(value) + + query = query.gsub("$#{index + 1}", quoted_value) + end + + new(query, started_at, finished_at) + end + + # query - The SQL query as a String (without placeholders). + # started_at - The start time of the query as a Time-like object. + # finished_at - The completion time of the query as a Time-like object. + def initialize(query, started_at, finished_at) + @id = SecureRandom.uuid + @query = query + @started_at = started_at + @finished_at = finished_at + @backtrace = caller_locations.map do |loc| + Location.from_ruby_location(loc) + end + + unless @query.end_with?(';') + @query += ';' + end + end + + # Returns the query duration in milliseconds. + def duration + @duration ||= (@finished_at - @started_at) * 1000.0 + end + + def to_param + @id + end + + # Returns a human readable version of the query. + def formatted_query + @formatted_query ||= format_sql(@query) + end + + # Returns the last application frame of the backtrace. + def last_application_frame + @last_application_frame ||= @backtrace.find(&:application?) + end + + # Returns an Array of application frames (excluding Gems and the likes). + def application_backtrace + @application_backtrace ||= @backtrace.select(&:application?) + end + + # Returns the query plan as a String. + def explain + unless @explain + ActiveRecord::Base.connection.transaction do + @explain = raw_explain(@query).values.flatten.join("\n") + + # Roll back any queries that mutate data so we don't mess up + # anything when running explain on an INSERT, UPDATE, DELETE, etc. + raise ActiveRecord::Rollback + end + end + + @explain + end + + private + + def raw_explain(query) + if Gitlab::Database.postgresql? + explain = "EXPLAIN ANALYZE #{query};" + else + explain = "EXPLAIN #{query};" + end + + ActiveRecord::Base.connection.execute(explain) + end + + def format_sql(query) + query.each_line. + map { |line| line.strip }. + join("\n"). + gsub(PREFIX_NEWLINE) { "\n#{$1} " } + end + end + end +end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb new file mode 100644 index 00000000000..3489fb251b6 --- /dev/null +++ b/lib/gitlab/sherlock/transaction.rb @@ -0,0 +1,136 @@ +module Gitlab + module Sherlock + class Transaction + attr_reader :id, :type, :path, :queries, :file_samples, :started_at, + :finished_at, :view_counts + + # type - The type of transaction (e.g. "GET", "POST", etc) + # path - The path of the transaction (e.g. the HTTP request path) + def initialize(type, path) + @id = SecureRandom.uuid + @type = type + @path = path + @queries = [] + @file_samples = [] + @started_at = nil + @finished_at = nil + @thread = Thread.current + @view_counts = Hash.new(0) + end + + # Runs the transaction and returns the block's return value. + def run + @started_at = Time.now + + retval = with_subscriptions do + profile_lines { yield } + end + + @finished_at = Time.now + + retval + end + + # Returns the duration in seconds. + def duration + @duration ||= started_at && finished_at ? finished_at - started_at : 0 + end + + # Returns the total query duration in seconds. + def query_duration + @query_duration ||= @queries.map { |q| q.duration }.inject(:+) / 1000.0 + end + + def to_param + @id + end + + # Returns the queries sorted in descending order by their durations. + def sorted_queries + @queries.sort { |a, b| b.duration <=> a.duration } + end + + # Returns the file samples sorted in descending order by their durations. + def sorted_file_samples + @file_samples.sort { |a, b| b.duration <=> a.duration } + end + + # Finds a query by the given ID. + # + # id - The query ID as a String. + # + # Returns a Query object if one could be found, nil otherwise. + def find_query(id) + @queries.find { |query| query.id == id } + end + + # Finds a file sample by the given ID. + # + # id - The query ID as a String. + # + # Returns a FileSample object if one could be found, nil otherwise. + def find_file_sample(id) + @file_samples.find { |sample| sample.id == id } + end + + def profile_lines + retval = nil + + if Sherlock.enable_line_profiler? + retval, @file_samples = LineProfiler.new.profile { yield } + else + retval = yield + end + + retval + end + + def subscribe_to_active_record + ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| + next unless same_thread? + + track_query(data[:sql].strip, data[:binds], start, finish) + end + end + + def subscribe_to_action_view + regex = /render_(template|partial)\.action_view/ + + ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data| + next unless same_thread? + + track_view(data[:identifier]) + end + end + + private + + def track_query(query, bindings, start, finish) + @queries << Query.new_with_bindings(query, bindings, start, finish) + end + + def track_view(path) + @view_counts[path] += 1 + end + + def with_subscriptions + ar_subscriber = subscribe_to_active_record + av_subscriber = subscribe_to_action_view + + retval = yield + + ActiveSupport::Notifications.unsubscribe(ar_subscriber) + ActiveSupport::Notifications.unsubscribe(av_subscriber) + + retval + end + + # In case somebody uses a multi-threaded server locally (e.g. Puma) we + # _only_ want to track notifications that originate from the transaction + # thread. + def same_thread? + Thread.current == @thread + end + end + end +end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb new file mode 100644 index 00000000000..1cd89b3a9c4 --- /dev/null +++ b/lib/gitlab/sql/union.rb @@ -0,0 +1,34 @@ +module Gitlab + module SQL + # Class for building SQL UNION statements. + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # union = Gitlab::SQL::Union.new(user.personal_projects, user.projects) + # sql = union.to_sql + # + # Project.where("id IN (#{sql})") + class Union + def initialize(relations) + @relations = relations + end + + def to_sql + # Some relations may include placeholders for prepared statements, these + # aren't incremented properly when joining relations together this way. + # By using "unprepared_statements" we remove the usage of placeholders + # (thus fixing this problem), at a slight performance cost. + fragments = ActiveRecord::Base.connection.unprepared_statement do + @relations.map do |rel| + rel.reorder(nil).to_sql + end + end + + fragments.join("\nUNION\n") + end + end + end +end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index cf040971c6e..f3567f3ef85 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -50,15 +50,15 @@ module Gitlab end def fetch_git_tags - remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) + remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) remote_tags.split("\n").grep(/tags\/v#{current_version.major}/) end def update_commands { - "Stash changed files" => %W(git stash), - "Get latest code" => %W(git fetch), - "Switch to new version" => %W(git checkout v#{latest_version}), + "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), + "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), + "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), "Install gems" => %W(bundle), "Migrate DB" => %W(bundle exec rake db:migrate), "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 335dc44be19..3160a3c7582 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -51,6 +51,15 @@ module Gitlab def allowed_fork_levels(origin_level) [PRIVATE, INTERNAL, PUBLIC].select{ |level| level <= origin_level } end + + def level_name(level) + level_name = 'Unknown' + options.each do |name, lvl| + level_name = name if lvl == level.to_i + end + + level_name + end end def private? |