diff options
Diffstat (limited to 'lib/gitlab')
26 files changed, 189 insertions, 39 deletions
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 12292f614e9..dc87aa52a3e 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -26,12 +26,8 @@ module Grack auth! if project && authorized_request? - if ENV['GITLAB_GRACK_AUTH_ONLY'] == '1' - # Tell gitlab-git-http-server the request is OK, and what the GL_ID is - render_grack_auth_ok - else - @app.call(env) - end + # Tell gitlab-git-http-server the request is OK, and what the GL_ID is + render_grack_auth_ok elsif @user.nil? && !@gitlab_ci unauthorized else @@ -132,7 +128,9 @@ module Grack case git_cmd when *Gitlab::GitAccess::DOWNLOAD_COMMANDS - if user + if !Gitlab.config.gitlab_shell.upload_pack + false + elsif user Gitlab::GitAccess.new(user, project).download_access_check.allowed? elsif project.public? # Allow clone/fetch for public projects @@ -141,7 +139,9 @@ module Grack false end when *Gitlab::GitAccess::PUSH_COMMANDS - if user + if !Gitlab.config.gitlab_shell.receive_pack + false + elsif user # Skip user authorization on upload request. # It will be done by the pre-receive hook in the repository. true diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb index aec44b8c87b..d88a6eaac6b 100644 --- a/lib/gitlab/bitbucket_import/client.rb +++ b/lib/gitlab/bitbucket_import/client.rb @@ -52,11 +52,26 @@ module Gitlab end def issues(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues").body) + all_issues = [] + offset = 0 + per_page = 50 # Maximum number allowed by Bitbucket + index = 0 + + begin + issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) + # Find out how many total issues are present + total = issues["count"] if index == 0 + all_issues.concat(issues["issues"]) + offset += issues["issues"].count + index += 1 + end while all_issues.count < total + + all_issues end def issue_comments(project_identifier, issue_id) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) + comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) + comments.sort_by { |comment| comment["utc_created_on"] } end def project(project_identifier) @@ -100,6 +115,10 @@ module Gitlab response end + def issue_api_endpoint(project_identifier, per_page, offset) + "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" + end + def config Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"} end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index d8a7d29f1bf..2355b3c6ddc 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -20,8 +20,18 @@ module Gitlab #Issues && Comments issues = client.issues(project_identifier) - issues["issues"].each do |issue| - body = @formatter.author_line(issue["reported_by"]["username"], issue["content"]) + issues.each do |issue| + body = '' + reporter = nil + author = 'Anonymous' + + if issue["reported_by"] && issue["reported_by"]["username"] + reporter = issue["reported_by"]["username"] + author = reporter + end + + body = @formatter.author_line(author) + body += issue["content"] comments = client.issue_comments(project_identifier, issue["local_id"]) @@ -30,14 +40,20 @@ module Gitlab end comments.each do |comment| - body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"]) + author = 'Anonymous' + + if comment["author_info"] && comment["author_info"]["username"] + author = comment["author_info"]["username"] + end + + body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) end project.issues.create!( description: body, title: issue["title"], state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gl_user_id(project, issue["reported_by"]["username"]) + author_id: gl_user_id(project, reporter) ) end @@ -47,9 +63,13 @@ module Gitlab private def gl_user_id(project, bitbucket_id) - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - end + if bitbucket_id + user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + (user && user.id) || project.creator_id + else + project.creator_id + end + end end end end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 8c106a61735..bd7340a80f1 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -18,7 +18,8 @@ module Gitlab direction: :asc).each do |issue| if issue.pull_request.nil? - body = @formatter.author_line(issue.user.login, issue.body) + body = @formatter.author_line(issue.user.login) + body += issue.body if issue.comments > 0 body += @formatter.comments_header diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 50594d2b24f..e24b94d6159 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -18,7 +18,8 @@ module Gitlab issues = client.issues(project_identifier) issues.each do |issue| - body = @formatter.author_line(issue["author"]["name"], issue["description"]) + body = @formatter.author_line(issue["author"]["name"]) + body += issue["description"] comments = client.issue_comments(project_identifier, issue["id"]) diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb index 72e041a90b1..3e54456e936 100644 --- a/lib/gitlab/import_formatter.rb +++ b/lib/gitlab/import_formatter.rb @@ -8,8 +8,8 @@ module Gitlab "\n\n\n**Imported comments:**\n" end - def author_line(author, body) - "*Created by: #{author}*\n\n#{body}" + def author_line(author) + "*Created by: #{author}*\n\n" end end end diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb index 9f6e19a09fd..097caf67a65 100644 --- a/lib/gitlab/markdown.rb +++ b/lib/gitlab/markdown.rb @@ -5,6 +5,32 @@ module Gitlab # # See the files in `lib/gitlab/markdown/` for specific processing information. module Markdown + # Convert a Markdown String into an HTML-safe String of HTML + # + # markdown - Markdown String + # context - Hash of context options passed to our HTML Pipeline + # + # Returns an HTML-safe String + def self.render(markdown, context = {}) + html = renderer.render(markdown) + html = gfm(html, context) + + html.html_safe + end + + # Convert a Markdown String into HTML without going through the HTML + # Pipeline. + # + # Note that because the pipeline is skipped, SanitizationFilter is as well. + # Do not output the result of this method to the user. + # + # markdown - Markdown String + # + # Returns a String + def self.render_without_gfm(markdown) + renderer.render(markdown) + end + # Provide autoload paths for filters to prevent a circular dependency error autoload :AutolinkFilter, 'gitlab/markdown/autolink_filter' autoload :CommitRangeReferenceFilter, 'gitlab/markdown/commit_range_reference_filter' @@ -18,6 +44,7 @@ module Gitlab autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' autoload :SanitizationFilter, 'gitlab/markdown/sanitization_filter' autoload :SnippetReferenceFilter, 'gitlab/markdown/snippet_reference_filter' + autoload :SyntaxHighlightFilter, 'gitlab/markdown/syntax_highlight_filter' autoload :TableOfContentsFilter, 'gitlab/markdown/table_of_contents_filter' autoload :TaskListFilter, 'gitlab/markdown/task_list_filter' autoload :UserReferenceFilter, 'gitlab/markdown/user_reference_filter' @@ -28,8 +55,7 @@ module Gitlab # options - A Hash of options used to customize output (default: {}): # :xhtml - output XHTML instead of HTML # :reference_only_path - Use relative path for reference links - # html_options - extra options for the reference links as given to link_to - def gfm(text, options = {}, html_options = {}) + def self.gfm(text, options = {}) return text if text.nil? # Duplicate the string so we don't alter the original, then call to_str @@ -40,8 +66,8 @@ module Gitlab options.reverse_merge!( xhtml: false, reference_only_path: true, - project: @project, - current_user: current_user + project: options[:project], + current_user: options[:current_user] ) @pipeline ||= HTML::Pipeline.new(filters) @@ -61,12 +87,11 @@ module Gitlab current_user: options[:current_user], only_path: options[:reference_only_path], project: options[:project], - reference_class: html_options[:class], # RelativeLinkFilter - ref: @ref, - requested_path: @path, - project_wiki: @project_wiki + ref: options[:ref], + requested_path: options[:path], + project_wiki: options[:project_wiki] } result = @pipeline.call(text, context) @@ -83,14 +108,36 @@ module Gitlab private + def self.renderer + @markdown ||= begin + renderer = Redcarpet::Render::HTML.new + Redcarpet::Markdown.new(renderer, redcarpet_options) + end + end + + def self.redcarpet_options + # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use + @redcarpet_options ||= { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + end + # Filters used in our pipeline # # SanitizationFilter should come first so that all generated reference HTML # goes through untouched. # # See https://github.com/jch/html-pipeline#filters for more filters. - def filters + def self.filters [ + Gitlab::Markdown::SyntaxHighlightFilter, Gitlab::Markdown::SanitizationFilter, Gitlab::Markdown::RelativeLinkFilter, diff --git a/lib/gitlab/markdown/autolink_filter.rb b/lib/gitlab/markdown/autolink_filter.rb index 541f1d88ffc..c37c3bc55bf 100644 --- a/lib/gitlab/markdown/autolink_filter.rb +++ b/lib/gitlab/markdown/autolink_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' require 'uri' diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/commit_range_reference_filter.rb index a9f1ee9c161..8613150894b 100644 --- a/lib/gitlab/markdown/commit_range_reference_filter.rb +++ b/lib/gitlab/markdown/commit_range_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces commit range references with links. diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/commit_reference_filter.rb index eacdf8a6d37..5696b4fa585 100644 --- a/lib/gitlab/markdown/commit_reference_filter.rb +++ b/lib/gitlab/markdown/commit_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces commit references with links. diff --git a/lib/gitlab/markdown/cross_project_reference.rb b/lib/gitlab/markdown/cross_project_reference.rb index 66c256c5104..855748fdccc 100644 --- a/lib/gitlab/markdown/cross_project_reference.rb +++ b/lib/gitlab/markdown/cross_project_reference.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # Common methods for ReferenceFilters that support an optional cross-project diff --git a/lib/gitlab/markdown/emoji_filter.rb b/lib/gitlab/markdown/emoji_filter.rb index 6794ab9c897..da10e4d3760 100644 --- a/lib/gitlab/markdown/emoji_filter.rb +++ b/lib/gitlab/markdown/emoji_filter.rb @@ -1,6 +1,7 @@ +require 'action_controller' +require 'gitlab/markdown' require 'gitlab_emoji' require 'html/pipeline/filter' -require 'action_controller' module Gitlab module Markdown diff --git a/lib/gitlab/markdown/external_issue_reference_filter.rb b/lib/gitlab/markdown/external_issue_reference_filter.rb index afd28dd8cf3..f7c43e1ca89 100644 --- a/lib/gitlab/markdown/external_issue_reference_filter.rb +++ b/lib/gitlab/markdown/external_issue_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces external issue tracker references with links. diff --git a/lib/gitlab/markdown/external_link_filter.rb b/lib/gitlab/markdown/external_link_filter.rb index c539e0fb823..29e51b6ade6 100644 --- a/lib/gitlab/markdown/external_link_filter.rb +++ b/lib/gitlab/markdown/external_link_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' module Gitlab diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb index ab6f6bc1cf7..01320f80796 100644 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ b/lib/gitlab/markdown/issue_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces issue references with links. References to diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/label_reference_filter.rb index 2186f36f854..3d7445a27f1 100644 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ b/lib/gitlab/markdown/label_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces label references with links. diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb index 884f60f9d53..48248f5219d 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/gitlab/markdown/merge_request_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces merge request references with links. References diff --git a/lib/gitlab/markdown/reference_filter.rb b/lib/gitlab/markdown/reference_filter.rb index 47ee1d99da3..9b293c957d6 100644 --- a/lib/gitlab/markdown/reference_filter.rb +++ b/lib/gitlab/markdown/reference_filter.rb @@ -1,4 +1,5 @@ require 'active_support/core_ext/string/output_safety' +require 'gitlab/markdown' require 'html/pipeline/filter' module Gitlab @@ -9,7 +10,6 @@ module Gitlab # # Context options: # :project (required) - Current project, ignored if reference is cross-project. - # :reference_class - Custom CSS class added to reference links. # :only_path - Generate path-only links. # # Results: @@ -70,7 +70,7 @@ module Gitlab end def reference_class(type) - "gfm gfm-#{type} #{context[:reference_class]}".strip + "gfm gfm-#{type}" end # Iterate through the document's text nodes, yielding the current node's diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb index 30f50b82996..8c5cf51bfe1 100644 --- a/lib/gitlab/markdown/relative_link_filter.rb +++ b/lib/gitlab/markdown/relative_link_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' require 'uri' diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb index 74b3a8d274f..68ed57f6257 100644 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ b/lib/gitlab/markdown/sanitization_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' require 'html/pipeline/sanitization_filter' diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb index 92979a356dc..9e1aab936cb 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/gitlab/markdown/snippet_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces snippet references with links. References to diff --git a/lib/gitlab/markdown/syntax_highlight_filter.rb b/lib/gitlab/markdown/syntax_highlight_filter.rb new file mode 100644 index 00000000000..86f4385753a --- /dev/null +++ b/lib/gitlab/markdown/syntax_highlight_filter.rb @@ -0,0 +1,39 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' +require 'rouge/plugins/redcarpet' + +module Gitlab + module Markdown + # HTML Filter to highlight fenced code blocks + # + class SyntaxHighlightFilter < HTML::Pipeline::Filter + include Rouge::Plugins::Redcarpet + + def call + doc.search('pre > code').each do |node| + highlight_node(node) + end + + doc + end + + def highlight_node(node) + language = node.attr('class') + code = node.text + + highlighted = block_code(code, language) + + # Replace the parent `pre` element with the entire highlighted block + node.parent.replace(highlighted) + end + + private + + # Override Rouge::Plugins::Redcarpet#rouge_formatter + def rouge_formatter(lexer) + Rouge::Formatters::HTMLGitlab.new( + cssclass: "code highlight js-syntax-highlight #{lexer.tag}") + end + end + end +end diff --git a/lib/gitlab/markdown/table_of_contents_filter.rb b/lib/gitlab/markdown/table_of_contents_filter.rb index 38887c9778c..bbb3bf7fc8b 100644 --- a/lib/gitlab/markdown/table_of_contents_filter.rb +++ b/lib/gitlab/markdown/table_of_contents_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' module Gitlab diff --git a/lib/gitlab/markdown/task_list_filter.rb b/lib/gitlab/markdown/task_list_filter.rb index c6eb2e2bf6d..2f133ae8500 100644 --- a/lib/gitlab/markdown/task_list_filter.rb +++ b/lib/gitlab/markdown/task_list_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'task_list/filter' module Gitlab diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb index a4aec7a05d1..1871e52df0e 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/gitlab/markdown/user_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces user or group references with links. diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index e836b05ff25..0961bd80421 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor @@ -10,7 +12,7 @@ module Gitlab def analyze(text) references.clear - @text = markdown.render(text.dup) + @text = Gitlab::Markdown.render_without_gfm(text) end %i(user label issue merge_request snippet commit commit_range).each do |type| @@ -21,10 +23,6 @@ module Gitlab private - def markdown - @markdown ||= Redcarpet::Markdown.new(Redcarpet::Render::HTML, GitlabMarkdownHelper::MARKDOWN_OPTIONS) - end - def references @references ||= Hash.new do |references, type| type = type.to_sym @@ -42,7 +40,7 @@ module Gitlab # Returns the results Array for the requested filter type def pipeline_result(filter_type) klass = filter_type.to_s.camelize + 'ReferenceFilter' - filter = "Gitlab::Markdown::#{klass}".constantize + filter = Gitlab::Markdown.const_get(klass) context = { project: project, |