diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 15:26:25 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 15:26:25 +0300 |
commit | a09983ae35713f5a2bbb100981116d31ce99826e (patch) | |
tree | 2ee2af7bd104d57086db360a7e6d8c9d5d43667a /lib/kramdown | |
parent | 18c5ab32b738c0b6ecb4d0df3994000482f34bd8 (diff) |
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
Diffstat (limited to 'lib/kramdown')
-rw-r--r-- | lib/kramdown/converter/commonmark.rb | 109 | ||||
-rw-r--r-- | lib/kramdown/parser/atlassian_document_format.rb | 381 |
2 files changed, 490 insertions, 0 deletions
diff --git a/lib/kramdown/converter/commonmark.rb b/lib/kramdown/converter/commonmark.rb new file mode 100644 index 00000000000..4abb34cc008 --- /dev/null +++ b/lib/kramdown/converter/commonmark.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module Kramdown + module Converter + # Overrides the base Kramdown converter to add any special + # behaviour for CommonMark. + # + # Currently we support an option `html_tables` that outputs + # an HTML table instead of a Markdown table. This is to + # support possibly being given complex tables, such as from ADF. + # + # Note: this is only an initial implementation. Currently don't + # strip out IALs or other specific kramdown syntax. + class Commonmark < ::Kramdown::Converter::Kramdown + # replaces the ^ used in kramdown. This forces the current + # block to end, so that a different list or codeblock can be + # started. https://kramdown.gettalong.org/syntax.html#eob-marker + END_OF_BLOCK = '<!-- -->' + + def convert(el, opts = { indent: 0 }) + res = super + + if [:ul, :dl, :ol, :codeblock].include?(el.type) && opts[:next] && + ([el.type, :codeblock].include?(opts[:next].type) || + (opts[:next].type == :blank && opts[:nnext] && + [el.type, :codeblock].include?(opts[:nnext].type))) + # replace the end of block character + res.sub!(/\^\n\n\z/m, "#{END_OF_BLOCK}\n\n") + end + + res + end + + def convert_codeblock(el, _opts) + # Although tildes are supported in CommonMark, backticks are more common + "```#{el.options[:lang]}\n" + + el.value.split(/\n/).map {|l| l.empty? ? "" : "#{l}" }.join("\n") + + "\n```\n\n" + end + + def convert_li(el, opts) + res = super + + if el.children.first && el.children.first.type == :p && !el.children.first.options[:transparent] + if el.children.size == 1 && @stack.last.children.last == el && + (@stack.last.children.any? {|c| c.children.first.type != :p } || @stack.last.children.size == 1) + # replace the end of block character + res.sub!(/\^\n\z/m, "#{END_OF_BLOCK}\n") + end + end + + res + end + + def convert_table(el, opts) + return super unless @options[:html_tables] + + opts[:alignment] = el.options[:alignment] + result = inner(el, opts) + + "<table>\n#{result}</table>\n\n" + end + + def convert_thead(el, opts) + return super unless @options[:html_tables] + + "<thead>\n#{inner(el, opts)}</thead>\n" + end + + def convert_tbody(el, opts) + return super unless @options[:html_tables] + + "<tbody>\n#{inner(el, opts)}</tbody>\n" + end + + def convert_tfoot(el, opts) + return super unless @options[:html_tables] + + "<tfoot>\n#{inner(el, opts)}</tfoot>\n" + end + + def convert_tr(el, opts) + return super unless @options[:html_tables] + + "<tr>\n#{el.children.map {|c| convert(c, opts) }.join}</tr>\n" + end + + def convert_td(el, opts) + return super unless @options[:html_tables] + + # We need to add two linefeeds in order for any inner text to + # be processed as markdown. The HTML block must be "closed", + # as referenced in the CommonMark spec + # @see https://spec.commonmark.org/0.29/#html-blocks + "<td>\n\n#{inner(el, opts)}</td>\n" + end + + def convert_th(el, opts) + return super unless @options[:html_tables] + + # We need to add two linefeeds in order for any inner text to + # be processed as markdown. The HTML block must be "closed", + # as referenced in the CommonMark spec + # @see https://spec.commonmark.org/0.29/#html-blocks + "<th>\n\n#{inner(el, opts)}</th>\n" + end + end + end +end diff --git a/lib/kramdown/parser/atlassian_document_format.rb b/lib/kramdown/parser/atlassian_document_format.rb new file mode 100644 index 00000000000..4ceb879a04c --- /dev/null +++ b/lib/kramdown/parser/atlassian_document_format.rb @@ -0,0 +1,381 @@ +# frozen_string_literal: true + +module Kramdown + module Parser + # Parses an Atlassian Document Format (ADF) json into a + # Kramdown AST tree, for conversion to another format. + # The primary goal is to convert in GitLab Markdown. + # + # This parser does NOT resolve external resources, such as media/attachments. + # A special url is generated for media based on the id, for example + # ![jira-10050-field-description](adf-media://79411c6b-50e0-477f-b4ed-ac3a5887750c) + # so that a later filter/process can resolve those. + # + # @see https://developer.atlassian.com/cloud/jira/platform/apis/document/structure/ ADF Document Structure + # @see https://developer.atlassian.com/cloud/jira/platform/apis/document/playground/ ADF Playground + # @see https://developer.atlassian.com/cloud/jira/platform/apis/document/viewer/ ADF Viewer + class AtlassianDocumentFormat < Kramdown::Parser::Base + unless defined?(TOP_LEVEL_BLOCK_NODES) + TOP_LEVEL_BLOCK_NODES = %w[blockquote + bulletList + codeBlock + heading + mediaGroup + mediaSingle + orderedList + panel + paragraph + rule + table].freeze + + CHILD_BLOCK_NODES = %w[listItem + media + table_cell + table_header + table_row].freeze + + INLINE_NODES = %w[emoji + hardBreak + inlineCard + mention + text].freeze + + MARKS = %w[code + em + link + strike + strong + subsup + textColor + underline].freeze + + TABLE_CELL_NODES = %w[blockquote + bulletList + codeBlock + heading + mediaGroup + orderedList + panel + paragraph + rule].freeze + + LIST_ITEM_NODES = %w[bulletList + codeBlock + mediaSingle + orderedList + paragraph].freeze + + PANEL_NODES = %w[bulletList + heading + orderedList + paragraph].freeze + + PANEL_EMOJIS = { info: ':information_source:', + note: ':notepad_spiral:', + warning: ':warning:', + success: ':white_check_mark:', + error: ':octagonal_sign:' }.freeze + + # The default language for code blocks is `java`, as indicated in + # You can't change the default in Jira. There was a comment that indicated + # Confluence can set the default language. + # @see https://jira.atlassian.com/secure/WikiRendererHelpAction.jspa?section=advanced&_ga=2.5135221.773220073.1591894917-438867908.1591894917 + # @see https://jira.atlassian.com/browse/JRASERVER-29184?focusedCommentId=832255&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-832255 + CODE_BLOCK_DEFAULT_LANGUAGE = 'java' + end + + def parse + ast = Gitlab::Json.parse(@source) + + validate_document(ast) + + process_content(@root, ast, TOP_LEVEL_BLOCK_NODES) + rescue ::JSON::ParserError => e + msg = 'Invalid Atlassian Document Format JSON' + Gitlab::AppLogger.error msg + Gitlab::AppLogger.error e + + raise ::Kramdown::Error, msg + end + + def process_content(element, ast_node, allowed_types) + ast_node['content'].each do |node| + next unless allowed_types.include?(node['type']) + + public_send("process_#{node['type'].underscore}", element, node) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def process_blockquote(element, ast_node) + new_element = Element.new(:blockquote) + element.children << new_element + + process_content(new_element, ast_node, TOP_LEVEL_BLOCK_NODES) + end + + def process_bullet_list(element, ast_node) + new_element = Element.new(:ul) + element.children << new_element + + process_content(new_element, ast_node, %w[listItem]) + end + + def process_code_block(element, ast_node) + code_text = gather_text(ast_node) + lang = ast_node.dig('attrs', 'language') || CODE_BLOCK_DEFAULT_LANGUAGE + + element.children << Element.new(:codeblock, code_text, {}, { lang: lang }) + end + + def process_emoji(element, ast_node) + emoji = ast_node.dig('attrs', 'text') || ast_node.dig('attrs', 'shortName') + return unless emoji + + add_text(emoji, element, :text) + end + + def process_hard_break(element, ast_node) + element.children << Element.new(:br) + end + + def process_heading(element, ast_node) + level = ast_node.dig('attrs', 'level').to_i.clamp(1, 6) + options = { level: level } + new_element = Element.new(:header, nil, nil, options) + element.children << new_element + + process_content(new_element, ast_node, INLINE_NODES) + extract_element_text(new_element, new_element.options[:raw_text] = +'') + end + + def process_inline_card(element, ast_node) + url = ast_node.dig('attrs', 'url') + data = ast_node.dig('attrs', 'data') + + if url + # we don't pull a description from the link and create a panel, + # just convert to a normal link + new_element = Element.new(:text, url) + element.children << wrap_element(new_element, :a, nil, { 'href' => url }) + elsif data + # data is JSONLD (https://json-ld.org/), so for now output + # as a codespan of text, with `adf-inlineCard: ` at the start + text = "adf-inlineCard: #{data}" + element.children << Element.new(:codespan, text, nil, { lang: 'adf-inlinecard' }) + end + end + + def process_list_item(element, ast_node) + new_element = Element.new(:li) + element.children << new_element + + process_content(new_element, ast_node, LIST_ITEM_NODES) + end + + def process_media(element, ast_node) + media_url = "adf-media://#{ast_node['attrs']['id']}" + + case ast_node['attrs']['type'] + when 'file' + attrs = { 'src' => media_url, 'alt' => ast_node['attrs']['collection'] } + media_element = Element.new(:img, nil, attrs) + when 'link' + attrs = { 'href' => media_url } + media_element = wrap_element(Element.new(:text, media_url), :a, nil, attrs) + end + + media_element = wrap_element(media_element, :p) + element.children << media_element + end + + # wraps a single media element. + # Currently ignore attrs.layout and attrs.width + def process_media_single(element, ast_node) + new_element = Element.new(:p) + element.children << new_element + + process_content(new_element, ast_node, %w[media]) + end + + # wraps a group media element. + # Currently ignore attrs.layout and attrs.width + def process_media_group(element, ast_node) + ul_element = Element.new(:ul) + element.children << ul_element + + ast_node['content'].each do |node| + next unless node['type'] == 'media' + + li_element = Element.new(:li) + ul_element.children << li_element + + process_media(li_element, node) + end + end + + def process_mention(element, ast_node) + # Make it `@adf-mention:` since there is no guarantee that it is + # a valid username in our system. This gives us an + # opportunity to replace it later. Mention name can have + # spaces, so double quote it + mention_text = ast_node.dig('attrs', 'text')&.gsub('@', '') + mention_text = %Q("#{mention_text}") if mention_text.match?(/ /) + mention_text = %Q(@adf-mention:#{mention_text}) + + add_text(mention_text, element, :text) + end + + def process_ordered_list(element, ast_node) + # `attrs.order` is not supported in the Kramdown AST + new_element = Element.new(:ol) + element.children << new_element + + process_content(new_element, ast_node, %w[listItem]) + end + + # since we don't have something similar, then put <hr> around it and + # add a bolded status text (eg: "Error:") to the front of it. + def process_panel(element, ast_node) + panel_type = ast_node.dig('attrs', 'panelType') + return unless %w[info note warning success error].include?(panel_type) + + panel_header_text = "#{PANEL_EMOJIS[panel_type.to_sym]} " + panel_header_element = Element.new(:text, panel_header_text) + + new_element = Element.new(:blockquote) + new_element.children << panel_header_element + element.children << new_element + + process_content(new_element, ast_node, PANEL_NODES) + end + + def process_paragraph(element, ast_node) + new_element = Element.new(:p) + element.children << new_element + + process_content(new_element, ast_node, INLINE_NODES) + end + + def process_rule(element, ast_node) + element.children << Element.new(:hr) + end + + def process_table(element, ast_node) + table = Element.new(:table, nil, nil, { alignment: [:default, :default] }) + element.children << table + + tbody = Element.new(:tbody) + table.children << tbody + + process_content(tbody, ast_node, %w[tableRow]) + end + + # we ignore the attributes, attrs.background, attrs.colspan, + # attrs.colwidth, and attrs.rowspan + def process_table_cell(element, ast_node) + new_element = Element.new(:td) + element.children << new_element + + process_content(new_element, ast_node, TABLE_CELL_NODES) + end + + # we ignore the attributes, attrs.background, attrs.colspan, + # attrs.colwidth, and attrs.rowspan + def process_table_header(element, ast_node) + new_element = Element.new(:th) + element.children << new_element + + process_content(new_element, ast_node, TABLE_CELL_NODES) + end + + def process_table_row(element, ast_node) + new_element = Element.new(:tr) + element.children << new_element + + process_content(new_element, ast_node, %w[tableHeader tableCell]) + end + + def process_text(element, ast_node) + new_element = Element.new(:text, ast_node['text']) + new_element = apply_marks(new_element, ast_node, MARKS) + element.children << new_element + end + + private + + def validate_document(ast) + return if ast['type'] == 'doc' + + raise ::JSON::ParserError, 'missing doc node' + end + + # ADF marks are an attribute on the node. For kramdown, + # we have to wrap the node with an element for the mark. + def apply_marks(element, ast_node, allowed_types) + return element unless ast_node['marks'] + + new_element = element + + ast_node['marks'].each do |mark| + next unless allowed_types.include?(mark['type']) + + case mark['type'] + when 'code' + new_element = Element.new(:codespan, ast_node['text']) + when 'em' + new_element = wrap_element(new_element, :em) + when 'link' + attrs = { 'href' => mark.dig('attrs', 'href') } + attrs['title'] = mark.dig('attrs', 'title') + new_element = wrap_element(new_element, :a, nil, attrs) + when 'strike' + new_element = wrap_element(new_element, :html_element, 'del', {}, category: :span) + when 'strong' + new_element = wrap_element(new_element, :strong) + when 'subsup' + type = mark.dig('attrs', 'type') + + case type + when 'sub' + new_element = wrap_element(new_element, :html_element, 'sub', {}, category: :span) + when 'sup' + new_element = wrap_element(new_element, :html_element, 'sup', {}, category: :span) + else + next + end + when 'textColor' + color = mark.dig('attrs', 'color') + new_element = wrap_element(new_element, :html_element, 'span', { color: color }, category: :span) + when 'underline' + new_element = wrap_element(new_element, :html_element, 'u', {}, category: :span) + else + next + end + end + + new_element + end + + def wrap_element(element, type, *args) + wrapper = Element.new(type, *args) + wrapper.children << element + wrapper + end + + def extract_element_text(element, raw) + raw << element.value.to_s if element.type == :text + element.children.each { |c| extract_element_text(c, raw) } + end + + def gather_text(ast_node) + ast_node['content'].inject('') do |memo, node| + node['type'] == 'text' ? (memo + node['text']) : memo + end + end + + def method_missing(method, *args) + raise NotImplementedError, "method `#{method}` not implemented yet" + end + end + end +end |