diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
tree | dceebdc68925362117480a5d672bcff122fb625b /lib/banzai | |
parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'lib/banzai')
44 files changed, 377 insertions, 275 deletions
diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb index 6371a8f23af..00ffdd3d809 100644 --- a/lib/banzai/filter/asset_proxy_filter.rb +++ b/lib/banzai/filter/asset_proxy_filter.rb @@ -62,7 +62,7 @@ module Banzai # whenever the application settings are changed def self.initialize_settings application_settings = Gitlab::CurrentSettings.current_application_settings - Gitlab.config['asset_proxy'] ||= Settingslogic.new({}) + Gitlab.config['asset_proxy'] ||= GitlabSettings::Options.build({}) if application_settings.respond_to?(:asset_proxy_enabled) Gitlab.config.asset_proxy['enabled'] = application_settings.asset_proxy_enabled diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb index 3b00d1a9824..0735fbb8d4c 100644 --- a/lib/banzai/filter/base_sanitization_filter.rb +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -25,7 +25,8 @@ module Banzai # Allow data-math-style attribute in order to support LaTeX formatting allowlist[:attributes]['code'] = %w(data-math-style) - allowlist[:attributes]['pre'] = %w(data-math-style data-mermaid-style data-kroki-style) + allowlist[:attributes]['pre'] = %w(data-canonical-lang data-lang-params + data-math-style data-mermaid-style data-kroki-style) # Allow html5 details/summary elements allowlist[:elements].push('details') diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb index 8681173b1f4..d4ff7d4c6b5 100644 --- a/lib/banzai/filter/blockquote_fence_filter.rb +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -2,7 +2,7 @@ module Banzai module Filter - class BlockquoteFenceFilter < HTML::Pipeline::TextFilter + class BlockquoteFenceFilter < TimeoutTextPipelineFilter REGEX = %r{ #{::Gitlab::Regex.markdown_code_or_html_blocks} | @@ -39,7 +39,7 @@ module Banzai @text = @text.delete("\r") end - def call + def call_with_timeout @text.gsub(REGEX) do if $~[:blockquote] # keep the same number of source lines/positions by replacing the diff --git a/lib/banzai/filter/code_language_filter.rb b/lib/banzai/filter/code_language_filter.rb new file mode 100644 index 00000000000..60e5a4063df --- /dev/null +++ b/lib/banzai/filter/code_language_filter.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML Filter to convert use of `lang` attribute into a common format, + # data-canonical-lang, as the `lang` attribute is really meant for accessibility + # and not for specifying code highlight language. + # See https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/lang#accessibility + # This also provides one place to transform the language specification format, whether it + # sits on the `pre` or `code`, or in a `class` or `lang` attribute + class CodeLanguageFilter < HTML::Pipeline::Filter + include OutputSafety + + LANG_PARAMS_DELIMITER = ':' + LANG_ATTR = 'data-canonical-lang' + LANG_PARAMS_ATTR = 'data-lang-params' + + CSS = 'pre > code:only-child' + XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze + + def call + doc.xpath(XPATH).each do |node| + transform_node(node) + end + + doc + end + + def transform_node(code_node) + return if code_node.parent&.parent.nil? + + lang, lang_params = parse_lang_params(code_node) + pre_node = code_node.parent + + pre_node.remove_attribute('lang') if lang.present? + pre_node.set_attribute(LANG_ATTR, escape_once(lang)) if lang.present? + pre_node.set_attribute(LANG_PARAMS_ATTR, escape_once(lang_params)) if lang_params.present? + + # cmark-gfm added this, it's now in data-lang-params + pre_node.remove_attribute('data-meta') + end + + private + + # cmark-gfm's FULL_INFO_STRING render option works with the space delimiter. + # Which means the language specified on a code block is parsed with spaces. Anything + # after the first space is placed in the `data-meta` attribute. + # However GitLab recognizes `:` as an additional delimiter on the lang attribute. + # So parse out the extra parameter. + # + # Original + # "```suggestion:+1-10 more```" -> '<pre data-canonical-lang="suggestion:+1-10" data-lang-params="more">'. + # + # With extra parsing + # "```suggestion:+1-10 more```" -> '<pre data-canonical-lang="suggestion" data-lang-params="+1-10 more">'. + def parse_lang_params(code_node) + pre_node = code_node.parent + language = pre_node.attr('lang') + + return unless language + + language, language_params = language.split(LANG_PARAMS_DELIMITER, 2) + + # cmark-gfm places extra lang parameters into data-meta + language_params = [pre_node.attr('data-meta'), language_params].compact.join(' ') + + [language, language_params] + end + end + end +end diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb index 817bea42757..c0160680a61 100644 --- a/lib/banzai/filter/commit_trailers_filter.rb +++ b/lib/banzai/filter/commit_trailers_filter.rb @@ -70,7 +70,7 @@ module Banzai # # Returns a String with a link to the user. def link_to_user_or_email(name, email, trailer) - link_to_user User.find_by_any_email(email), + link_to_user User.with_public_email(email).first, name: name, email: email, trailer: trailer diff --git a/lib/banzai/filter/dollar_math_pre_filter.rb b/lib/banzai/filter/dollar_math_pre_filter.rb index aaa186f87a6..937328a2056 100644 --- a/lib/banzai/filter/dollar_math_pre_filter.rb +++ b/lib/banzai/filter/dollar_math_pre_filter.rb @@ -16,31 +16,30 @@ module Banzai # by converting it into the ```math syntax. In this way, we can ensure # that it's considered a code block and will not have any markdown processed inside it. - # Corresponds to the "$$\n...\n$$" syntax - REGEX = %r{ - #{::Gitlab::Regex.markdown_code_or_html_blocks} - | - (?=(?<=^\n|\A)\$\$\ *\n.*\n\$\$\ *(?=\n$|\z))(?: - # Display math block: - # $$ - # latex math - # $$ - - (?<=^\n|\A)\$\$\ *\n - (?<display_math> - (?:.)+? - ) - \n\$\$\ *(?=\n$|\z) - ) - }mx.freeze + # Display math block: + # $$ + # latex math + # $$ + REGEX = + "#{::Gitlab::Regex.markdown_code_or_html_blocks_or_html_comments_untrusted}" \ + '|' \ + '^\$\$\ *\n' \ + '(?P<display_math>' \ + '(?:\n|.)*?' \ + ')' \ + '\n\$\$\ *$' \ + .freeze def call - @text.gsub(REGEX) do - if $~[:display_math] - # change from $$ to ```math - "```math\n#{$~[:display_math]}\n```" + regex = Gitlab::UntrustedRegexp.new(REGEX, multiline: true) + return @text unless regex.match?(@text) + + regex.replace_gsub(@text) do |match| + # change from $$ to ```math + if match[:display_math] + "```math\n#{match[:display_math]}\n```" else - $~[0] + match.to_s end end end diff --git a/lib/banzai/filter/inline_embeds_filter.rb b/lib/banzai/filter/inline_embeds_filter.rb index c1077674cf0..a16166123f8 100644 --- a/lib/banzai/filter/inline_embeds_filter.rb +++ b/lib/banzai/filter/inline_embeds_filter.rb @@ -10,6 +10,8 @@ module Banzai # the link, and insert this node after any html content # surrounding the link. def call + return doc if Feature.enabled?(:remove_monitor_metrics) + doc.xpath(xpath_search).each do |node| next unless element = element_to_embed(node) diff --git a/lib/banzai/filter/inline_observability_filter.rb b/lib/banzai/filter/inline_observability_filter.rb index 50d4aac70cc..8e38f689959 100644 --- a/lib/banzai/filter/inline_observability_filter.rb +++ b/lib/banzai/filter/inline_observability_filter.rb @@ -1,14 +1,22 @@ # frozen_string_literal: true -require 'uri' - module Banzai module Filter - class InlineObservabilityFilter < ::Banzai::Filter::InlineEmbedsFilter + class InlineObservabilityFilter < HTML::Pipeline::Filter + include Gitlab::Utils::StrongMemoize + def call - return doc unless can_view_observability? + return doc unless Gitlab::Observability.enabled?(group) + + doc.xpath(xpath_search).each do |node| + next unless element = element_to_embed(node) + + # We want this to follow any surrounding content. For example, + # if a link is inline in a paragraph. + node.parent.children.last.add_next_sibling(element) + end - super + doc end # Placeholder element for the frontend to use as an @@ -17,40 +25,39 @@ module Banzai doc.document.create_element( 'div', class: 'js-render-observability', - 'data-frame-url': url, - 'data-observability-url': Gitlab::Observability.observability_url + 'data-frame-url': url ) end # Search params for selecting observability links. def xpath_search - "descendant-or-self::a[starts-with(@href, '#{Gitlab::Observability.observability_url}')]" + "descendant-or-self::a[starts-with(@href, '#{gitlab_domain}/groups/') and contains(@href,'/-/observability/')]" end # Creates a new element based on the parameters # obtained from the target link def element_to_embed(node) url = node['href'] - uri = URI.parse(url) - observability_uri = URI.parse(Gitlab::Observability.observability_url) - - if uri.scheme == observability_uri.scheme && - uri.port == observability_uri.port && - uri.host.casecmp?(observability_uri.host) && - uri.path.downcase.exclude?("auth/start") - create_element(url) - end + + embeddable_url = extract_embeddable_url(url) + create_element(embeddable_url) if embeddable_url end private - def can_view_observability? - Feature.enabled?(:observability_group_tab, group) + def extract_embeddable_url(url) + strong_memoize_with(:embeddable_url, url) do + Gitlab::Observability.embeddable_url(url) + end end def group context[:group] || context[:project]&.group end + + def gitlab_domain + ::Gitlab.config.gitlab.url + end end end end diff --git a/lib/banzai/filter/issuable_reference_expansion_filter.rb b/lib/banzai/filter/issuable_reference_expansion_filter.rb index 6822e36c9be..ec7778a3630 100644 --- a/lib/banzai/filter/issuable_reference_expansion_filter.rb +++ b/lib/banzai/filter/issuable_reference_expansion_filter.rb @@ -10,13 +10,17 @@ module Banzai class IssuableReferenceExpansionFilter < HTML::Pipeline::Filter include Gitlab::Utils::StrongMemoize + NUMBER_OF_SUMMARY_ASSIGNEES = 2 VISIBLE_STATES = %w(closed merged).freeze + EXTENDED_FORMAT_XPATH = Gitlab::Utils::Nokogiri.css_to_xpath('a[data-reference-format="+s"]') def call return doc unless context[:issuable_reference_expansion_enabled] - context = RenderContext.new(project, current_user) - extractor = Banzai::IssuableExtractor.new(context) + options = { extended_preload: doc.xpath(EXTENDED_FORMAT_XPATH).present? } + extractor_context = RenderContext.new(project, current_user, options: options) + + extractor = Banzai::IssuableExtractor.new(extractor_context) issuables = extractor.extract([doc]) issuables.each do |node, issuable| @@ -26,6 +30,9 @@ module Banzai case node.attr('data-reference-format') when '+' expand_reference_with_title_and_state(node, issuable) + when '+s' + expand_reference_with_title_and_state(node, issuable) + expand_reference_with_summary(node, issuable) else expand_reference_with_state(node, issuable) end @@ -43,17 +50,47 @@ module Banzai node.content += ')' end + # rubocop:disable Style/AsciiComments + # Example: Issue Title (#123 - closed) assignee name 1, assignee name 2+ • v15.9 • On track + def expand_reference_with_summary(node, issuable) + summary = [] + + summary << assignees_text(issuable) if issuable.supports_assignee? + summary << milestone_text(issuable.milestone) if issuable.supports_milestone? + summary << health_status_text(issuable.health_status) if issuable.supports_health_status? + + node.content = [node.content, *summary].compact_blank.join(' • ') + end + # rubocop:enable Style/AsciiComments + # Example: #123 (closed) def expand_reference_with_state(node, issuable) node.content += " (#{issuable_state_text(issuable)})" end + def assignees_text(issuable) + assignee_names = issuable.assignees.first(NUMBER_OF_SUMMARY_ASSIGNEES + 1).map(&:sanitize_name) + + return _('Unassigned') if assignee_names.empty? + + "#{assignee_names.first(NUMBER_OF_SUMMARY_ASSIGNEES).to_sentence(two_words_connector: ', ')}" \ + "#{assignee_names.size > NUMBER_OF_SUMMARY_ASSIGNEES ? '+' : ''}" + end + + def milestone_text(milestone) + milestone&.title + end + + def health_status_text(health_status) + health_status&.humanize + end + def issuable_state_text(issuable) moved_issue?(issuable) ? s_("IssuableStatus|moved") : issuable.state end def moved_issue?(issuable) - issuable.instance_of?(Issue) && issuable.moved? + issuable.is_a?(Issue) && issuable.moved? end def should_expand?(node, issuable) diff --git a/lib/banzai/filter/kroki_filter.rb b/lib/banzai/filter/kroki_filter.rb index 2b9e2a22c11..04f1a1b4f3c 100644 --- a/lib/banzai/filter/kroki_filter.rb +++ b/lib/banzai/filter/kroki_filter.rb @@ -18,8 +18,8 @@ module Banzai diagram_selectors = ::Gitlab::Kroki.formats(settings) .map do |diagram_type| - %(pre[lang="#{diagram_type}"] > code, - pre > code[lang="#{diagram_type}"]) + %(pre[data-canonical-lang="#{diagram_type}"] > code, + pre > code[data-canonical-lang="#{diagram_type}"]) end .join(', ') @@ -28,7 +28,7 @@ module Banzai diagram_format = "svg" doc.xpath(xpath).each do |node| - diagram_type = node.parent['lang'] || node['lang'] + diagram_type = node.parent['data-canonical-lang'] || node['data-canonical-lang'] next unless diagram_selectors.include?(diagram_type) diagram_src = node.content diff --git a/lib/banzai/filter/markdown_engines/base.rb b/lib/banzai/filter/markdown_engines/base.rb new file mode 100644 index 00000000000..34f1d4d3da9 --- /dev/null +++ b/lib/banzai/filter/markdown_engines/base.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module MarkdownEngines + class Base + def initialize(context) + @context = context + end + + def render(text) + raise NotImplementedError + end + + private + + def sourcepos_disabled? + @context[:no_sourcepos] + end + end + end + end +end diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index 7abfadc612b..63680aa102c 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -9,7 +9,7 @@ module Banzai module Filter module MarkdownEngines - class CommonMark + class CommonMark < Base EXTENSIONS = [ :autolink, # provides support for automatically converting URLs to anchor tags. :strikethrough, # provides support for strikethroughs. @@ -29,9 +29,7 @@ module Banzai :UNSAFE # allow raw/custom HTML and unsafe links. ].freeze - def initialize(context) - @context = context - end + RENDER_OPTIONS_SOURCEPOS = RENDER_OPTIONS + [:SOURCEPOS].freeze def render(text) CommonMarker.render_html(text, render_options, EXTENSIONS) @@ -40,17 +38,7 @@ module Banzai private def render_options - @context[:no_sourcepos] ? render_options_no_sourcepos : render_options_sourcepos - end - - def render_options_no_sourcepos - RENDER_OPTIONS - end - - def render_options_sourcepos - render_options_no_sourcepos + [ - :SOURCEPOS # enable embedding of source position information - ].freeze + sourcepos_disabled? ? RENDER_OPTIONS : RENDER_OPTIONS_SOURCEPOS end end end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index 242e39f5495..a546a72da5d 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -3,10 +3,12 @@ module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter + DEFAULT_ENGINE = :common_mark + def initialize(text, context = nil, result = nil) super(text, context, result) - @renderer = renderer(context[:markdown_engine]).new(context) + @renderer = self.class.render_engine(context[:markdown_engine]).new(context) @text = @text.delete("\r") end @@ -14,20 +16,20 @@ module Banzai @renderer.render(@text).rstrip end - private + class << self + def render_engine(engine_from_context) + "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize + rescue NameError + raise NameError, "`#{engine_from_context}` is unknown markdown engine" + end - DEFAULT_ENGINE = :common_mark + private - def engine(engine_from_context) - engine_from_context ||= DEFAULT_ENGINE - - engine_from_context.to_s.classify - end + def engine(engine_from_context) + engine_from_context ||= DEFAULT_ENGINE - def renderer(engine_from_context) - "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize - rescue NameError - raise NameError, "`#{engine_from_context}` is unknown markdown engine" + engine_from_context.to_s.classify + end end end end diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 9b6fc71077a..e568f51652f 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -12,7 +12,7 @@ module Banzai # Handle the $`...`$ and ```math syntax in this filter. # Also add necessary classes any existing math blocks. - CSS_MATH = 'pre[lang="math"] > code' + CSS_MATH = 'pre[data-canonical-lang="math"] > code' XPATH_MATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_MATH).freeze CSS_CODE = 'code' XPATH_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_CODE).freeze diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb index aaaf851ccf0..b562cbcdefe 100644 --- a/lib/banzai/filter/mermaid_filter.rb +++ b/lib/banzai/filter/mermaid_filter.rb @@ -4,7 +4,7 @@ module Banzai module Filter class MermaidFilter < HTML::Pipeline::Filter - CSS = 'pre[lang="mermaid"] > code' + CSS = 'pre[data-canonical-lang="mermaid"] > code' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze def call diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index 6a1fa64fb76..2e5f1b29c52 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -32,7 +32,7 @@ module Banzai def lang_tag @lang_tag ||= Gitlab::Utils::Nokogiri - .css_to_xpath('pre[lang="plantuml"] > code, pre > code[lang="plantuml"]').freeze + .css_to_xpath('pre[data-canonical-lang="plantuml"] > code, pre > code[data-canonical-lang="plantuml"]').freeze end def settings diff --git a/lib/banzai/filter/reference_redactor_filter.rb b/lib/banzai/filter/reference_redactor_filter.rb index 485d3fd5fc7..9fae46a24a9 100644 --- a/lib/banzai/filter/reference_redactor_filter.rb +++ b/lib/banzai/filter/reference_redactor_filter.rb @@ -10,9 +10,9 @@ module Banzai class ReferenceRedactorFilter < HTML::Pipeline::Filter def call unless context[:skip_redaction] - context = RenderContext.new(project, current_user) + redactor_context = RenderContext.new(project, current_user) - ReferenceRedactor.new(context).redact([doc]) + ReferenceRedactor.new(redactor_context).redact([doc]) end doc diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb index 1ca38d2612d..3e48fe33b03 100644 --- a/lib/banzai/filter/references/abstract_reference_filter.rb +++ b/lib/banzai/filter/references/abstract_reference_filter.rb @@ -202,9 +202,13 @@ module Banzai title = object_link_title(object, matches) klass = reference_class(object_sym) - data_attributes = data_attributes_for(link_content || match, parent, object, - link_content: !!link_content, - link_reference: link_reference) + data_attributes = data_attributes_for( + link_content || match, + parent, + object, + link_content: !!link_content, + link_reference: link_reference + ) data_attributes[:reference_format] = matches[:format] if matches.names.include?("format") data_attributes.merge!(additional_object_attributes(object)) diff --git a/lib/banzai/filter/references/commit_range_reference_filter.rb b/lib/banzai/filter/references/commit_range_reference_filter.rb index df7f42eaa70..d0a24f3f0f0 100644 --- a/lib/banzai/filter/references/commit_range_reference_filter.rb +++ b/lib/banzai/filter/references/commit_range_reference_filter.rb @@ -32,8 +32,7 @@ module Banzai def url_for_object(range, project) h = Gitlab::Routing.url_helpers - h.project_compare_url(project, - range.to_param.merge(only_path: context[:only_path])) + h.project_compare_url(project, range.to_param.merge(only_path: context[:only_path])) end def object_link_title(range, matches) diff --git a/lib/banzai/filter/references/commit_reference_filter.rb b/lib/banzai/filter/references/commit_reference_filter.rb index 86ab8597cf5..0f412c1fe8d 100644 --- a/lib/banzai/filter/references/commit_reference_filter.rb +++ b/lib/banzai/filter/references/commit_reference_filter.rb @@ -49,14 +49,14 @@ module Banzai h = Gitlab::Routing.url_helpers if referenced_merge_request_commit_shas.include?(commit.id) - h.diffs_project_merge_request_url(project, - noteable, - commit_id: commit.id, - only_path: only_path?) + h.diffs_project_merge_request_url( + project, + noteable, + commit_id: commit.id, + only_path: only_path? + ) else - h.project_commit_url(project, - commit, - only_path: only_path?) + h.project_commit_url(project, commit, only_path: only_path?) end end diff --git a/lib/banzai/filter/references/design_reference_filter.rb b/lib/banzai/filter/references/design_reference_filter.rb index 01e1036dcec..16a2a2835e9 100644 --- a/lib/banzai/filter/references/design_reference_filter.rb +++ b/lib/banzai/filter/references/design_reference_filter.rb @@ -43,7 +43,7 @@ module Banzai return [] unless project.design_management_enabled? iids = identifiers.map(&:issue_iid).to_set - issues = project.issues.where(iid: iids) + issues = project.issues.where(iid: iids).includes(:project, :namespace) id_for_iid = issues.index_by(&:iid).transform_values(&:id) issue_by_id = issues.index_by(&:id) diff --git a/lib/banzai/filter/references/issue_reference_filter.rb b/lib/banzai/filter/references/issue_reference_filter.rb index b536d900a02..9ad9d286ce3 100644 --- a/lib/banzai/filter/references/issue_reference_filter.rb +++ b/lib/banzai/filter/references/issue_reference_filter.rb @@ -22,7 +22,7 @@ module Banzai end def parent_records(parent, ids) - parent.issues.where(iid: ids.to_a) + parent.issues.where(iid: ids.to_a).includes(:project, :namespace, :work_item_type) end def object_link_text_extras(issue, matches) @@ -40,7 +40,7 @@ module Banzai private def additional_object_attributes(issue) - { issue_type: issue.issue_type } + { issue_type: issue.work_item_type.base_type } end def issue_path(issue, project) diff --git a/lib/banzai/filter/references/iteration_reference_filter.rb b/lib/banzai/filter/references/iteration_reference_filter.rb deleted file mode 100644 index 591e07013c3..00000000000 --- a/lib/banzai/filter/references/iteration_reference_filter.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - module References - # The actual filter is implemented in the EE mixin - class IterationReferenceFilter < AbstractReferenceFilter - self.reference_type = :iteration - self.object_class = Iteration - end - end - end -end - -Banzai::Filter::References::IterationReferenceFilter.prepend_mod_with('Banzai::Filter::References::IterationReferenceFilter') diff --git a/lib/banzai/filter/references/merge_request_reference_filter.rb b/lib/banzai/filter/references/merge_request_reference_filter.rb index 5bc18ee6985..2518d7653f6 100644 --- a/lib/banzai/filter/references/merge_request_reference_filter.rb +++ b/lib/banzai/filter/references/merge_request_reference_filter.rb @@ -13,8 +13,7 @@ module Banzai def url_for_object(mr, project) h = Gitlab::Routing.url_helpers - h.project_merge_request_url(project, mr, - only_path: context[:only_path]) + h.project_merge_request_url(project, mr, only_path: context[:only_path]) end def object_link_text_extras(object, matches) diff --git a/lib/banzai/filter/references/snippet_reference_filter.rb b/lib/banzai/filter/references/snippet_reference_filter.rb index 502bfca1ab7..1f5ab0645fe 100644 --- a/lib/banzai/filter/references/snippet_reference_filter.rb +++ b/lib/banzai/filter/references/snippet_reference_filter.rb @@ -23,8 +23,7 @@ module Banzai def url_for_object(snippet, project) h = Gitlab::Routing.url_helpers - h.project_snippet_url(project, snippet, - only_path: context[:only_path]) + h.project_snippet_url(project, snippet, only_path: context[:only_path]) end end end diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb index 1709b607c2e..5983036a8e5 100644 --- a/lib/banzai/filter/references/user_reference_filter.rb +++ b/lib/banzai/filter/references/user_reference_filter.rb @@ -139,11 +139,7 @@ module Banzai end def team_member?(user) - if parent_group? - parent.member?(user) - else - parent.team.member?(user) - end + parent.member?(user) end def parent_url(link_content, author) diff --git a/lib/banzai/filter/references/work_item_reference_filter.rb b/lib/banzai/filter/references/work_item_reference_filter.rb new file mode 100644 index 00000000000..ed62b9a1be1 --- /dev/null +++ b/lib/banzai/filter/references/work_item_reference_filter.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces work item references with links. References to + # work items that do not exist are ignored. + # + # This filter supports cross-project references. + class WorkItemReferenceFilter < IssueReferenceFilter + self.reference_type = :work_item + self.object_class = WorkItem + + def parent_records(parent, ids) + parent.work_items.where(iid: ids.to_a) + end + + private + + def additional_object_attributes(work_item) + { work_item_type: work_item.work_item_type.base_type } + end + end + end + end +end diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index ddc3f5cf715..e06126bdf0f 100644 --- a/lib/banzai/filter/repository_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -204,7 +204,7 @@ module Banzai end def repo_visible_to_user? - project && Ability.allowed?(current_user, :download_code, project) + project && Ability.allowed?(current_user, :read_code, project) end def ref diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 3da6ce5c90c..e02d668a1ca 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -12,8 +12,6 @@ module Banzai class SyntaxHighlightFilter < TimeoutHtmlPipelineFilter include OutputSafety - LANG_PARAMS_DELIMITER = ':' - LANG_PARAMS_ATTR = 'data-lang-params' CSS_CLASSES = 'code highlight js-syntax-highlight' CSS = 'pre:not([data-kroki-style]) > code:only-child' @@ -27,10 +25,13 @@ module Banzai doc end - def highlight_node(node) - return if node.parent&.parent.nil? + def highlight_node(code_node) + return if code_node.parent&.parent.nil? - lang, lang_params = parse_lang_params(node) + # maintain existing attributes already added. e.g math and mermaid nodes + pre_node = code_node.parent + + lang = pre_node['data-canonical-lang'] retried = false if use_rouge?(lang) @@ -42,7 +43,7 @@ module Banzai end begin - code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code_node.text), tag: language) rescue StandardError # Gracefully handle syntax highlighter bugs/errors to ensure users can # still access an issue/comment/etc. First, retry with the plain text @@ -57,21 +58,16 @@ module Banzai retry end - # maintain existing attributes already added. e.g math and mermaid nodes - node.children = code - pre_node = node.parent + code_node.children = code # ensure there are no extra children, such as a text node that might # show up from an XSS attack - pre_node.children = node + pre_node.children = code_node - pre_node[:lang] = language pre_node.add_class(CSS_CLASSES) pre_node.add_class("language-#{language}") if language - pre_node.set_attribute('data-canonical-lang', escape_once(lang)) if lang != language - pre_node.set_attribute(LANG_PARAMS_ATTR, escape_once(lang_params)) if lang_params.present? + pre_node.set_attribute('lang', language) pre_node.set_attribute('v-pre', 'true') - pre_node.remove_attribute('data-meta') copy_code_btn = "<copy-code></copy-code>" unless language == 'suggestion' highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code">#{pre_node.to_html}#{copy_code_btn}</div>) @@ -82,33 +78,6 @@ module Banzai private - def parse_lang_params(node) - node = node.parent - - # Commonmarker's FULL_INFO_STRING render option works with the space delimiter. - # But the current behavior of GitLab's markdown renderer is different - it grabs everything as the single - # line, including language and its options. To keep backward compatibility, we have to parse the old format and - # merge with the new one. - # - # Behaviors before separating language and its parameters: - # Old ones: - # "```ruby with options```" -> '<pre><code lang="ruby with options">'. - # "```ruby:with:options```" -> '<pre><code lang="ruby:with:options">'. - # - # New ones: - # "```ruby with options```" -> '<pre><code lang="ruby" data-meta="with options">'. - # "```ruby:with:options```" -> '<pre><code lang="ruby:with:options">'. - - language = node.attr('lang') - - return unless language - - language, language_params = language.split(LANG_PARAMS_DELIMITER, 2) - language_params = [node.attr('data-meta'), language_params].compact.join(' ') - - [language, language_params] - end - # Separate method so it can be instrumented. def lex(lexer, code) lexer.lex(code) diff --git a/lib/banzai/filter/timeout_html_pipeline_filter.rb b/lib/banzai/filter/timeout_html_pipeline_filter.rb index b9b71163ab1..23bbeec8284 100644 --- a/lib/banzai/filter/timeout_html_pipeline_filter.rb +++ b/lib/banzai/filter/timeout_html_pipeline_filter.rb @@ -16,7 +16,6 @@ module Banzai Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { call_with_timeout } rescue Timeout::Error => e class_name = self.class.name.demodulize - timeout_counter.increment(source: class_name) Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name) # we've timed out, but some work may have already been completed, @@ -27,12 +26,6 @@ module Banzai def call_with_timeout raise NotImplementedError end - - private - - def timeout_counter - Gitlab::Metrics.counter(:banzai_filter_timeouts_total, 'Count of the Banzai filters that time out') - end end end end diff --git a/lib/banzai/filter/timeout_text_pipeline_filter.rb b/lib/banzai/filter/timeout_text_pipeline_filter.rb new file mode 100644 index 00000000000..318959065a2 --- /dev/null +++ b/lib/banzai/filter/timeout_text_pipeline_filter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Text Filter that wraps a filter in a Gitlab::RenderTimeout. + # This way partial results can be returned, and the entire pipeline + # is not killed. + # + # This should not be used for any filter that must be allowed to complete, + # like a `ReferenceRedactorFilter` + # + class TimeoutTextPipelineFilter < HTML::Pipeline::TextFilter + RENDER_TIMEOUT = 10.seconds + + def call + Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { call_with_timeout } + rescue Timeout::Error => e + class_name = self.class.name.demodulize + Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name) + + # we've timed out, but some work may have already been completed, + # so go ahead and return the text + @text + end + + def call_with_timeout + raise NotImplementedError + end + end + end +end diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index 34b6ca99e32..6428f71eb8f 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -12,6 +12,7 @@ module Banzai attr_reader :context ISSUE_REFERENCE_TYPE = '@data-reference-type="issue"' + WORK_ITEM_REFERENCE_TYPE = '@data-reference-type="work_item"' MERGE_REQUEST_REFERENCE_TYPE = '@data-reference-type="merge_request"' # context - An instance of Banzai::RenderContext. @@ -41,6 +42,7 @@ module Banzai def parsers [ Banzai::ReferenceParser::IssueParser.new(context), + Banzai::ReferenceParser::WorkItemParser.new(context), Banzai::ReferenceParser::MergeRequestParser.new(context) ] end @@ -53,7 +55,7 @@ module Banzai end def reference_types - [ISSUE_REFERENCE_TYPE, MERGE_REQUEST_REFERENCE_TYPE] + [ISSUE_REFERENCE_TYPE, WORK_ITEM_REFERENCE_TYPE, MERGE_REQUEST_REFERENCE_TYPE] end end end diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index 8764367426c..54306eadd41 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -6,6 +6,7 @@ module Banzai def self.filters FilterArray[ Filter::AsciiDocSanitizationFilter, + Filter::CodeLanguageFilter, Filter::AssetProxyFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 6bd9e65f431..53f938c044f 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -11,6 +11,7 @@ module Banzai # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ + Filter::CodeLanguageFilter, Filter::PlantumlFilter, # Must always be before the SanitizationFilter to prevent XSS attacks Filter::SpacedLinkFilter, @@ -57,6 +58,7 @@ module Banzai Filter::References::ProjectReferenceFilter, Filter::References::DesignReferenceFilter, Filter::References::IssueReferenceFilter, + Filter::References::WorkItemReferenceFilter, Filter::References::ExternalIssueReferenceFilter, Filter::References::MergeRequestReferenceFilter, Filter::References::SnippetReferenceFilter, diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb index 635d4c0884e..cb421ff77b6 100644 --- a/lib/banzai/pipeline/markup_pipeline.rb +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -6,6 +6,7 @@ module Banzai def self.filters @filters ||= FilterArray[ Filter::SanitizationFilter, + Filter::CodeLanguageFilter, Filter::AssetProxyFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb index c51f4976c28..7c2bccc68d9 100644 --- a/lib/banzai/reference_parser/commit_parser.rb +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -5,6 +5,8 @@ module Banzai class CommitParser < BaseParser self.reference_type = :commit + COMMITS_LIMIT = 1000 + def referenced_by(nodes, options = {}) commit_ids = commit_ids_per_project(nodes) projects = find_projects_for_hash_keys(commit_ids) @@ -19,24 +21,11 @@ module Banzai end def find_commits(project, ids) - commits = [] - - return commits unless project.valid_repo? - - ids.each do |id| - commit = project.commit(id) - - commits << commit if commit - end - - commits - end + return [] unless project.valid_repo? - def nodes_visible_to_user(user, nodes) - projects = lazy { projects_for_nodes(nodes) } - user.preloaded_member_roles_for_projects(projects.values) if user + ids = ids.take(COMMITS_LIMIT) - super + project.commits_by(oids: ids) end private diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index 3d09bc83151..fb4a392105f 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -38,13 +38,6 @@ module Banzai range.valid_commits? ? range : nil end - def nodes_visible_to_user(user, nodes) - projects = lazy { projects_for_nodes(nodes) } - user.preloaded_member_roles_for_projects(projects.values) if user - - super - end - private def can_read_reference?(user, ref_project, node) diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 6b1491cc56b..d0e74044bba 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -59,19 +59,30 @@ module Banzai def records_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, - Issue.all.includes( - :author, - :assignees, - { - # These associations are primarily used for checking permissions. - # Eager loading these ensures we don't end up running dozens of - # queries in this process. - project: [:namespace, :project_feature, :route] - } - ), + Issue.all.includes(node_includes), self.class.data_attribute ) end + + private + + def node_includes + includes = [ + :work_item_type, + :namespace, + :author, + :assignees, + { + # These associations are primarily used for checking permissions. + # Eager loading these ensures we don't end up running dozens of + # queries in this process. + project: [:namespace, :project_feature, :route] + } + ] + includes << :milestone if context.options[:extended_preload] + + includes + end end end end diff --git a/lib/banzai/reference_parser/iteration_parser.rb b/lib/banzai/reference_parser/iteration_parser.rb deleted file mode 100644 index 981354aa8e1..00000000000 --- a/lib/banzai/reference_parser/iteration_parser.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module ReferenceParser - # The actual parser is implemented in the EE mixin - class IterationParser < BaseParser - self.reference_type = :iteration - - def references_relation - Iteration - end - - private - - def can_read_reference?(_user, _ref_project, _node) - false - end - end - end -end - -Banzai::ReferenceParser::IterationParser.prepend_mod_with('Banzai::ReferenceParser::IterationParser') diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 3e28f06b783..2bd06e79e96 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -19,17 +19,21 @@ module Banzai end def records_for_nodes(nodes) + node_includes = [ + :author, + :assignees, + { + # These associations are primarily used for checking permissions. + # Eager loading these ensures we don't end up running dozens of + # queries in this process. + target_project: [{ namespace: :route }, :project_feature, :route] + } + ] + node_includes << :milestone if context.options[:extended_preload] + @merge_requests_for_nodes ||= grouped_objects_for_nodes( nodes, - MergeRequest.includes( - :author, - :assignees, - { - # These associations are primarily used for checking permissions. - # Eager loading these ensures we don't end up running dozens of - # queries in this process. - target_project: [{ namespace: :route }, :project_feature, :route] - }), + MergeRequest.includes(node_includes), self.class.data_attribute ) end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index c40ca9dc7cd..48e2bcc9a11 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -81,7 +81,7 @@ module Banzai project = projects[node] user = users[node] - project && user ? project.team.member?(user) : false + project&.member?(user) else true end diff --git a/lib/banzai/reference_parser/work_item_parser.rb b/lib/banzai/reference_parser/work_item_parser.rb new file mode 100644 index 00000000000..1ce0b067687 --- /dev/null +++ b/lib/banzai/reference_parser/work_item_parser.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Banzai + module ReferenceParser + class WorkItemParser < IssueParser + self.reference_type = :work_item + + def records_for_nodes(nodes) + @work_items_for_nodes ||= grouped_objects_for_nodes( + nodes, + WorkItem.all.includes(node_includes), + self.class.data_attribute + ) + end + end + end +end diff --git a/lib/banzai/render_context.rb b/lib/banzai/render_context.rb index e30fc9f469b..a69732a26e2 100644 --- a/lib/banzai/render_context.rb +++ b/lib/banzai/render_context.rb @@ -4,13 +4,14 @@ module Banzai # Object storing the current user, project, and other details used when # parsing Markdown references. class RenderContext - attr_reader :current_user + attr_reader :current_user, :options # default_project - The default project to use for all documents, if any. # current_user - The user viewing the document, if any. - def initialize(default_project = nil, current_user = nil) + def initialize(default_project = nil, current_user = nil, options: {}) @current_user = current_user @projects = Hash.new(default_project) + @options = options end # Associates an HTML document with a Project. diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index b16af78841a..b860fc0c6ae 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -21,10 +21,8 @@ module Banzai cache_key = full_cache_key(cache_key, context[:pipeline]) if cache_key - Gitlab::Metrics.measure(:banzai_cached_render) do - Rails.cache.fetch(cache_key) do - cacheless_render(text, context) - end + Rails.cache.fetch(cache_key) do + cacheless_render(text, context) end else cacheless_render(text, context) @@ -160,40 +158,14 @@ module Banzai def self.cacheless_render(text, context = {}) return text.to_s unless text.present? - real_start = Gitlab::Metrics::System.monotonic_time - cpu_start = Gitlab::Metrics::System.cpu_time - result = render_result(text, context) output = result[:output] - rendered = if output.respond_to?(:to_html) - output.to_html - else - output.to_s - end - - cpu_duration_histogram.observe({}, Gitlab::Metrics::System.cpu_time - cpu_start) - real_duration_histogram.observe({}, Gitlab::Metrics::System.monotonic_time - real_start) - - rendered - end - - def self.real_duration_histogram - Gitlab::Metrics.histogram( - :gitlab_banzai_cacheless_render_real_duration_seconds, - 'Duration of Banzai pipeline rendering in real time', - {}, - [0.01, 0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10.0, 50, 100] - ) - end - - def self.cpu_duration_histogram - Gitlab::Metrics.histogram( - :gitlab_banzai_cacheless_render_cpu_duration_seconds, - 'Duration of Banzai pipeline rendering in cpu time', - {}, - Gitlab::Metrics::EXECUTION_MEASUREMENT_BUCKETS - ) + if output.respond_to?(:to_html) + output.to_html + else + output.to_s + end end def self.full_cache_key(cache_key, pipeline_name) |