Add latest changes from gitlab-org/gitlab@13-2-stable-ee

12 files changed, 178 insertions, 39 deletions

diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index f142333d797..38105e2237c 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -146,16 +146,16 @@ module Banzai
         link_pattern_start = /\A#{link_pattern}/
         link_pattern_anchor = /\A#{link_pattern}\z/
 
-        nodes.each do |node|
+        nodes.each_with_index do |node, index|
           if text_node?(node) && ref_pattern
-            replace_text_when_pattern_matches(node, ref_pattern) do |content|
+            replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
               object_link_filter(content, ref_pattern)
             end
 
           elsif element_node?(node)
             yield_valid_link(node) do |link, inner_html|
               if ref_pattern && link =~ ref_pattern_anchor
-                replace_link_node_with_href(node, link) do
+                replace_link_node_with_href(node, index, link) do
                   object_link_filter(link, ref_pattern, link_content: inner_html)
                 end
 
@@ -165,7 +165,7 @@ module Banzai
               next unless link_pattern
 
               if link == inner_html && inner_html =~ link_pattern_start
-                replace_link_node_with_text(node, link) do
+                replace_link_node_with_text(node, index) do
                   object_link_filter(inner_html, link_pattern, link_reference: true)
                 end
 
@@ -173,7 +173,7 @@ module Banzai
               end
 
               if link =~ link_pattern_anchor
-                replace_link_node_with_href(node, link) do
+                replace_link_node_with_href(node, index, link) do
                   object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true)
                 end
 
diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb
index 02a47556151..5288db3b0cb 100644
--- a/lib/banzai/filter/commit_trailers_filter.rb
+++ b/lib/banzai/filter/commit_trailers_filter.rb
@@ -144,10 +144,7 @@ module Banzai
       end
 
       def data_attributes_from_hash(data = {})
-        data.reject! {|_, value| value.nil?}
-        data.map do |key, value|
-          [%(data-#{key.to_s.dasherize}), value]
-        end.to_h
+        data.compact.transform_keys { |key| %(data-#{key.to_s.dasherize}) }
       end
     end
   end
diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb
index 74bc102320c..fcf4863ab4f 100644
--- a/lib/banzai/filter/external_issue_reference_filter.rb
+++ b/lib/banzai/filter/external_issue_reference_filter.rb
@@ -34,16 +34,16 @@ module Banzai
         ref_pattern = issue_reference_pattern
         ref_start_pattern = /\A#{ref_pattern}\z/
 
-        each_node do |node|
+        nodes.each_with_index do |node, index|
           if text_node?(node)
-            replace_text_when_pattern_matches(node, ref_pattern) do |content|
+            replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
               issue_link_filter(content)
             end
 
           elsif element_node?(node)
             yield_valid_link(node) do |link, inner_html|
               if link =~ ref_start_pattern
-                replace_link_node_with_href(node, link) do
+                replace_link_node_with_href(node, index, link) do
                   issue_link_filter(link, link_content: inner_html)
                 end
               end
diff --git a/lib/banzai/filter/inline_cluster_metrics_filter.rb b/lib/banzai/filter/inline_cluster_metrics_filter.rb
new file mode 100644
index 00000000000..5ef68388ea9
--- /dev/null
+++ b/lib/banzai/filter/inline_cluster_metrics_filter.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    class InlineClusterMetricsFilter < ::Banzai::Filter::InlineEmbedsFilter
+      def embed_params(node)
+        url = node['href']
+        @query_params = query_params(url)
+        return unless [:group, :title, :y_label].all? do |param|
+          @query_params.include?(param)
+        end
+
+        link_pattern.match(url) { |m| m.named_captures }.symbolize_keys
+      end
+
+      def xpath_search
+        "descendant-or-self::a[contains(@href,'clusters') and \
+          starts-with(@href, '#{::Gitlab.config.gitlab.url}')]"
+      end
+
+      def link_pattern
+        ::Gitlab::Metrics::Dashboard::Url.clusters_regex
+      end
+
+      def metrics_dashboard_url(params)
+        ::Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_cluster_url(
+          params[:namespace],
+          params[:project],
+          params[:cluster_id],
+          # Only Project clusters are supported for now
+          # admin and group cluster types may be supported in the future
+          cluster_type: :project,
+          embedded: true,
+          format: :json,
+          **@query_params
+        )
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/inline_metrics_redactor_filter.rb b/lib/banzai/filter/inline_metrics_redactor_filter.rb
index 75bd3325bd4..7f98a52d421 100644
--- a/lib/banzai/filter/inline_metrics_redactor_filter.rb
+++ b/lib/banzai/filter/inline_metrics_redactor_filter.rb
@@ -77,6 +77,10 @@ module Banzai
           Route.new(
             ::Gitlab::Metrics::Dashboard::Url.grafana_regex,
             :read_project
+          ),
+          Route.new(
+            ::Gitlab::Metrics::Dashboard::Url.clusters_regex,
+            :read_cluster
           )
         ]
       end
diff --git a/lib/banzai/filter/jira_import/adf_to_commonmark_filter.rb b/lib/banzai/filter/jira_import/adf_to_commonmark_filter.rb
new file mode 100644
index 00000000000..3db2244d641
--- /dev/null
+++ b/lib/banzai/filter/jira_import/adf_to_commonmark_filter.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    module JiraImport
+      # Uses Kramdown to convert from the Atlassian Document Format (json)
+      # into CommonMark
+      # @see https://developer.atlassian.com/cloud/jira/platform/apis/document/structure/
+      class AdfToCommonmarkFilter < HTML::Pipeline::TextFilter
+        def initialize(text, context = nil, result = nil)
+          super(text, context, result)
+        end
+
+        def call
+          Kramdown::Document.new(@text, input: 'AtlassianDocumentFormat', html_tables: true).to_commonmark
+        rescue ::Kramdown::Error => e
+          # If we get an error, then just return the original text so at
+          # least the user knows something went wrong
+          "#{e.message}\n\n#{@text}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/project_reference_filter.rb b/lib/banzai/filter/project_reference_filter.rb
index 292d4b1d56c..50e23460cb8 100644
--- a/lib/banzai/filter/project_reference_filter.rb
+++ b/lib/banzai/filter/project_reference_filter.rb
@@ -27,15 +27,15 @@ module Banzai
         ref_pattern = Project.markdown_reference_pattern
         ref_pattern_start = /\A#{ref_pattern}\z/
 
-        nodes.each do |node|
+        nodes.each_with_index do |node, index|
           if text_node?(node)
-            replace_text_when_pattern_matches(node, ref_pattern) do |content|
+            replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
               project_link_filter(content)
             end
           elsif element_node?(node)
             yield_valid_link(node) do |link, inner_html|
               if link =~ ref_pattern_start
-                replace_link_node_with_href(node, link) do
+                replace_link_node_with_href(node, index, link) do
                   project_link_filter(link, link_content: inner_html)
                 end
               end
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 9e932ccf9f8..9032ca6ddc6 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -16,6 +16,23 @@ module Banzai
 
       class << self
         attr_accessor :reference_type
+
+        def call(doc, context = nil, result = nil)
+          new(doc, context, result).call_and_update_nodes
+        end
+      end
+
+      def initialize(doc, context = nil, result = nil)
+        super
+
+        if update_nodes_enabled?
+          @new_nodes = {}
+          @nodes = self.result[:reference_filter_nodes]
+        end
+      end
+
+      def call_and_update_nodes
+        update_nodes_enabled? ? with_update_nodes { call } : call
       end
 
       # Returns a data attribute String to attach to a reference link
@@ -89,11 +106,6 @@ module Banzai
       def each_node
         return to_enum(__method__) unless block_given?
 
-        query = %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})]
-        | descendant-or-self::a[
-          not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "")
-        ]}
-
         doc.xpath(query).each do |node|
           yield node
         end
@@ -114,25 +126,25 @@ module Banzai
         yield link, inner_html
       end
 
-      def replace_text_when_pattern_matches(node, pattern)
+      def replace_text_when_pattern_matches(node, index, pattern)
         return unless node.text =~ pattern
 
         content = node.to_html
         html = yield content
 
-        node.replace(html) unless content == html
+        replace_text_with_html(node, index, html) unless html == content
       end
 
-      def replace_link_node_with_text(node, link)
+      def replace_link_node_with_text(node, index)
         html = yield
 
-        node.replace(html) unless html == node.text
+        replace_text_with_html(node, index, html) unless html == node.text
       end
 
-      def replace_link_node_with_href(node, link)
+      def replace_link_node_with_href(node, index, link)
         html = yield
 
-        node.replace(html) unless html == link
+        replace_text_with_html(node, index, html) unless html == link
       end
 
       def text_node?(node)
@@ -145,9 +157,62 @@ module Banzai
 
       private
 
+      def query
+        @query ||= %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})]
+        | descendant-or-self::a[
+          not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "")
+        ]}
+      end
+
+      def replace_text_with_html(node, index, html)
+        if update_nodes_enabled?
+          replace_and_update_new_nodes(node, index, html)
+        else
+          node.replace(html)
+        end
+      end
+
+      def replace_and_update_new_nodes(node, index, html)
+        previous_node = node.previous
+        next_node = node.next
+        parent_node = node.parent
+        # Unfortunately node.replace(html) returns re-parented nodes, not the actual replaced nodes in the doc
+        # We need to find the actual nodes in the doc that were replaced
+        node.replace(html)
+        @new_nodes[index] = []
+
+        # We replaced node with new nodes, so we find first new node. If previous_node is nil, we take first parent child
+        new_node = previous_node ? previous_node.next : parent_node&.children&.first
+
+        # We iterate from first to last replaced node and store replaced nodes in @new_nodes
+        while new_node && new_node != next_node
+          @new_nodes[index] << new_node.xpath(query)
+          new_node = new_node.next
+        end
+
+        @new_nodes[index].flatten!
+      end
+
       def only_path?
         context[:only_path]
       end
+
+      def with_update_nodes
+        @new_nodes = {}
+        yield.tap { update_nodes! }
+      end
+
+      # Once Filter completes replacing nodes, we update nodes with @new_nodes
+      def update_nodes!
+        @new_nodes.sort_by { |index, _new_nodes| -index }.each do |index, new_nodes|
+          nodes[index, 1] = new_nodes
+        end
+        result[:reference_filter_nodes] = nodes
+      end
+
+      def update_nodes_enabled?
+        Feature.enabled?(:update_nodes_for_banzai_reference_filter, project)
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb
index a2c8e92e560..b362607aed2 100644
--- a/lib/banzai/filter/table_of_contents_filter.rb
+++ b/lib/banzai/filter/table_of_contents_filter.rb
@@ -17,7 +17,7 @@ module Banzai
     #   :toc - String containing Table of Contents data as a `ul` element with
     #          `li` child elements.
     class TableOfContentsFilter < HTML::Pipeline::Filter
-      PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze
+      include Gitlab::Utils::Markdown
 
       def call
         return doc if context[:no_header_anchors]
@@ -29,14 +29,7 @@ module Banzai
 
         doc.css('h1, h2, h3, h4, h5, h6').each do |node|
           if header_content = node.children.first
-            id = node
-              .text
-              .strip
-              .downcase
-              .gsub(PUNCTUATION_REGEXP, '') # remove punctuation
-              .tr(' ', '-') # replace spaces with dash
-              .squeeze('-') # replace multiple dashes with one
-              .gsub(/\A(\d+)\z/, 'anchor-\1') # digits-only hrefs conflict with issue refs
+            id = string_to_anchor(node.text)
 
             uniq = headers[id] > 0 ? "-#{headers[id]}" : ''
             headers[id] += 1
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index 9268ff1a827..262385524f4 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -31,15 +31,15 @@ module Banzai
         ref_pattern = User.reference_pattern
         ref_pattern_start = /\A#{ref_pattern}\z/
 
-        nodes.each do |node|
+        nodes.each_with_index do |node, index|
           if text_node?(node)
-            replace_text_when_pattern_matches(node, ref_pattern) do |content|
+            replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
               user_link_filter(content)
             end
           elsif element_node?(node)
             yield_valid_link(node) do |link, inner_html|
               if link =~ ref_pattern_start
-                replace_link_node_with_href(node, link) do
+                replace_link_node_with_href(node, index, link) do
                   user_link_filter(link, link_content: inner_html)
                 end
               end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 2ea5fd3388a..10ac813ea15 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -48,7 +48,8 @@ module Banzai
       def self.metrics_filters
         [
           Filter::InlineMetricsFilter,
-          Filter::InlineGrafanaMetricsFilter
+          Filter::InlineGrafanaMetricsFilter,
+          Filter::InlineClusterMetricsFilter
         ]
       end
 
diff --git a/lib/banzai/pipeline/jira_import/adf_commonmark_pipeline.rb b/lib/banzai/pipeline/jira_import/adf_commonmark_pipeline.rb
new file mode 100644
index 00000000000..8af0279673c
--- /dev/null
+++ b/lib/banzai/pipeline/jira_import/adf_commonmark_pipeline.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Pipeline
+    module JiraImport
+      class AdfCommonmarkPipeline < BasePipeline
+        def self.filters
+          FilterArray[
+            Filter::JiraImport::AdfToCommonmarkFilter
+          ]
+        end
+      end
+    end
+  end
+end