Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40

7 files changed, 67 insertions, 109 deletions

diff --git a/spec/lib/banzai/filter/custom_emoji_filter_spec.rb b/spec/lib/banzai/filter/custom_emoji_filter_spec.rb
index ca8c9750e7f..5e76e8164dd 100644
--- a/spec/lib/banzai/filter/custom_emoji_filter_spec.rb
+++ b/spec/lib/banzai/filter/custom_emoji_filter_spec.rb
@@ -10,6 +10,10 @@ RSpec.describe Banzai::Filter::CustomEmojiFilter do
   let_it_be(:custom_emoji) { create(:custom_emoji, name: 'tanuki', group: group) }
   let_it_be(:custom_emoji2) { create(:custom_emoji, name: 'happy_tanuki', group: group, file: 'https://foo.bar/happy.png') }
 
+  it_behaves_like 'emoji filter' do
+    let(:emoji_name) { ':tanuki:' }
+  end
+
   it 'replaces supported name custom emoji' do
     doc = filter('<p>:tanuki:</p>', project: project)
 
@@ -17,25 +21,12 @@ RSpec.describe Banzai::Filter::CustomEmojiFilter do
     expect(doc.css('gl-emoji img').size).to eq 1
   end
 
-  it 'ignores non existent custom emoji' do
-    exp = act = '<p>:foo:</p>'
-    doc = filter(act)
-
-    expect(doc.to_html).to match Regexp.escape(exp)
-  end
-
   it 'correctly uses the custom emoji URL' do
     doc = filter('<p>:tanuki:</p>')
 
     expect(doc.css('img').first.attributes['src'].value).to eq(custom_emoji.file)
   end
 
-  it 'matches with adjacent text' do
-    doc = filter('tanuki (:tanuki:)')
-
-    expect(doc.css('img').size).to eq 1
-  end
-
   it 'matches multiple same custom emoji' do
     doc = filter(':tanuki: :tanuki:')
 
@@ -54,18 +45,6 @@ RSpec.describe Banzai::Filter::CustomEmojiFilter do
     expect(doc.css('img').size).to be 0
   end
 
-  it 'keeps whitespace intact' do
-    doc = filter('This deserves a :tanuki:, big time.')
-
-    expect(doc.to_html).to match(/^This deserves a <gl-emoji.+>, big time\.\z/)
-  end
-
-  it 'does not match emoji in a string' do
-    doc = filter("'2a00:tanuki:100::1'")
-
-    expect(doc.css('gl-emoji').size).to eq 0
-  end
-
   it 'does not do N+1 query' do
     create(:custom_emoji, name: 'party-parrot', group: group)
 
diff --git a/spec/lib/banzai/filter/emoji_filter_spec.rb b/spec/lib/banzai/filter/emoji_filter_spec.rb
index 9005b4401b7..cb0b470eaa1 100644
--- a/spec/lib/banzai/filter/emoji_filter_spec.rb
+++ b/spec/lib/banzai/filter/emoji_filter_spec.rb
@@ -5,6 +5,10 @@ require 'spec_helper'
 RSpec.describe Banzai::Filter::EmojiFilter do
   include FilterSpecHelper
 
+  it_behaves_like 'emoji filter' do
+    let(:emoji_name) { ':+1:' }
+  end
+
   it 'replaces supported name emoji' do
     doc = filter('<p>:heart:</p>')
     expect(doc.css('gl-emoji').first.text).to eq '❤'
@@ -15,12 +19,6 @@ RSpec.describe Banzai::Filter::EmojiFilter do
     expect(doc.css('gl-emoji').first.text).to eq '❤'
   end
 
-  it 'ignores unsupported emoji' do
-    exp = act = '<p>:foo:</p>'
-    doc = filter(act)
-    expect(doc.to_html).to match Regexp.escape(exp)
-  end
-
   it 'ignores unicode versions of trademark, copyright, and registered trademark' do
     exp = act = '<p>™ © ®</p>'
     doc = filter(act)
@@ -65,11 +63,6 @@ RSpec.describe Banzai::Filter::EmojiFilter do
     expect(doc.css('gl-emoji').size).to eq 1
   end
 
-  it 'matches with adjacent text' do
-    doc = filter('+1 (:+1:)')
-    expect(doc.css('gl-emoji').size).to eq 1
-  end
-
   it 'unicode matches with adjacent text' do
     doc = filter('+1 (👍)')
     expect(doc.css('gl-emoji').size).to eq 1
@@ -90,12 +83,6 @@ RSpec.describe Banzai::Filter::EmojiFilter do
     expect(doc.css('gl-emoji').size).to eq 6
   end
 
-  it 'does not match emoji in a string' do
-    doc = filter("'2a00:a4c0:100::1'")
-
-    expect(doc.css('gl-emoji').size).to eq 0
-  end
-
   it 'has a data-name attribute' do
     doc = filter(':-1:')
     expect(doc.css('gl-emoji').first.attr('data-name')).to eq 'thumbsdown'
@@ -106,12 +93,6 @@ RSpec.describe Banzai::Filter::EmojiFilter do
     expect(doc.css('gl-emoji').first.attr('data-unicode-version')).to eq '6.0'
   end
 
-  it 'keeps whitespace intact' do
-    doc = filter('This deserves a :+1:, big time.')
-
-    expect(doc.to_html).to match(/^This deserves a <gl-emoji.+>, big time\.\z/)
-  end
-
   it 'unicode keeps whitespace intact' do
     doc = filter('This deserves a 🎱, big time.')
 
diff --git a/spec/lib/banzai/filter/gollum_tags_filter_spec.rb b/spec/lib/banzai/filter/gollum_tags_filter_spec.rb
index f39b5280490..ec17bb26346 100644
--- a/spec/lib/banzai/filter/gollum_tags_filter_spec.rb
+++ b/spec/lib/banzai/filter/gollum_tags_filter_spec.rb
@@ -22,7 +22,7 @@ RSpec.describe Banzai::Filter::GollumTagsFilter do
                                   path: 'images/image.jpg',
                                   raw_data: '')
       wiki_file = Gitlab::Git::WikiFile.new(gollum_file_double)
-      expect(wiki).to receive(:find_file).with('images/image.jpg').and_return(wiki_file)
+      expect(wiki).to receive(:find_file).with('images/image.jpg', load_content: false).and_return(wiki_file)
 
       tag = '[[images/image.jpg]]'
       doc = filter("See #{tag}", wiki: wiki)
@@ -31,7 +31,7 @@ RSpec.describe Banzai::Filter::GollumTagsFilter do
     end
 
     it 'does not creates img tag if image does not exist' do
-      expect(wiki).to receive(:find_file).with('images/image.jpg').and_return(nil)
+      expect(wiki).to receive(:find_file).with('images/image.jpg', load_content: false).and_return(nil)
 
       tag = '[[images/image.jpg]]'
       doc = filter("See #{tag}", wiki: wiki)
diff --git a/spec/lib/banzai/filter/sanitization_filter_spec.rb b/spec/lib/banzai/filter/sanitization_filter_spec.rb
index bc4b60dfe60..f880fe06ce3 100644
--- a/spec/lib/banzai/filter/sanitization_filter_spec.rb
+++ b/spec/lib/banzai/filter/sanitization_filter_spec.rb
@@ -33,14 +33,14 @@ RSpec.describe Banzai::Filter::SanitizationFilter do
     end
 
     it 'sanitizes `class` attribute from all elements' do
-      act = %q{<pre class="code highlight white c"><code>&lt;span class="k"&gt;def&lt;/span&gt;</code></pre>}
-      exp = %q{<pre><code>&lt;span class="k"&gt;def&lt;/span&gt;</code></pre>}
+      act = %q(<pre class="code highlight white c"><code>&lt;span class="k"&gt;def&lt;/span&gt;</code></pre>)
+      exp = %q(<pre><code>&lt;span class="k"&gt;def&lt;/span&gt;</code></pre>)
       expect(filter(act).to_html).to eq exp
     end
 
     it 'sanitizes `class` attribute from non-highlight spans' do
-      act = %q{<span class="k">def</span>}
-      expect(filter(act).to_html).to eq %q{<span>def</span>}
+      act = %q(<span class="k">def</span>)
+      expect(filter(act).to_html).to eq %q(<span>def</span>)
     end
 
     it 'allows `text-align` property in `style` attribute on table elements' do
@@ -82,12 +82,12 @@ RSpec.describe Banzai::Filter::SanitizationFilter do
     end
 
     it 'allows `span` elements' do
-      exp = act = %q{<span>Hello</span>}
+      exp = act = %q(<span>Hello</span>)
       expect(filter(act).to_html).to eq exp
     end
 
     it 'allows `abbr` elements' do
-      exp = act = %q{<abbr title="HyperText Markup Language">HTML</abbr>}
+      exp = act = %q(<abbr title="HyperText Markup Language">HTML</abbr>)
       expect(filter(act).to_html).to eq exp
     end
 
@@ -132,7 +132,7 @@ RSpec.describe Banzai::Filter::SanitizationFilter do
     end
 
     it 'allows the `data-sourcepos` attribute globally' do
-      exp = %q{<p data-sourcepos="1:1-1:10">foo/bar.md</p>}
+      exp = %q(<p data-sourcepos="1:1-1:10">foo/bar.md</p>)
       act = filter(exp)
 
       expect(act.to_html).to eq exp
@@ -140,41 +140,41 @@ RSpec.describe Banzai::Filter::SanitizationFilter do
 
     describe 'footnotes' do
       it 'allows correct footnote id property on links' do
-        exp = %q{<a href="#fn1" id="fnref1">foo/bar.md</a>}
+        exp = %q(<a href="#fn1" id="fnref1">foo/bar.md</a>)
         act = filter(exp)
 
         expect(act.to_html).to eq exp
       end
 
       it 'allows correct footnote id property on li element' do
-        exp = %q{<ol><li id="fn1">footnote</li></ol>}
+        exp = %q(<ol><li id="fn1">footnote</li></ol>)
         act = filter(exp)
 
         expect(act.to_html).to eq exp
       end
 
       it 'removes invalid id for footnote links' do
-        exp = %q{<a href="#fn1">link</a>}
+        exp = %q(<a href="#fn1">link</a>)
 
         %w[fnrefx test xfnref1].each do |id|
-          act = filter(%Q{<a href="#fn1" id="#{id}">link</a>})
+          act = filter(%(<a href="#fn1" id="#{id}">link</a>))
 
           expect(act.to_html).to eq exp
         end
       end
 
       it 'removes invalid id for footnote li' do
-        exp = %q{<ol><li>footnote</li></ol>}
+        exp = %q(<ol><li>footnote</li></ol>)
 
         %w[fnx test xfn1].each do |id|
-          act = filter(%Q{<ol><li id="#{id}">footnote</li></ol>})
+          act = filter(%(<ol><li id="#{id}">footnote</li></ol>))
 
           expect(act.to_html).to eq exp
         end
       end
 
       it 'allows footnotes numbered higher than 9' do
-        exp = %q{<a href="#fn15" id="fnref15">link</a><ol><li id="fn15">footnote</li></ol>}
+        exp = %q(<a href="#fn15" id="fnref15">link</a><ol><li id="fn15">footnote</li></ol>)
         act = filter(exp)
 
         expect(act.to_html).to eq exp
diff --git a/spec/lib/banzai/filter/video_link_filter_spec.rb b/spec/lib/banzai/filter/video_link_filter_spec.rb
index 32fbc6b687f..ec954aa9163 100644
--- a/spec/lib/banzai/filter/video_link_filter_spec.rb
+++ b/spec/lib/banzai/filter/video_link_filter_spec.rb
@@ -33,6 +33,7 @@ RSpec.describe Banzai::Filter::VideoLinkFilter do
       expect(video.name).to eq 'video'
       expect(video['src']).to eq src
       expect(video['width']).to eq "400"
+      expect(video['preload']).to eq 'metadata'
 
       expect(paragraph.name).to eq 'p'
 
diff --git a/spec/lib/banzai/pipeline/full_pipeline_spec.rb b/spec/lib/banzai/pipeline/full_pipeline_spec.rb
index bcee6f8f65d..989e06a992d 100644
--- a/spec/lib/banzai/pipeline/full_pipeline_spec.rb
+++ b/spec/lib/banzai/pipeline/full_pipeline_spec.rb
@@ -142,5 +142,12 @@ RSpec.describe Banzai::Pipeline::FullPipeline do
 
       expect(output).to include("<span>#</span>#{issue.iid}")
     end
+
+    it 'converts user reference with escaped underscore because of italics' do
+      markdown = '_@test\__'
+      output = described_class.to_html(markdown, project: project)
+
+      expect(output).to include('<em>@test_</em>')
+    end
   end
 end
diff --git a/spec/lib/banzai/pipeline/plain_markdown_pipeline_spec.rb b/spec/lib/banzai/pipeline/plain_markdown_pipeline_spec.rb
index 241d6db4f11..5f31ad0c8f6 100644
--- a/spec/lib/banzai/pipeline/plain_markdown_pipeline_spec.rb
+++ b/spec/lib/banzai/pipeline/plain_markdown_pipeline_spec.rb
@@ -31,11 +31,13 @@ RSpec.describe Banzai::Pipeline::PlainMarkdownPipeline do
       end
     end
 
-    # Test strings taken from https://spec.commonmark.org/0.29/#backslash-escapes
     describe 'CommonMark tests', :aggregate_failures do
-      it 'converts all ASCII punctuation to literals' do
-        markdown = %q(\!\"\#\$\%\&\'\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\_\`\{\|\}\~) + %q[\(\)\\\\]
-        punctuation = %w(! " # $ % &amp; ' * + , - . / : ; &lt; = &gt; ? @ [ \\ ] ^ _ ` { | } ~) + %w[( )]
+      it 'converts all reference punctuation to literals' do
+        reference_chars = Banzai::Filter::MarkdownPreEscapeFilter::REFERENCE_CHARACTERS
+        markdown = reference_chars.split('').map {|char| char.prepend("\\") }.join
+        punctuation = Banzai::Filter::MarkdownPreEscapeFilter::REFERENCE_CHARACTERS.split('')
+        punctuation = punctuation.delete_if {|char| char == '&' }
+        punctuation << '&amp;'
 
         result = described_class.call(markdown, project: project)
         output = result[:output].to_html
@@ -44,57 +46,45 @@ RSpec.describe Banzai::Pipeline::PlainMarkdownPipeline do
         expect(result[:escaped_literals]).to be_truthy
       end
 
-      it 'does not convert other characters to literals' do
-        markdown = %q(\→\A\a\ \3\φ\«)
-        expected = '\→\A\a\ \3\φ\«'
-
-        result = correct_html_included(markdown, expected)
-        expect(result[:escaped_literals]).to be_falsey
-      end
+      it 'ensure we handle all the GitLab reference characters' do
+        reference_chars = ObjectSpace.each_object(Class).map do |klass|
+          next unless klass.included_modules.include?(Referable)
+          next unless klass.respond_to?(:reference_prefix)
+          next unless klass.reference_prefix.length == 1
 
-      describe 'escaped characters are treated as regular characters and do not have their usual Markdown meanings' do
-        where(:markdown, :expected) do
-          %q(\*not emphasized*)              | %q(<span>*</span>not emphasized*)
-          %q(\<br/> not a tag)               | %q(<span>&lt;</span>br/&gt; not a tag)
-          %q!\[not a link](/foo)!            | %q!<span>[</span>not a link](/foo)!
-          %q(\`not code`)                    | %q(<span>`</span>not code`)
-          %q(1\. not a list)                 | %q(1<span>.</span> not a list)
-          %q(\# not a heading)               | %q(<span>#</span> not a heading)
-          %q(\[foo]: /url "not a reference") | %q(<span>[</span>foo]: /url "not a reference")
-          %q(\&ouml; not a character entity) | %q(<span>&amp;</span>ouml; not a character entity)
-        end
+          klass.reference_prefix
+        end.compact
 
-        with_them do
-          it 'keeps them as literals' do
-            correct_html_included(markdown, expected)
-          end
+        reference_chars.all? do |char|
+          Banzai::Filter::MarkdownPreEscapeFilter::REFERENCE_CHARACTERS.include?(char)
         end
       end
 
-      it 'backslash is itself escaped, the following character is not' do
-        markdown = %q(\\\\*emphasis*)
-        expected = %q(<span>\</span><em>emphasis</em>)
+      it 'does not convert non-reference punctuation to spans' do
+        markdown = %q(\"\'\*\+\,\-\.\/\:\;\<\=\>\?\[\]\_\`\{\|\}) + %q[\(\)\\\\]
 
-        correct_html_included(markdown, expected)
+        result = described_class.call(markdown, project: project)
+        output = result[:output].to_html
+
+        expect(output).not_to include('<span>')
+        expect(result[:escaped_literals]).to be_falsey
       end
 
-      it 'backslash at the end of the line is a hard line break' do
-        markdown = <<~MARKDOWN
-          foo\\
-          bar
-        MARKDOWN
-        expected = "foo<br>\nbar"
+      it 'does not convert other characters to literals' do
+        markdown = %q(\→\A\a\ \3\φ\«)
+        expected = '\→\A\a\ \3\φ\«'
 
-        correct_html_included(markdown, expected)
+        result = correct_html_included(markdown, expected)
+        expect(result[:escaped_literals]).to be_falsey
       end
 
       describe 'backslash escapes do not work in code blocks, code spans, autolinks, or raw HTML' do
         where(:markdown, :expected) do
-          %q(`` \[\` ``)       | %q(<code>\[\`</code>)
-          %q(    \[\])         | %Q(<code>\\[\\]\n</code>)
-          %Q(~~~\n\\[\\]\n~~~) | %Q(<code>\\[\\]\n</code>)
-          %q(<http://example.com?find=\*>) | %q(<a href="http://example.com?find=%5C*">http://example.com?find=\*</a>)
-          %q[<a href="/bar\/)">]           | %q[<a href="/bar%5C/)">]
+          %q(`` \@\! ``)       | %q(<code>\@\!</code>)
+          %q(    \@\!)         | %Q(<code>\\@\\!\n</code>)
+          %Q(~~~\n\\@\\!\n~~~) | %Q(<code>\\@\\!\n</code>)
+          %q(<http://example.com?find=\@>) | %q(<a href="http://example.com?find=%5C@">http://example.com?find=\@</a>)
+          %q[<a href="/bar\@)">]           | %q[<a href="/bar%5C@)">]
         end
 
         with_them do
@@ -104,9 +94,9 @@ RSpec.describe Banzai::Pipeline::PlainMarkdownPipeline do
 
       describe 'work in all other contexts, including URLs and link titles, link references, and info strings in fenced code blocks' do
         where(:markdown, :expected) do
-          %q![foo](/bar\* "ti\*tle")!            | %q(<a href="/bar*" title="ti*tle">foo</a>)
-          %Q![foo]\n\n[foo]: /bar\\* "ti\\*tle"! | %q(<a href="/bar*" title="ti*tle">foo</a>)
-          %Q(``` foo\\+bar\nfoo\n```)            | %Q(<code lang="foo+bar">foo\n</code>)
+          %q![foo](/bar\@ "\@title")!            | %q(<a href="/bar@" title="@title">foo</a>)
+          %Q![foo]\n\n[foo]: /bar\\@ "\\@title"! | %q(<a href="/bar@" title="@title">foo</a>)
+          %Q(``` foo\\@bar\nfoo\n```)            | %Q(<code lang="foo@bar">foo\n</code>)
         end
 
         with_them do