Use nodes and marks to power Copy-as-GFM

The spec needed to be updated because in some cases the resulting
Markdown is slightly different, though equally valid.

5 files changed, 174 insertions, 391 deletions

diff --git a/app/assets/javascripts/behaviors/markdown/copy_as_gfm.js b/app/assets/javascripts/behaviors/markdown/copy_as_gfm.js
index fe02096d903..947d019c725 100644
--- a/app/assets/javascripts/behaviors/markdown/copy_as_gfm.js
+++ b/app/assets/javascripts/behaviors/markdown/copy_as_gfm.js
@@ -1,320 +1,8 @@
-/* eslint-disable object-shorthand, no-unused-vars, no-use-before-define, no-restricted-syntax, guard-for-in, no-continue */
-
 import $ from 'jquery';
-import _ from 'underscore';
-import { insertText, getSelectedFragment, nodeMatchesSelector } from '~/lib/utils/common_utils';
-import { placeholderImage } from '~/lazy_loader';
-
-const gfmRules = {
-  // The filters referenced in lib/banzai/pipeline/gfm_pipeline.rb convert
-  // GitLab Flavored Markdown (GFM) to HTML.
-  // These handlers consequently convert that same HTML to GFM to be copied to the clipboard.
-  // Every filter in lib/banzai/pipeline/gfm_pipeline.rb that generates HTML
-  // from GFM should have a handler here, in reverse order.
-  // The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb.
-  InlineDiffFilter: {
-    'span.idiff.addition'(el, text) {
-      return `{+${text}+}`;
-    },
-    'span.idiff.deletion'(el, text) {
-      return `{-${text}-}`;
-    },
-  },
-  TaskListFilter: {
-    'input[type=checkbox].task-list-item-checkbox'(el) {
-      return `[${el.checked ? 'x' : ' '}]`;
-    },
-  },
-  ReferenceFilter: {
-    '.tooltip'(el) {
-      return '';
-    },
-    'a.gfm:not([data-link=true])'(el, text) {
-      return el.dataset.original || text;
-    },
-  },
-  AutolinkFilter: {
-    a(el, text) {
-      // Fallback on the regular MarkdownFilter's `a` handler.
-      if (text !== el.getAttribute('href')) return false;
-
-      return text;
-    },
-  },
-  TableOfContentsFilter: {
-    'ul.section-nav'(el) {
-      return '[[_TOC_]]';
-    },
-  },
-  EmojiFilter: {
-    'img.emoji'(el) {
-      return el.getAttribute('alt');
-    },
-    'gl-emoji'(el) {
-      return `:${el.getAttribute('data-name')}:`;
-    },
-  },
-  ImageLinkFilter: {
-    'a.no-attachment-icon'(el, text) {
-      return text;
-    },
-  },
-  ImageLazyLoadFilter: {
-    img(el, text) {
-      return `![${el.getAttribute('alt')}](${el.getAttribute('src')})`;
-    },
-  },
-  VideoLinkFilter: {
-    '.video-container'(el) {
-      const videoEl = el.querySelector('video');
-      if (!videoEl) return false;
-
-      return CopyAsGFM.nodeToGFM(videoEl);
-    },
-    video(el) {
-      return `![${el.dataset.title}](${el.getAttribute('src')})`;
-    },
-  },
-  MermaidFilter: {
-    'svg.mermaid'(el, text) {
-      const sourceEl = el.querySelector('text.source');
-      if (!sourceEl) return false;
-
-      return `\`\`\`mermaid\n${CopyAsGFM.nodeToGFM(sourceEl)}\n\`\`\``;
-    },
-    'svg.mermaid style, svg.mermaid g'(el, text) {
-      // We don't want to include the content of these elements in the copied text.
-      return '';
-    },
-  },
-  MathFilter: {
-    'pre.code.math[data-math-style=display]'(el, text) {
-      return `\`\`\`math\n${text.trim()}\n\`\`\``;
-    },
-    'code.code.math[data-math-style=inline]'(el, text) {
-      return `$\`${text}\`$`;
-    },
-    'span.katex-display span.katex-mathml'(el) {
-      const mathAnnotation = el.querySelector('annotation[encoding="application/x-tex"]');
-      if (!mathAnnotation) return false;
-
-      return `\`\`\`math\n${CopyAsGFM.nodeToGFM(mathAnnotation)}\n\`\`\``;
-    },
-    'span.katex-mathml'(el) {
-      const mathAnnotation = el.querySelector('annotation[encoding="application/x-tex"]');
-      if (!mathAnnotation) return false;
-
-      return `$\`${CopyAsGFM.nodeToGFM(mathAnnotation)}\`$`;
-    },
-    'span.katex-html'(el) {
-      // We don't want to include the content of this element in the copied text.
-      return '';
-    },
-    'annotation[encoding="application/x-tex"]'(el, text) {
-      return text.trim();
-    },
-  },
-  SanitizationFilter: {
-    'a[name]:not([href]):empty'(el) {
-      return el.outerHTML;
-    },
-    dl(el, text) {
-      let lines = text
-        .replace(/\n\n/g, '\n')
-        .trim()
-        .split('\n');
-      // Add two spaces to the front of subsequent list items lines,
-      // or leave the line entirely blank.
-      lines = lines.map(l => {
-        const line = l.trim();
-        if (line.length === 0) return '';
-
-        return `  ${line}`;
-      });
-
-      return `<dl>\n${lines.join('\n')}\n</dl>\n`;
-    },
-    'dt, dd, summary, details'(el, text) {
-      const tag = el.nodeName.toLowerCase();
-      return `<${tag}>${text}</${tag}>\n`;
-    },
-    'sup, sub, kbd, q, samp, var, ruby, rt, rp, abbr'(el, text) {
-      const tag = el.nodeName.toLowerCase();
-      return `<${tag}>${text}</${tag}>`;
-    },
-  },
-  SyntaxHighlightFilter: {
-    'pre.code.highlight'(el, t) {
-      const text = t.trimRight();
-
-      let lang = el.getAttribute('lang');
-      if (!lang || lang === 'plaintext') {
-        lang = '';
-      }
-
-      // Prefixes lines with 4 spaces if the code contains triple backticks
-      if (lang === '' && text.match(/^```/gm)) {
-        return text
-          .split('\n')
-          .map(l => {
-            const line = l.trim();
-            if (line.length === 0) return '';
-
-            return `    ${line}`;
-          })
-          .join('\n');
-      }
-
-      return `\`\`\`${lang}\n${text}\n\`\`\``;
-    },
-    'pre > code'(el, text) {
-      // Don't wrap code blocks in ``
-      return text;
-    },
-  },
-  MarkdownFilter: {
-    br(el) {
-      // Two spaces at the end of a line are turned into a BR
-      return '  ';
-    },
-    code(el, text) {
-      let backtickCount = 1;
-      const backtickMatch = text.match(/`+/);
-      if (backtickMatch) {
-        backtickCount = backtickMatch[0].length + 1;
-      }
-
-      const backticks = Array(backtickCount + 1).join('`');
-      const spaceOrNoSpace = backtickCount > 1 ? ' ' : '';
-
-      return backticks + spaceOrNoSpace + text.trim() + spaceOrNoSpace + backticks;
-    },
-    blockquote(el, text) {
-      return text
-        .trim()
-        .split('\n')
-        .map(s => `> ${s}`.trim())
-        .join('\n');
-    },
-    img(el) {
-      const imageSrc = el.src;
-      const imageUrl = imageSrc && imageSrc !== placeholderImage ? imageSrc : el.dataset.src || '';
-      return `![${el.getAttribute('alt')}](${imageUrl})`;
-    },
-    'a.anchor'(el, text) {
-      // Don't render a Markdown link for the anchor link inside a heading
-      return text;
-    },
-    a(el, text) {
-      return `[${text}](${el.getAttribute('href')})`;
-    },
-    li(el, text) {
-      const lines = text.trim().split('\n');
-      const firstLine = `- ${lines.shift()}`;
-      // Add four spaces to the front of subsequent list items lines,
-      // or leave the line entirely blank.
-      const nextLines = lines.map(s => {
-        if (s.trim().length === 0) return '';
-
-        return `    ${s}`;
-      });
-
-      return `${firstLine}\n${nextLines.join('\n')}`;
-    },
-    ul(el, text) {
-      return text;
-    },
-    ol(el, text) {
-      // LIs get a `- ` prefix by default, which we replace by `1. ` for ordered lists.
-      return text.replace(/^- /gm, '1. ');
-    },
-    h1(el, text) {
-      return `# ${text.trim()}\n`;
-    },
-    h2(el, text) {
-      return `## ${text.trim()}\n`;
-    },
-    h3(el, text) {
-      return `### ${text.trim()}\n`;
-    },
-    h4(el, text) {
-      return `#### ${text.trim()}\n`;
-    },
-    h5(el, text) {
-      return `##### ${text.trim()}\n`;
-    },
-    h6(el, text) {
-      return `###### ${text.trim()}\n`;
-    },
-    strong(el, text) {
-      return `**${text}**`;
-    },
-    em(el, text) {
-      return `_${text}_`;
-    },
-    del(el, text) {
-      return `~~${text}~~`;
-    },
-    hr(el) {
-      // extra leading \n is to ensure that there is a blank line between
-      // a list followed by an hr, otherwise this breaks old redcarpet rendering
-      return '\n-----\n';
-    },
-    p(el, text) {
-      return `${text.trim()}\n`;
-    },
-    table(el) {
-      const theadEl = el.querySelector('thead');
-      const tbodyEl = el.querySelector('tbody');
-      if (!theadEl || !tbodyEl) return false;
-
-      const theadText = CopyAsGFM.nodeToGFM(theadEl);
-      const tbodyText = CopyAsGFM.nodeToGFM(tbodyEl);
-
-      return [theadText, tbodyText].join('\n');
-    },
-    thead(el, text) {
-      const cells = _.map(el.querySelectorAll('th'), cell => {
-        let chars = CopyAsGFM.nodeToGFM(cell).length + 2;
-
-        let before = '';
-        let after = '';
-        const alignment = cell.align || cell.style.textAlign;
-
-        switch (alignment) {
-          case 'center':
-            before = ':';
-            after = ':';
-            chars -= 2;
-            break;
-          case 'right':
-            after = ':';
-            chars -= 1;
-            break;
-          default:
-            break;
-        }
-
-        chars = Math.max(chars, 3);
-
-        const middle = Array(chars + 1).join('-');
-
-        return before + middle + after;
-      });
-
-      const separatorRow = `|${cells.join('|')}|`;
-
-      return [text, separatorRow].join('\n');
-    },
-    tr(el) {
-      const cellEls = el.querySelectorAll('td, th');
-      if (cellEls.length === 0) return false;
-
-      const cells = _.map(cellEls, cell => CopyAsGFM.nodeToGFM(cell));
-      return `| ${cells.join(' | ')} |`;
-    },
-  },
-};
+import { DOMParser } from 'prosemirror-model';
+import { getSelectedFragment } from '~/lib/utils/common_utils';
+import schema from './schema';
+import markdownSerializer from './serializer';
 
 export class CopyAsGFM {
   constructor() {
@@ -347,8 +35,13 @@ export class CopyAsGFM {
     e.preventDefault();
     e.stopPropagation();
 
+    const div = document.createElement('div');
+    div.appendChild(el.cloneNode(true));
+    const html = div.innerHTML;
+
     clipboardData.setData('text/plain', el.textContent);
     clipboardData.setData('text/x-gfm', this.nodeToGFM(el));
+    clipboardData.setData('text/html', html);
   }
 
   static pasteGFM(e) {
@@ -361,7 +54,7 @@ export class CopyAsGFM {
 
     e.preventDefault();
 
-    window.gl.utils.insertText(e.target, (textBefore, textAfter) => {
+    window.gl.utils.insertText(e.target, textBefore => {
       // If the text before the cursor contains an odd number of backticks,
       // we are either inside an inline code span that starts with 1 backtick
       // or a code block that starts with 3 backticks.
@@ -443,75 +136,12 @@ export class CopyAsGFM {
     return codeElement;
   }
 
-  static nodeToGFM(node, respectWhitespaceParam = false) {
-    if (node.nodeType === Node.COMMENT_NODE) {
-      return '';
-    }
-
-    if (node.nodeType === Node.TEXT_NODE) {
-      return node.textContent;
-    }
-
-    const respectWhitespace =
-      respectWhitespaceParam || (node.nodeName === 'PRE' || node.nodeName === 'CODE');
-
-    const text = this.innerGFM(node, respectWhitespace);
-
-    if (node.nodeType === Node.DOCUMENT_FRAGMENT_NODE) {
-      return text;
-    }
-
-    for (const filter in gfmRules) {
-      const rules = gfmRules[filter];
-
-      for (const selector in rules) {
-        const func = rules[selector];
-
-        if (!nodeMatchesSelector(node, selector)) continue;
-
-        let result;
-        if (func.length === 2) {
-          // if `func` takes 2 arguments, it depends on text.
-          // if there is no text, we don't need to generate GFM for this node.
-          if (text.length === 0) continue;
-
-          result = func(node, text);
-        } else {
-          result = func(node);
-        }
-
-        if (result === false) continue;
-
-        return result;
-      }
-    }
-
-    return text;
-  }
-
-  static innerGFM(parentNode, respectWhitespace = false) {
-    const nodes = parentNode.childNodes;
-
-    const clonedParentNode = parentNode.cloneNode(true);
-    const clonedNodes = Array.prototype.slice.call(clonedParentNode.childNodes, 0);
-
-    for (let i = 0; i < nodes.length; i += 1) {
-      const node = nodes[i];
-      const clonedNode = clonedNodes[i];
-
-      const text = this.nodeToGFM(node, respectWhitespace);
-
-      // `clonedNode.replaceWith(text)` is not yet widely supported
-      clonedNode.parentNode.replaceChild(document.createTextNode(text), clonedNode);
-    }
-
-    let nodeText = clonedParentNode.innerText || clonedParentNode.textContent;
-
-    if (!respectWhitespace) {
-      nodeText = nodeText.trim();
-    }
+  static nodeToGFM(node) {
+    const wrapEl = document.createElement('div');
+    wrapEl.appendChild(node.cloneNode(true));
+    const doc = DOMParser.fromSchema(schema).parse(wrapEl);
 
-    return nodeText;
+    return markdownSerializer.serialize(doc);
   }
 }
 
diff --git a/app/assets/javascripts/behaviors/markdown/editor_extensions.js b/app/assets/javascripts/behaviors/markdown/editor_extensions.js
new file mode 100644
index 00000000000..47e5fc65c48
--- /dev/null
+++ b/app/assets/javascripts/behaviors/markdown/editor_extensions.js
@@ -0,0 +1,106 @@
+import Doc from './nodes/doc';
+import Paragraph from './nodes/paragraph';
+import Text from './nodes/text';
+
+import Blockquote from './nodes/blockquote';
+import CodeBlock from './nodes/code_block';
+import HardBreak from './nodes/hard_break';
+import Heading from './nodes/heading';
+import HorizontalRule from './nodes/horizontal_rule';
+import Image from './nodes/image';
+
+import Table from './nodes/table';
+import TableHead from './nodes/table_head';
+import TableBody from './nodes/table_body';
+import TableHeaderRow from './nodes/table_header_row';
+import TableRow from './nodes/table_row';
+import TableCell from './nodes/table_cell';
+
+import Emoji from './nodes/emoji';
+import Reference from './nodes/reference';
+
+import TableOfContents from './nodes/table_of_contents';
+import Video from './nodes/video';
+
+import BulletList from './nodes/bullet_list';
+import OrderedList from './nodes/ordered_list';
+import ListItem from './nodes/list_item';
+
+import DescriptionList from './nodes/description_list';
+import DescriptionTerm from './nodes/description_term';
+import DescriptionDetails from './nodes/description_details';
+
+import TaskList from './nodes/task_list';
+import OrderedTaskList from './nodes/ordered_task_list';
+import TaskListItem from './nodes/task_list_item';
+
+import Summary from './nodes/summary';
+import Details from './nodes/details';
+
+import Bold from './marks/bold';
+import Italic from './marks/italic';
+import Strike from './marks/strike';
+import InlineDiff from './marks/inline_diff';
+
+import Link from './marks/link';
+import Code from './marks/code';
+import MathMark from './marks/math';
+import InlineHTML from './marks/inline_html';
+
+// The filters referenced in lib/banzai/pipeline/gfm_pipeline.rb transform
+// GitLab Flavored Markdown (GFM) to HTML.
+// The nodes and marks referenced here transform that same HTML to GFM to be copied to the clipboard.
+// Every filter in lib/banzai/pipeline/gfm_pipeline.rb that generates HTML
+// from GFM should have a node or mark here.
+// The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb.
+
+export default [
+  new Doc(),
+  new Paragraph(),
+  new Text(),
+
+  new Blockquote(),
+  new CodeBlock(),
+  new HardBreak(),
+  new Heading({ maxLevel: 6 }),
+  new HorizontalRule(),
+  new Image(),
+
+  new Table(),
+  new TableHead(),
+  new TableBody(),
+  new TableHeaderRow(),
+  new TableRow(),
+  new TableCell(),
+
+  new Emoji(),
+  new Reference(),
+
+  new TableOfContents(),
+  new Video(),
+
+  new BulletList(),
+  new OrderedList(),
+  new ListItem(),
+
+  new DescriptionList(),
+  new DescriptionTerm(),
+  new DescriptionDetails(),
+
+  new TaskList(),
+  new OrderedTaskList(),
+  new TaskListItem(),
+
+  new Summary(),
+  new Details(),
+
+  new Bold(),
+  new Italic(),
+  new Strike(),
+  new InlineDiff(),
+
+  new Link(),
+  new Code(),
+  new MathMark(),
+  new InlineHTML(),
+];
diff --git a/app/assets/javascripts/behaviors/markdown/schema.js b/app/assets/javascripts/behaviors/markdown/schema.js
new file mode 100644
index 00000000000..163182ab778
--- /dev/null
+++ b/app/assets/javascripts/behaviors/markdown/schema.js
@@ -0,0 +1,24 @@
+import { Schema } from 'prosemirror-model';
+import editorExtensions from './editor_extensions';
+
+const nodes = editorExtensions
+  .filter(extension => extension.type === 'node')
+  .reduce(
+    (ns, { name, schema }) => ({
+      ...ns,
+      [name]: schema,
+    }),
+    {},
+  );
+
+const marks = editorExtensions
+  .filter(extension => extension.type === 'mark')
+  .reduce(
+    (ms, { name, schema }) => ({
+      ...ms,
+      [name]: schema,
+    }),
+    {},
+  );
+
+export default new Schema({ nodes, marks });
diff --git a/app/assets/javascripts/behaviors/markdown/serializer.js b/app/assets/javascripts/behaviors/markdown/serializer.js
new file mode 100644
index 00000000000..70dbd8bd206
--- /dev/null
+++ b/app/assets/javascripts/behaviors/markdown/serializer.js
@@ -0,0 +1,24 @@
+import { MarkdownSerializer } from 'prosemirror-markdown';
+import editorExtensions from './editor_extensions';
+
+const nodes = editorExtensions
+  .filter(extension => extension.type === 'node')
+  .reduce(
+    (ns, { name, toMarkdown }) => ({
+      ...ns,
+      [name]: toMarkdown,
+    }),
+    {},
+  );
+
+const marks = editorExtensions
+  .filter(extension => extension.type === 'mark')
+  .reduce(
+    (ms, { name, toMarkdown }) => ({
+      ...ms,
+      [name]: toMarkdown,
+    }),
+    {},
+  );
+
+export default new MarkdownSerializer(nodes, marks);
diff --git a/app/assets/javascripts/behaviors/shortcuts/shortcuts_issuable.js b/app/assets/javascripts/behaviors/shortcuts/shortcuts_issuable.js
index 2918e1486a7..0eb067d4963 100644
--- a/app/assets/javascripts/behaviors/shortcuts/shortcuts_issuable.js
+++ b/app/assets/javascripts/behaviors/shortcuts/shortcuts_issuable.js
@@ -1,6 +1,5 @@
 import $ from 'jquery';
 import Mousetrap from 'mousetrap';
-import _ from 'underscore';
 import Sidebar from '../../right_sidebar';
 import Shortcuts from './shortcuts';
 import { CopyAsGFM } from '../markdown/copy_as_gfm';
@@ -63,18 +62,18 @@ export default class ShortcutsIssuable extends Shortcuts {
     }
 
     const el = CopyAsGFM.transformGFMSelection(documentFragment.cloneNode(true));
-    const selected = CopyAsGFM.nodeToGFM(el);
+    const blockquoteEl = document.createElement('blockquote');
+    blockquoteEl.appendChild(el);
+    const text = CopyAsGFM.nodeToGFM(blockquoteEl);
 
-    if (selected.trim() === '') {
+    if (text.trim() === '') {
       return false;
     }
 
-    const quote = _.map(selected.split('\n'), val => `${`> ${val}`.trim()}\n`);
-
     // If replyField already has some content, add a newline before our quote
     const separator = ($replyField.val().trim() !== '' && '\n\n') || '';
     $replyField
-      .val((a, current) => `${current}${separator}${quote.join('')}\n`)
+      .val((a, current) => `${current}${separator}${text}\n\n`)
       .trigger('input')
       .trigger('change');