Merge branch 'himkp-serializer-tests' into 'master'

Improve serialization of content editor extensions

See merge request gitlab-org/gitlab!68877

app/assets/javascripts/content_editor/extensions/blockquote.js

-export { Blockquote as default } from '@tiptap/extension-blockquote';
+import { Blockquote } from '@tiptap/extension-blockquote';
+import { wrappingInputRule } from 'prosemirror-inputrules';
+import { getParents } from '~/lib/utils/dom_utils';
+import { getMarkdownSource } from '../services/markdown_sourcemap';
+
+export const multilineInputRegex = /^\s*>>>\s$/gm;
+
+export default Blockquote.extend({
+  addAttributes() {
+    return {
+      ...this.parent?.(),
+
+      multiline: {
+        default: false,
+        parseHTML: (element) => {
+          const source = getMarkdownSource(element);
+          const parentsIncludeBlockquote = getParents(element).some(
+            (p) => p.nodeName.toLowerCase() === 'blockquote',
+          );
+
+          return {
+            multiline: source && !source.startsWith('>') && !parentsIncludeBlockquote,
+          };
+        },
+      },
+    };
+  },
+
+  addInputRules() {
+    return [
+      ...this.parent?.(),
+      wrappingInputRule(multilineInputRegex, this.type, () => ({ multiline: true })),
+    ];
+  },
+});

app/assets/javascripts/content_editor/extensions/ordered_list.js

-export { OrderedList as default } from '@tiptap/extension-ordered-list';
+import { OrderedList } from '@tiptap/extension-ordered-list';
+import { getMarkdownSource } from '../services/markdown_sourcemap';
+
+export default OrderedList.extend({
+  addAttributes() {
+    return {
+      ...this.parent?.(),
+
+      parens: {
+        default: false,
+        parseHTML: (element) => ({
+          parens: /^[0-9]+\)/.test(getMarkdownSource(element)),
+        }),
+      },
+    };
+  },
+});

app/assets/javascripts/content_editor/extensions/task_list.js

 import { mergeAttributes } from '@tiptap/core';
 import { TaskList } from '@tiptap/extension-task-list';
 import { PARSE_HTML_PRIORITY_HIGHEST } from '../constants';
+import { getMarkdownSource } from '../services/markdown_sourcemap';
 
 export default TaskList.extend({
   addAttributes() {
     return {
-      type: {
-        default: 'ul',
-        parseHTML: (element) => {
-          return {
-            type: element.tagName.toLowerCase() === 'ol' ? 'ol' : 'ul',
-          };
-        },
+      numeric: {
+        default: false,
+        parseHTML: (element) => ({
+          numeric: element.tagName.toLowerCase() === 'ol',
+        }),
+      },
+
+      start: {
+        default: 1,
+        parseHTML: (element) => ({
+          start: element.hasAttribute('start')
+            ? parseInt(element.getAttribute('start') || '', 10)
+            : 1,
+        }),
+      },
+
+      parens: {
+        default: false,
+        parseHTML: (element) => ({
+          parens: /^[0-9]+\)/.test(getMarkdownSource(element)),
+        }),
       },
     };
   },
@@ -25,7 +40,7 @@ export default TaskList.extend({
     ];
   },
 
-  renderHTML({ HTMLAttributes: { type, ...HTMLAttributes } }) {
-    return [type, mergeAttributes(HTMLAttributes, { 'data-type': 'taskList' }), 0];
+  renderHTML({ HTMLAttributes: { numeric, ...HTMLAttributes } }) {
+    return [numeric ? 'ol' : 'ul', mergeAttributes(HTMLAttributes, { 'data-type': 'taskList' }), 0];
   },
 });

app/assets/javascripts/content_editor/services/markdown_serializer.js

@@ -32,12 +32,14 @@ import TaskItem from '../extensions/task_item';
 import TaskList from '../extensions/task_list';
 import Text from '../extensions/text';
 import {
+  isPlainURL,
   renderHardBreak,
   renderTable,
   renderTableCell,
   renderTableRow,
   openTag,
   closeTag,
+  renderOrderedList,
 } from './serialization_helpers';
 
 const defaultSerializerConfig = {
@@ -57,14 +59,15 @@ const defaultSerializerConfig = {
       },
     },
     [Link.name]: {
-      open() {
-        return '[';
+      open(state, mark, parent, index) {
+        return isPlainURL(mark, parent, index, 1) ? '<' : '[';
       },
-      close(state, mark) {
+      close(state, mark, parent, index) {
         const href = mark.attrs.canonicalSrc || mark.attrs.href;
-        return `](${state.esc(href)}${
-          mark.attrs.title ? ` ${state.quote(mark.attrs.title)}` : ''
-        })`;
+
+        return isPlainURL(mark, parent, index, -1)
+          ? '>'
+          : `](${state.esc(href)}${mark.attrs.title ? ` ${state.quote(mark.attrs.title)}` : ''})`;
       },
     },
     [Strike.name]: {
@@ -89,7 +92,18 @@ const defaultSerializerConfig = {
   },
 
   nodes: {
-    [Blockquote.name]: defaultMarkdownSerializer.nodes.blockquote,
+    [Blockquote.name]: (state, node) => {
+      if (node.attrs.multiline) {
+        state.write('>>>');
+        state.ensureNewLine();
+        state.renderContent(node);
+        state.ensureNewLine();
+        state.write('>>>');
+        state.closeBlock(node);
+      } else {
+        state.wrapBlock('> ', null, node, () => state.renderContent(node));
+      }
+    },
     [BulletList.name]: defaultMarkdownSerializer.nodes.bullet_list,
     [CodeBlockHighlight.name]: (state, node) => {
       state.write(`\`\`\`${node.attrs.language || ''}\n`);
@@ -113,7 +127,7 @@ const defaultSerializerConfig = {
       state.write(`![${state.esc(alt || '')}](${state.esc(canonicalSrc || src)}${quotedTitle})`);
     },
     [ListItem.name]: defaultMarkdownSerializer.nodes.list_item,
-    [OrderedList.name]: defaultMarkdownSerializer.nodes.ordered_list,
+    [OrderedList.name]: renderOrderedList,
     [Paragraph.name]: defaultMarkdownSerializer.nodes.paragraph,
     [Reference.name]: (state, node) => {
       state.write(node.attrs.originalText || node.attrs.text);
@@ -127,8 +141,8 @@ const defaultSerializerConfig = {
       state.renderContent(node);
     },
     [TaskList.name]: (state, node) => {
-      if (node.attrs.type === 'ul') defaultMarkdownSerializer.nodes.bullet_list(state, node);
-      else defaultMarkdownSerializer.nodes.ordered_list(state, node);
+      if (node.attrs.numeric) renderOrderedList(state, node);
+      else defaultMarkdownSerializer.nodes.bullet_list(state, node);
     },
     [Text.name]: defaultMarkdownSerializer.nodes.text,
   },

app/assets/javascripts/content_editor/services/serialization_helpers.js

@@ -8,6 +8,22 @@ const defaultAttrs = {
 
 const tableMap = new WeakMap();
 
+// Source taken from
+// prosemirror-markdown/src/to_markdown.js
+export function isPlainURL(link, parent, index, side) {
+  if (link.attrs.title || !/^\w+:/.test(link.attrs.href)) return false;
+  const content = parent.child(index + (side < 0 ? -1 : 0));
+  if (
+    !content.isText ||
+    content.text !== link.attrs.href ||
+    content.marks[content.marks.length - 1] !== link
+  )
+    return false;
+  if (index === (side < 0 ? 1 : parent.childCount - 1)) return true;
+  const next = parent.child(index + (side < 0 ? -2 : 1));
+  return !link.isInSet(next.marks);
+}
+
 function shouldRenderCellInline(cell) {
   if (cell.childCount === 1) {
     const parent = cell.child(0);
@@ -206,6 +222,19 @@ function renderTableRowAsHTML(state, node) {
   renderTagClose(state, 'tr');
 }
 
+export function renderOrderedList(state, node) {
+  const { parens } = node.attrs;
+  const start = node.attrs.start || 1;
+  const maxW = String(start + node.childCount - 1).length;
+  const space = state.repeat(' ', maxW + 2);
+  const delimiter = parens ? ')' : '.';
+
+  state.renderList(node, space, (i) => {
+    const nStr = String(start + i);
+    return `${state.repeat(' ', maxW - nStr.length) + nStr}${delimiter} `;
+  });
+}
+
 export function renderTableCell(state, node) {
   if (!isBlockTablesFeatureEnabled()) {
     state.renderInline(node);

app/assets/javascripts/lib/utils/dom_utils.js

@@ -77,3 +77,15 @@ export const isElementVisible = (element) =>
  * @returns {Boolean} `true` if the element is currently hidden, otherwise false
  */
 export const isElementHidden = (element) => !isElementVisible(element);
+
+export const getParents = (element) => {
+  const parents = [];
+  let parent = element.parentNode;
+
+  do {
+    parents.push(parent);
+    parent = parent.parentNode;
+  } while (parent);
+
+  return parents;
+};

spec/frontend/content_editor/extensions/blockquote_spec.js

+import { multilineInputRegex } from '~/content_editor/extensions/blockquote';
+
+describe('content_editor/extensions/blockquote', () => {
+  describe.each`
+    input       | matches
+    ${'>>> '}   | ${true}
+    ${' >>> '}  | ${true}
+    ${'\t>>> '} | ${true}
+    ${'>> '}    | ${false}
+    ${'>>>x '}  | ${false}
+    ${'> '}     | ${false}
+  `('multilineInputRegex', ({ input, matches }) => {
+    it(`${matches ? 'matches' : 'does not match'}: "${input}"`, () => {
+      const match = new RegExp(multilineInputRegex).test(input);
+
+      expect(match).toBe(matches);
+    });
+  });
+});

spec/frontend/content_editor/services/markdown_serializer_spec.js

@@ -8,6 +8,7 @@ import HardBreak from '~/content_editor/extensions/hard_break';
 import Heading from '~/content_editor/extensions/heading';
 import HorizontalRule from '~/content_editor/extensions/horizontal_rule';
 import Image from '~/content_editor/extensions/image';
+import InlineDiff from '~/content_editor/extensions/inline_diff';
 import Italic from '~/content_editor/extensions/italic';
 import Link from '~/content_editor/extensions/link';
 import ListItem from '~/content_editor/extensions/list_item';
@@ -18,6 +19,8 @@ import Table from '~/content_editor/extensions/table';
 import TableCell from '~/content_editor/extensions/table_cell';
 import TableHeader from '~/content_editor/extensions/table_header';
 import TableRow from '~/content_editor/extensions/table_row';
+import TaskItem from '~/content_editor/extensions/task_item';
+import TaskList from '~/content_editor/extensions/task_list';
 import Text from '~/content_editor/extensions/text';
 import markdownSerializer from '~/content_editor/services/markdown_serializer';
 import { createTestEditor, createDocBuilder } from '../test_utils';
@@ -40,6 +43,7 @@ const tiptapEditor = createTestEditor({
     Heading,
     HorizontalRule,
     Image,
+    InlineDiff,
     Italic,
     Link,
     ListItem,
@@ -50,6 +54,8 @@ const tiptapEditor = createTestEditor({
     TableCell,
     TableHeader,
     TableRow,
+    TaskItem,
+    TaskList,
     Text,
   ],
 });
@@ -67,6 +73,7 @@ const {
     hardBreak,
     horizontalRule,
     image,
+    inlineDiff,
     italic,
     link,
     listItem,
@@ -77,6 +84,8 @@ const {
     tableCell,
     tableHeader,
     tableRow,
+    taskItem,
+    taskList,
   },
 } = createDocBuilder({
   tiptapEditor,
@@ -91,6 +100,7 @@ const {
     heading: { nodeType: Heading.name },
     horizontalRule: { nodeType: HorizontalRule.name },
     image: { nodeType: Image.name },
+    inlineDiff: { markType: InlineDiff.name },
     italic: { nodeType: Italic.name },
     link: { markType: Link.name },
     listItem: { nodeType: ListItem.name },
@@ -101,6 +111,8 @@ const {
     tableCell: { nodeType: TableCell.name },
     tableHeader: { nodeType: TableHeader.name },
     tableRow: { nodeType: TableRow.name },
+    taskItem: { nodeType: TaskItem.name },
+    taskList: { nodeType: TaskList.name },
   },
 });
 
@@ -111,6 +123,25 @@ const serialize = (...content) =>
   });
 
 describe('markdownSerializer', () => {
+  it('correctly serializes bold', () => {
+    expect(serialize(paragraph(bold('bold')))).toBe('**bold**');
+  });
+
+  it('correctly serializes italics', () => {
+    expect(serialize(paragraph(italic('italics')))).toBe('_italics_');
+  });
+
+  it('correctly serializes inline diff', () => {
+    expect(
+      serialize(
+        paragraph(
+          inlineDiff({ type: 'addition' }, '+30 lines'),
+          inlineDiff({ type: 'deletion' }, '-10 lines'),
+        ),
+      ),
+    ).toBe('{++30 lines+}{--10 lines-}');
+  });
+
   it('correctly serializes a line break', () => {
     expect(serialize(paragraph('hello', hardBreak(), 'world'))).toBe('hello\\\nworld');
   });
@@ -121,6 +152,12 @@ describe('markdownSerializer', () => {
     );
   });
 
+  it('correctly serializes a plain URL link', () => {
+    expect(serialize(paragraph(link({ href: 'https://example.com' }, 'https://example.com')))).toBe(
+      '<https://example.com>',
+    );
+  });
+
   it('correctly serializes a link with a title', () => {
     expect(
       serialize(
@@ -129,6 +166,16 @@ describe('markdownSerializer', () => {
     ).toBe('[example url](https://example.com "click this link")');
   });
 
+  it('correctly serializes a plain URL link with a title', () => {
+    expect(
+      serialize(
+        paragraph(
+          link({ href: 'https://example.com', title: 'link title' }, 'https://example.com'),
+        ),
+      ),
+    ).toBe('[https://example.com](https://example.com "link title")');
+  });
+
   it('correctly serializes a link with a canonicalSrc', () => {
     expect(
       serialize(
@@ -146,6 +193,115 @@ describe('markdownSerializer', () => {
     ).toBe('[download file](file.zip "click here to download")');
   });
 
+  it('correctly serializes strikethrough', () => {
+    expect(serialize(paragraph(strike('deleted content')))).toBe('~~deleted content~~');
+  });
+
+  it('correctly serializes blockquotes with hard breaks', () => {
+    expect(serialize(blockquote('some text', hardBreak(), hardBreak(), 'new line'))).toBe(
+      `
+> some text\\
+> \\
+> new line
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes blockquote with multiple block nodes', () => {
+    expect(serialize(blockquote(paragraph('some paragraph'), codeBlock('var x = 10;')))).toBe(
+      `
+> some paragraph
+>
+> \`\`\`
+> var x = 10;
+> \`\`\`
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a multiline blockquote', () => {
+    expect(
+      serialize(
+        blockquote(
+          { multiline: true },
+          paragraph('some paragraph with ', bold('bold')),
+          codeBlock('var y = 10;'),
+        ),
+      ),
+    ).toBe(
+      `
+>>>
+some paragraph with **bold**
+
+\`\`\`
+var y = 10;
+\`\`\`
+
+>>>
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a code block with language', () => {
+    expect(
+      serialize(
+        codeBlock(
+          { language: 'json' },
+          'this is not really json but just trying out whether this case works or not',
+        ),
+      ),
+    ).toBe(
+      `
+\`\`\`json
+this is not really json but just trying out whether this case works or not
+\`\`\`
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes emoji', () => {
+    expect(serialize(paragraph(emoji({ name: 'dog' })))).toBe(':dog:');
+  });
+
+  it('correctly serializes headings', () => {
+    expect(
+      serialize(
+        heading({ level: 1 }, 'Heading 1'),
+        heading({ level: 2 }, 'Heading 2'),
+        heading({ level: 3 }, 'Heading 3'),
+        heading({ level: 4 }, 'Heading 4'),
+        heading({ level: 5 }, 'Heading 5'),
+        heading({ level: 6 }, 'Heading 6'),
+      ),
+    ).toBe(
+      `
+# Heading 1
+
+## Heading 2
+
+### Heading 3
+
+#### Heading 4
+
+##### Heading 5
+
+###### Heading 6
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes horizontal rule', () => {
+    expect(serialize(horizontalRule(), horizontalRule(), horizontalRule())).toBe(
+      `
+---
+
+---
+
+---
+      `.trim(),
+    );
+  });
+
   it('correctly serializes an image', () => {
     expect(serialize(paragraph(image({ src: 'img.jpg', alt: 'foo bar' })))).toBe(
       '![foo bar](img.jpg)',
@@ -173,6 +329,210 @@ describe('markdownSerializer', () => {
     ).toBe('![this is an image](file.png "foo bar baz")');
   });
 
+  it('correctly serializes bullet list', () => {
+    expect(
+      serialize(
+        bulletList(
+          listItem(paragraph('list item 1')),
+          listItem(paragraph('list item 2')),
+          listItem(paragraph('list item 3')),
+        ),
+      ),
+    ).toBe(
+      `
+* list item 1
+* list item 2
+* list item 3
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes bullet list with different bullet styles', () => {
+    expect(
+      serialize(
+        bulletList(
+          { bullet: '+' },
+          listItem(paragraph('list item 1')),
+          listItem(paragraph('list item 2')),
+          listItem(
+            paragraph('list item 3'),
+            bulletList(
+              { bullet: '-' },
+              listItem(paragraph('sub-list item 1')),
+              listItem(paragraph('sub-list item 2')),
+            ),
+          ),
+        ),
+      ),
+    ).toBe(
+      `
++ list item 1
++ list item 2
++ list item 3
+  - sub-list item 1
+  - sub-list item 2
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a numeric list', () => {
+    expect(
+      serialize(
+        orderedList(
+          listItem(paragraph('list item 1')),
+          listItem(paragraph('list item 2')),
+          listItem(paragraph('list item 3')),
+        ),
+      ),
+    ).toBe(
+      `
+1. list item 1
+2. list item 2
+3. list item 3
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a numeric list with parens', () => {
+    expect(
+      serialize(
+        orderedList(
+          { parens: true },
+          listItem(paragraph('list item 1')),
+          listItem(paragraph('list item 2')),
+          listItem(paragraph('list item 3')),
+        ),
+      ),
+    ).toBe(
+      `
+1) list item 1
+2) list item 2
+3) list item 3
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a numeric list with a different start order', () => {
+    expect(
+      serialize(
+        orderedList(
+          { start: 17 },
+          listItem(paragraph('list item 1')),
+          listItem(paragraph('list item 2')),
+          listItem(paragraph('list item 3')),
+        ),
+      ),
+    ).toBe(
+      `
+17. list item 1
+18. list item 2
+19. list item 3
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a numeric list with an invalid start order', () => {
+    expect(
+      serialize(
+        orderedList(
+          { start: NaN },
+          listItem(paragraph('list item 1')),
+          listItem(paragraph('list item 2')),
+          listItem(paragraph('list item 3')),
+        ),
+      ),
+    ).toBe(
+      `
+1. list item 1
+2. list item 2
+3. list item 3
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a bullet list inside an ordered list', () => {
+    expect(
+      serialize(
+        orderedList(
+          { start: 17 },
+          listItem(paragraph('list item 1')),
+          listItem(paragraph('list item 2')),
+          listItem(
+            paragraph('list item 3'),
+            bulletList(
+              listItem(paragraph('sub-list item 1')),
+              listItem(paragraph('sub-list item 2')),
+            ),
+          ),
+        ),
+      ),
+    ).toBe(
+      // notice that 4 space indent works fine in this case,
+      // when it usually wouldn't
+      `
+17. list item 1
+18. list item 2
+19. list item 3
+    * sub-list item 1
+    * sub-list item 2
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a task list', () => {
+    expect(
+      serialize(
+        taskList(
+          taskItem({ checked: true }, paragraph('list item 1')),
+          taskItem(paragraph('list item 2')),
+          taskItem(
+            paragraph('list item 3'),
+            taskList(
+              taskItem({ checked: true }, paragraph('sub-list item 1')),
+              taskItem(paragraph('sub-list item 2')),
+            ),
+          ),
+        ),
+      ),
+    ).toBe(
+      `
+* [x] list item 1
+* [ ] list item 2
+* [ ] list item 3
+  * [x] sub-list item 1
+  * [ ] sub-list item 2
+      `.trim(),
+    );
+  });
+
+  it('correctly serializes a numeric task list + with start order', () => {
+    expect(
+      serialize(
+        taskList(
+          { numeric: true },
+          taskItem({ checked: true }, paragraph('list item 1')),
+          taskItem(paragraph('list item 2')),
+          taskItem(
+            paragraph('list item 3'),
+            taskList(
+              { numeric: true, start: 1351, parens: true },
+              taskItem({ checked: true }, paragraph('sub-list item 1')),
+              taskItem(paragraph('sub-list item 2')),
+            ),
+          ),
+        ),
+      ),
+    ).toBe(
+      `
+1. [x] list item 1
+2. [ ] list item 2
+3. [ ] list item 3
+   1351) [x] sub-list item 1
+   1352) [ ] sub-list item 2
+      `.trim(),
+    );
+  });
+
   it('correctly serializes a table with inline content', () => {
     expect(
       serialize(

spec/frontend/fixtures/api_markdown.yml

@@ -99,6 +99,11 @@
     1. list item 1
     2. list item 2
     3. list item 3
+- name: ordered_list_with_start_order
+  markdown: |-
+    134. list item 1
+    135. list item 2
+    136. list item 3
 - name: task_list
   markdown: |-
     * [x] hello
@@ -115,6 +120,11 @@
        1. [ ] of nested
           1. [x] task list
           2. [ ] items
+- name: ordered_task_list_with_order
+  markdown: |-
+    4893. [x] hello
+    4894. [x] world
+    4895. [ ] example
 - name: image
   markdown: '![alt text](https://gitlab.com/logo.png)'
 - name: hard_break

spec/frontend/lib/utils/dom_utils_spec.js

@@ -5,6 +5,7 @@ import {
   parseBooleanDataAttributes,
   isElementVisible,
   isElementHidden,
+  getParents,
 } from '~/lib/utils/dom_utils';
 
 const TEST_MARGIN = 5;
@@ -193,4 +194,18 @@ describe('DOM Utils', () => {
       });
     },
   );
+
+  describe('getParents', () => {
+    it('gets all parents of an element', () => {
+      const el = document.createElement('div');
+      el.innerHTML = '<p><span><strong><mark>hello world';
+
+      expect(getParents(el.querySelector('mark'))).toEqual([
+        el.querySelector('strong'),
+        el.querySelector('span'),
+        el.querySelector('p'),
+        el,
+      ]);
+    });
+  });
 });