Add reference_definition token, stripped by default, #1055

src/parser_core.ts

@@ -3,6 +3,7 @@ import StateCore from './rules_core/state_core.ts'
 
 import r_normalize from './rules_core/normalize.ts'
 import r_block from './rules_core/block.ts'
+import r_strip_references from './rules_core/strip_references.ts'
 import r_inline from './rules_core/inline.ts'
 import r_linkify from './rules_core/linkify.ts'
 import r_replacements from './rules_core/replacements.ts'
@@ -15,6 +16,7 @@ const _rules: Array<[
 ]> = [
   ['normalize', r_normalize],
   ['block', r_block],
+  ['strip_references', r_strip_references],
   ['inline', r_inline],
   ['linkify', r_linkify],
   ['replacements', r_replacements],

src/presets/commonmark.ts

@@ -49,6 +49,7 @@ export default {
       rules: [
         'normalize',
         'block',
+        'strip_references',
         'inline',
         'text_join'
       ]

src/presets/zero.ts

@@ -50,6 +50,7 @@ export default {
       rules: [
         'normalize',
         'block',
+        'strip_references',
         'inline',
         'text_join'
       ]

src/renderer.ts

@@ -215,8 +215,14 @@ class Renderer {
     //
     // Only closing hidden tokens count, to not break on other hidden ones.
     //
-    if (token.block && token.nesting !== -1 && idx &&
-        tokens[idx - 1].hidden && tokens[idx - 1].nesting === -1) {
+    // Hidden tokens without nesting (`reference_definition`) are skipped here
+    // and below, or they would break line feeds around neighbour blocks.
+    //
+    let prev = idx - 1
+    while (prev >= 0 && tokens[prev].hidden && tokens[prev].nesting === 0) { prev-- }
+
+    if (token.block && token.nesting !== -1 && prev >= 0 &&
+        tokens[prev].hidden && tokens[prev].nesting === -1) {
       result += '\n'
     }
 
@@ -237,8 +243,11 @@ class Renderer {
       needLf = true
 
       if (token.nesting === 1) {
-        if (idx + 1 < tokens.length) {
-          const nextToken = tokens[idx + 1]
+        let next = idx + 1
+        while (next < tokens.length && tokens[next].hidden && tokens[next].nesting === 0) { next++ }
+
+        if (next < tokens.length) {
+          const nextToken = tokens[next]
 
           if (nextToken.type === 'inline' || nextToken.hidden) {
           // Block-level tag containing an inline tag.

src/rules_block/reference.ts

@@ -208,6 +208,16 @@ export default function reference (state: StateBlock, startLine: number, _endLin
     state.env.references[label] = { title, href }
   }
 
+  // Marks the place definition took in the source. Renders to nothing,
+  // href/title stay in `env.references`.
+  const token = state.push('reference_definition', '', 0)
+  token.map = [startLine, nextLine]
+  token.hidden = true
+
+  const meta: Record<string, unknown> = Object.create(null)
+  meta.label = label
+  token.meta = meta
+
   state.line = nextLine
   return true
 }

src/rules_core/strip_references.ts

@@ -0,0 +1,23 @@
+// Drop `reference_definition` tokens to keep the stream backward compatible
+//
+// Those tokens mark places link definitions took in the source. They are new,
+// and plugins walking block tokens may not expect them, so by default the
+// stream stays as it always was. Disable this rule to opt in.
+//
+
+import type StateCore from './state_core.ts'
+
+export default function strip_references (state: StateCore): void {
+  const tokens = state.tokens
+  let last = 0
+
+  for (let curr = 0; curr < tokens.length; curr++) {
+    if (tokens[curr].type === 'reference_definition') continue
+
+    if (curr !== last) { tokens[last] = tokens[curr] }
+
+    last++
+  }
+
+  if (tokens.length !== last) { tokens.length = last }
+}

test/markdown-it/misc.test.mjs

@@ -213,6 +213,33 @@ describe('Misc', function () {
     assert.deepEqual(inlineTokens[1].children[0].meta, expectedMeta)
   })
 
+  it('Should strip reference definition tokens by default', function () {
+    const md = markdownit()
+    const src = '[foo]: /url\n\ntext'
+
+    assert.deepEqual(md.parse(src, {}).filter(t => t.type === 'reference_definition'), [])
+
+    md.core.ruler.disable('strip_references')
+    const tokens = md.parse(src, {}).filter(t => t.type === 'reference_definition')
+    const expectedMeta = Object.create(null)
+    expectedMeta.label = 'FOO'
+
+    assert.strictEqual(tokens.length, 1)
+    assert.deepEqual(tokens[0].map, [0, 1])
+    assert.deepEqual(tokens[0].meta, expectedMeta)
+  })
+
+  it('Should not affect line breaks with reference definition tokens', function () {
+    const md = markdownit()
+    md.core.ruler.disable('strip_references')
+
+    assert.strictEqual(md.render('- [foo]: /url'), '<ul>\n<li></li>\n</ul>\n')
+    assert.strictEqual(md.render('> [foo]: /url'), '<blockquote></blockquote>\n')
+    assert.strictEqual(md.render('> [foo]: /url\n> bar'), '<blockquote>\n<p>bar</p>\n</blockquote>\n')
+    assert.strictEqual(md.render('- [foo]: /url\n\n  > bar'),
+      '<ul>\n<li>\n<blockquote>\n<p>bar</p>\n</blockquote>\n</li>\n</ul>\n')
+  })
+
   it('Should replace NULL characters', function () {
     const md = markdownit()