From daad6568b334529d3eeaf459aaf9ba3ece6ae121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 10:27:06 +0100 Subject: [PATCH 01/12] Add position and size for heading tokens --- lib/rules_block/heading.js | 10 +++++++++- lib/token.js | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/rules_block/heading.js b/lib/rules_block/heading.js index 9b8eee4..d38e346 100644 --- a/lib/rules_block/heading.js +++ b/lib/rules_block/heading.js @@ -6,11 +6,13 @@ var isSpace = require('../common/utils').isSpace; module.exports = function heading(state, startLine, endLine, silent) { - var ch, level, tmp, token, + var ch, level, tmp, token, originalPos, originalMax, pos = state.bMarks[startLine] + state.tShift[startLine], max = state.eMarks[startLine]; ch = state.src.charCodeAt(pos); + originalPos = pos; + originalMax = max; if (ch !== 0x23/* # */ || pos >= max) { return false; } @@ -39,14 +41,20 @@ module.exports = function heading(state, startLine, endLine, silent) { token = state.push('heading_open', 'h' + String(level), 1); token.markup = '########'.slice(0, level); token.map = [ startLine, state.line ]; + token.position = originalPos; + token.size = pos - originalPos; token = state.push('inline', '', 0); token.content = state.src.slice(pos, max).trim(); token.map = [ startLine, state.line ]; token.children = []; + token.position = pos; + token.size = max - pos; token = state.push('heading_close', 'h' + String(level), -1); token.markup = '########'.slice(0, level); + token.position = max; + token.size = originalMax - max; return true; }; diff --git a/lib/token.js b/lib/token.js index 448ff49..c644cd3 100644 --- a/lib/token.js +++ b/lib/token.js @@ -110,6 +110,20 @@ function Token(type, tag, nesting) { * to hide paragraphs. **/ this.hidden = false; + + /** + * Token#position -> Number + * + * Position in the original string + **/ + this.position = 0; + + /** + * Token#size -> Number + * + * Size of the token + **/ + this.size = 0; } From ea17ef0e35f35866d2b74eaa4efba014760fceed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 10:41:58 +0100 Subject: [PATCH 02/12] Set position and size for hr tokens --- lib/rules_block/hr.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/rules_block/hr.js b/lib/rules_block/hr.js index 8638f04..8d9fb20 100644 --- a/lib/rules_block/hr.js +++ b/lib/rules_block/hr.js @@ -6,10 +6,11 @@ var isSpace = require('../common/utils').isSpace; module.exports = function hr(state, startLine, endLine, silent) { - var marker, cnt, ch, token, + var marker, cnt, ch, token, originalPos, pos = state.bMarks[startLine] + state.tShift[startLine], max = state.eMarks[startLine]; + originalPos = pos; marker = state.src.charCodeAt(pos++); // Check hr marker @@ -37,6 +38,8 @@ module.exports = function hr(state, startLine, endLine, silent) { token = state.push('hr', 'hr', 0); token.map = [ startLine, state.line ]; token.markup = Array(cnt + 1).join(String.fromCharCode(marker)); + token.position = originalPos; + token.size = pos - originalPos; return true; }; From 6b58d255e6ff71110945eef7b8205b6e96e605cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 10:49:24 +0100 Subject: [PATCH 03/12] Add position and size to code_inline --- lib/rules_inline/backticks.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/rules_inline/backticks.js b/lib/rules_inline/backticks.js index 0cd1a99..5a1b986 100644 --- a/lib/rules_inline/backticks.js +++ b/lib/rules_inline/backticks.js @@ -31,6 +31,8 @@ module.exports = function backtick(state, silent) { token.content = state.src.slice(pos, matchStart) .replace(/[ \n]+/g, ' ') .trim(); + token.position = start; + token.size = (matchStart + 1) - token.position; } state.pos = matchEnd; return true; From c3d4a39f6d9010232b4d15a1960bb678adf53b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 10:59:09 +0100 Subject: [PATCH 04/12] Add position and size for fences --- lib/rules_block/fence.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/rules_block/fence.js b/lib/rules_block/fence.js index a02bd5f..d115463 100644 --- a/lib/rules_block/fence.js +++ b/lib/rules_block/fence.js @@ -4,7 +4,7 @@ module.exports = function fence(state, startLine, endLine, silent) { - var marker, len, params, nextLine, mem, token, markup, + var marker, len, params, nextLine, mem, token, markup, originalPos, haveEndMarker = false, pos = state.bMarks[startLine] + state.tShift[startLine], max = state.eMarks[startLine]; @@ -25,6 +25,7 @@ module.exports = function fence(state, startLine, endLine, silent) { if (len < 3) { return false; } + originalPos = mem; markup = state.src.slice(mem, pos); params = state.src.slice(pos, max); @@ -87,5 +88,8 @@ module.exports = function fence(state, startLine, endLine, silent) { token.markup = markup; token.map = [ startLine, state.line ]; + token.position = originalPos; + token.size = pos - originalPos; + return true; }; From dfeba0a3bde190941250ed419fb86de5fc7ca4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 11:02:31 +0100 Subject: [PATCH 05/12] Add position and size for code_block --- lib/rules_block/code.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/rules_block/code.js b/lib/rules_block/code.js index 470cb01..ed2e304 100644 --- a/lib/rules_block/code.js +++ b/lib/rules_block/code.js @@ -4,7 +4,9 @@ module.exports = function code(state, startLine, endLine/*, silent*/) { - var nextLine, last, token, emptyLines = 0; + var nextLine, last, token, emptyLines = 0, + pos = state.bMarks[startLine], + endPos; if (state.sCount[startLine] - state.blkIndent < 4) { return false; } @@ -34,11 +36,15 @@ module.exports = function code(state, startLine, endLine/*, silent*/) { break; } + endPos = state.bMarks[last] + state.tShift[last]; + state.line = last; token = state.push('code_block', 'code', 0); token.content = state.getLines(startLine, last, 4 + state.blkIndent, true); token.map = [ startLine, state.line ]; + token.position = pos; + token.size = endPos - pos; return true; }; From 4e1007edae5687e85a7fdcfa397a3b34fd8cce3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 11:13:38 +0100 Subject: [PATCH 06/12] Fix calcul of token.size for backticks --- lib/rules_inline/backticks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rules_inline/backticks.js b/lib/rules_inline/backticks.js index 5a1b986..efec079 100644 --- a/lib/rules_inline/backticks.js +++ b/lib/rules_inline/backticks.js @@ -32,7 +32,7 @@ module.exports = function backtick(state, silent) { .replace(/[ \n]+/g, ' ') .trim(); token.position = start; - token.size = (matchStart + 1) - token.position; + token.size = matchEnd - token.position; } state.pos = matchEnd; return true; From ace5e1c058610cfe6492097e6f8bcae216ec12fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 14:33:23 +0100 Subject: [PATCH 07/12] Position in end result should be absolute --- lib/rules_core/inline.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/rules_core/inline.js b/lib/rules_core/inline.js index 4c33d0d..6148c82 100644 --- a/lib/rules_core/inline.js +++ b/lib/rules_core/inline.js @@ -8,6 +8,11 @@ module.exports = function inline(state) { tok = tokens[i]; if (tok.type === 'inline') { state.md.inline.parse(tok.content, state.md, state.env, tok.children); + + // Update position of all children to be absolute + for (var child = 0; child < tok.children.length; child++) { + tok.children[child].position = tok.children[child].position + tok.position; + } } } }; From 9514121052963340a7876a6f19804dd0ff12d310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 14:38:33 +0100 Subject: [PATCH 08/12] Add position and size for paragraph --- lib/rules_block/paragraph.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/rules_block/paragraph.js b/lib/rules_block/paragraph.js index 18a860d..4ab7e61 100644 --- a/lib/rules_block/paragraph.js +++ b/lib/rules_block/paragraph.js @@ -7,7 +7,8 @@ module.exports = function paragraph(state, startLine/*, endLine*/) { var content, terminate, i, l, token, nextLine = startLine + 1, terminatorRules = state.md.block.ruler.getRules('paragraph'), - endLine = state.lineMax; + endLine = state.lineMax, + pos = state.bMarks[startLine]; // jump line-by-line until empty one or EOF for (; nextLine < endLine && !state.isEmpty(nextLine); nextLine++) { @@ -35,13 +36,19 @@ module.exports = function paragraph(state, startLine/*, endLine*/) { token = state.push('paragraph_open', 'p', 1); token.map = [ startLine, state.line ]; + token.position = pos; + token.size = 0; token = state.push('inline', '', 0); token.content = content; token.map = [ startLine, state.line ]; token.children = []; + token.position = pos; + token.size = content.length; token = state.push('paragraph_close', 'p', -1); + token.size = 0; + token.position = content.length + pos; return true; }; From 3cc71eff4abfd6a5bf47dc1e177566fff066d7f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 14:46:28 +0100 Subject: [PATCH 09/12] Start tests for annotations --- test/annotation.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/annotation.js diff --git a/test/annotation.js b/test/annotation.js new file mode 100644 index 0000000..b54d135 --- /dev/null +++ b/test/annotation.js @@ -0,0 +1,35 @@ +'use strict'; + +var assert = require('chai').assert; + +describe('Annotation', function() { + var md = require('../')({ + html: true, + langPrefix: '', + typographer: true, + linkify: true + }); + + it('should annotate paragraph', function () { + var tokens = md.parse('Hello World\n\nThis is great !'); + assert.strictEqual(tokens.length, 6); + + // First paragraph + assert.strictEqual(tokens[0].position, 0); + assert.strictEqual(tokens[0].size, 0); + assert.strictEqual(tokens[1].position, 0); + assert.strictEqual(tokens[1].size, 11); + assert.strictEqual(tokens[2].position, 11); + assert.strictEqual(tokens[2].size, 0); + + // Second paragraph + assert.strictEqual(tokens[3].position, 13); + assert.strictEqual(tokens[3].size, 0); + assert.strictEqual(tokens[4].position, 13); + assert.strictEqual(tokens[4].size, 15); + assert.strictEqual(tokens[5].position, 28); + assert.strictEqual(tokens[5].size, 0); + }); + +}); + From 286125b326fb6df9d50e2f9cc71333830dcc44da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 14:53:09 +0100 Subject: [PATCH 10/12] Add test for annotations of headings and code blocks --- test/annotation.js | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/test/annotation.js b/test/annotation.js index b54d135..e863012 100644 --- a/test/annotation.js +++ b/test/annotation.js @@ -2,12 +2,12 @@ var assert = require('chai').assert; -describe('Annotation', function() { +describe.only('Annotation', function() { var md = require('../')({ html: true, langPrefix: '', - typographer: true, - linkify: true + typographer: false, + linkify: false }); it('should annotate paragraph', function () { @@ -31,5 +31,38 @@ describe('Annotation', function() { assert.strictEqual(tokens[5].size, 0); }); + it('should annotate headings', function () { + var tokens = md.parse('# Hello\n\n## World ##\n'); + assert.strictEqual(tokens.length, 6); + + // First heading + assert.strictEqual(tokens[0].position, 0); + assert.strictEqual(tokens[0].size, 1); + assert.strictEqual(tokens[1].position, 1); + assert.strictEqual(tokens[1].size, 6); + assert.strictEqual(tokens[2].position, 7); + assert.strictEqual(tokens[2].size, 0); + + // Second heading + assert.strictEqual(tokens[3].position, 9); + assert.strictEqual(tokens[3].size, 2); + assert.strictEqual(tokens[4].position, 11); + assert.strictEqual(tokens[4].size, 7); + assert.strictEqual(tokens[5].position, 18); + assert.strictEqual(tokens[5].size, 2); + }); + + it('should annotate code blocks', function () { + var tokens = md.parse('\tHello\n\tWorld\n\nt\n\n\tBlock 2\n'); + assert.strictEqual(tokens.length, 5); + + assert.strictEqual(tokens[0].position, 0); + assert.strictEqual(tokens[0].size, 14); + + assert.strictEqual(tokens[4].position, 18); + assert.strictEqual(tokens[4].size, 9); + }); + + }); From 1e7db76630759786ac61a752a77426722d41c675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 15:03:19 +0100 Subject: [PATCH 11/12] Add position and size for lheading --- lib/rules_block/lheading.js | 6 ++++++ test/annotation.js | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/lib/rules_block/lheading.js b/lib/rules_block/lheading.js index 482084c..e22d817 100644 --- a/lib/rules_block/lheading.js +++ b/lib/rules_block/lheading.js @@ -62,14 +62,20 @@ module.exports = function lheading(state, startLine, endLine/*, silent*/) { token = state.push('heading_open', 'h' + String(level), 1); token.markup = String.fromCharCode(marker); token.map = [ startLine, state.line ]; + token.position = state.bMarks[startLine]; + token.size = 0; token = state.push('inline', '', 0); token.content = content; token.map = [ startLine, state.line - 1 ]; token.children = []; + token.position = state.bMarks[startLine]; + token.size = content.length; token = state.push('heading_close', 'h' + String(level), -1); token.markup = String.fromCharCode(marker); + token.position = state.bMarks[state.line - 1]; + token.size = state.bMarks[state.line] - state.bMarks[state.line - 1]; return true; }; diff --git a/test/annotation.js b/test/annotation.js index e863012..811084d 100644 --- a/test/annotation.js +++ b/test/annotation.js @@ -2,6 +2,10 @@ var assert = require('chai').assert; +function asertTokenContent(src, token, content) { + assert.strictEqual(src.slice(token.position, token.position + token.size), content); +} + describe.only('Annotation', function() { var md = require('../')({ html: true, @@ -52,6 +56,24 @@ describe.only('Annotation', function() { assert.strictEqual(tokens[5].size, 2); }); + it('should annotate lheadings', function () { + var src = 'Hello\n=====\n\nWorld\n======='; + var tokens = md.parse(src); + assert.strictEqual(tokens.length, 6); + + // First heading + assert.strictEqual(tokens[0].position, 0); + asertTokenContent(src, tokens[0], ''); + asertTokenContent(src, tokens[1], 'Hello'); + asertTokenContent(src, tokens[2], '=====\n'); + + // Second heading + assert.strictEqual(tokens[3].position, 13); + assert.strictEqual(tokens[3].size, 0); + asertTokenContent(src, tokens[4], 'World'); + asertTokenContent(src, tokens[5], '======='); + }); + it('should annotate code blocks', function () { var tokens = md.parse('\tHello\n\tWorld\n\nt\n\n\tBlock 2\n'); assert.strictEqual(tokens.length, 5); From e113df738e4d164b1a5bdd0e345a7847c84d29bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Wed, 24 Feb 2016 16:21:12 +0100 Subject: [PATCH 12/12] Add position and size for tables --- lib/rules_block/table.js | 73 +++++++++++++++++++++++++++++++++++----- test/annotation.js | 68 +++++++++++++++++++++++++++++++++---- 2 files changed, 126 insertions(+), 15 deletions(-) diff --git a/lib/rules_block/table.js b/lib/rules_block/table.js index 6993928..eaba69a 100644 --- a/lib/rules_block/table.js +++ b/lib/rules_block/table.js @@ -55,7 +55,7 @@ function escapedSplit(str) { module.exports = function table(state, startLine, endLine, silent) { var ch, lineText, pos, i, nextLine, columns, columnCount, token, - aligns, t, tableLines, tbodyLines; + aligns, t, tableLines, tbodyLines, columnVIndex; // should have at least three lines if (startLine + 2 > endLine) { return false; } @@ -110,18 +110,30 @@ module.exports = function table(state, startLine, endLine, silent) { if (silent) { return true; } - token = state.push('table_open', 'table', 1); - token.map = tableLines = [ startLine, 0 ]; + token = state.push('table_open', 'table', 1); + token.map = tableLines = [ startLine, 0 ]; + token.size = 0; + token.position = state.bMarks[startLine]; - token = state.push('thead_open', 'thead', 1); - token.map = [ startLine, startLine + 1 ]; - token = state.push('tr_open', 'tr', 1); - token.map = [ startLine, startLine + 1 ]; + token = state.push('thead_open', 'thead', 1); + token.map = [ startLine, startLine + 1 ]; + token.size = 0; + token.position = state.bMarks[startLine]; + token = state.push('tr_open', 'tr', 1); + token.map = [ startLine, startLine + 1 ]; + token.size = 0; + token.position = state.bMarks[startLine]; + + columnVIndex = state.bMarks[startLine] + state.tShift[startLine]; for (i = 0; i < columns.length; i++) { token = state.push('th_open', 'th', 1); token.map = [ startLine, startLine + 1 ]; + token.size = 1; + token.position = columnVIndex; + columnVIndex = columnVIndex + 1; + if (aligns[i]) { token.attrs = [ [ 'style', 'text-align:' + aligns[i] ] ]; } @@ -130,15 +142,33 @@ module.exports = function table(state, startLine, endLine, silent) { token.content = columns[i].trim(); token.map = [ startLine, startLine + 1 ]; token.children = []; + token.position = columnVIndex; + token.size = columns[i].length; + columnVIndex = columnVIndex + columns[i].length; token = state.push('th_close', 'th', -1); + token.position = columnVIndex; + + // Last column? + if (i === (columns.length - 1)) { + token.size = 1; + columnVIndex = columnVIndex + 1; + } + } - token = state.push('tr_close', 'tr', -1); - token = state.push('thead_close', 'thead', -1); + token = state.push('tr_close', 'tr', -1); + token.size = 0; + token.position = state.eMarks[startLine]; + + token = state.push('thead_close', 'thead', -1); + token.size = state.eMarks[startLine + 1] - state.bMarks[startLine + 1]; + token.position = state.bMarks[startLine + 1]; token = state.push('tbody_open', 'tbody', 1); token.map = tbodyLines = [ startLine + 2, 0 ]; + token.size = 0; + token.position = state.bMarks[startLine + 2]; for (nextLine = startLine + 2; nextLine < endLine; nextLine++) { if (state.sCount[nextLine] < state.blkIndent) { break; } @@ -148,8 +178,16 @@ module.exports = function table(state, startLine, endLine, silent) { columns = escapedSplit(lineText.replace(/^\||\|$/g, '')); token = state.push('tr_open', 'tr', 1); + token.size = 0; + token.position = state.bMarks[nextLine]; + + columnVIndex = state.bMarks[nextLine] + state.tShift[nextLine]; for (i = 0; i < columnCount; i++) { token = state.push('td_open', 'td', 1); + token.size = 1; + token.position = columnVIndex; + columnVIndex++; + if (aligns[i]) { token.attrs = [ [ 'style', 'text-align:' + aligns[i] ] ]; } @@ -157,13 +195,30 @@ module.exports = function table(state, startLine, endLine, silent) { token = state.push('inline', '', 0); token.content = columns[i] ? columns[i].trim() : ''; token.children = []; + token.size = (columns[i] || '').length; + token.position = columnVIndex; + columnVIndex += token.size; token = state.push('td_close', 'td', -1); + token.position = columnVIndex; + + // Last column? + if (i === (columns.length - 1)) { + token.size = 1; + } } token = state.push('tr_close', 'tr', -1); + token.size = 0; + token.position = state.eMarks[nextLine]; } + token = state.push('tbody_close', 'tbody', -1); + token.size = 0; + token.position = state.eMarks[nextLine]; + token = state.push('table_close', 'table', -1); + token.size = 0; + token.position = state.eMarks[nextLine]; tableLines[1] = tbodyLines[1] = nextLine; state.line = nextLine; diff --git a/test/annotation.js b/test/annotation.js index 811084d..7f88341 100644 --- a/test/annotation.js +++ b/test/annotation.js @@ -2,7 +2,7 @@ var assert = require('chai').assert; -function asertTokenContent(src, token, content) { +function assertTokenContent(src, token, content) { assert.strictEqual(src.slice(token.position, token.position + token.size), content); } @@ -63,15 +63,15 @@ describe.only('Annotation', function() { // First heading assert.strictEqual(tokens[0].position, 0); - asertTokenContent(src, tokens[0], ''); - asertTokenContent(src, tokens[1], 'Hello'); - asertTokenContent(src, tokens[2], '=====\n'); + assertTokenContent(src, tokens[0], ''); + assertTokenContent(src, tokens[1], 'Hello'); + assertTokenContent(src, tokens[2], '=====\n'); // Second heading assert.strictEqual(tokens[3].position, 13); assert.strictEqual(tokens[3].size, 0); - asertTokenContent(src, tokens[4], 'World'); - asertTokenContent(src, tokens[5], '======='); + assertTokenContent(src, tokens[4], 'World'); + assertTokenContent(src, tokens[5], '======='); }); it('should annotate code blocks', function () { @@ -85,6 +85,62 @@ describe.only('Annotation', function() { assert.strictEqual(tokens[4].size, 9); }); + it('should annotate tables', function () { + var src = 'Test:\n\n' + + ' | Type | Message |\n' + + ' | ---- | ------- |\n' + + '| Hello | World\n' + + ' | Bonjour | Monde |\n'; + var tokens = md.parse(src); + assert.strictEqual(tokens.length, 33); + + // Begin + assert.strictEqual(tokens[3].position, 7); + assert.strictEqual(tokens[3].size, 0); + + // head (open) + assert.strictEqual(tokens[4].position, 7); + assert.strictEqual(tokens[4].size, 0); + + // head -> TR (open) + assert.strictEqual(tokens[5].position, 7); + assert.strictEqual(tokens[5].size, 0); + + // head -> columns + assertTokenContent(src, tokens[6], '|'); + assertTokenContent(src, tokens[7], ' Type '); + assertTokenContent(src, tokens[8], ''); + assertTokenContent(src, tokens[9], '|'); + assertTokenContent(src, tokens[10], ' Message '); + assertTokenContent(src, tokens[11], '|'); + + // head -> TR (close) + assert.strictEqual(tokens[12].position, 26); + assert.strictEqual(tokens[12].size, 0); + + // head (close) + assertTokenContent(src, tokens[13], ' | ---- | ------- |'); + + // body (open) + assert.strictEqual(tokens[14].position, 47); + assert.strictEqual(tokens[14].size, 0); + + // body -> rows + assertTokenContent(src, tokens[16], '|'); + assertTokenContent(src, tokens[17], ' Hello '); + assertTokenContent(src, tokens[18], ''); + assertTokenContent(src, tokens[19], '|'); + assertTokenContent(src, tokens[20], ' World'); + assertTokenContent(src, tokens[21], '\n'); + + assertTokenContent(src, tokens[24], '|'); + assertTokenContent(src, tokens[25], ' Bonjour '); + assertTokenContent(src, tokens[26], ''); + assertTokenContent(src, tokens[27], '|'); + assertTokenContent(src, tokens[28], ' Monde '); + assertTokenContent(src, tokens[29], '|'); + }); + });