Merge e113df738e into 0e51825a5c

lib/rules_block/code.js

@@ -4,7 +4,9 @@
 
 
 module.exports = function code(state, startLine, endLine/*, silent*/) {
-  var nextLine, last, token, emptyLines = 0;
+  var nextLine, last, token, emptyLines = 0,
+    pos = state.bMarks[startLine],
+    endPos;
 
   if (state.sCount[startLine] - state.blkIndent < 4) { return false; }
 
@@ -34,11 +36,15 @@ module.exports = function code(state, startLine, endLine/*, silent*/) {
     break;
   }
 
+  endPos = state.bMarks[last] + state.tShift[last];
+
   state.line = last;
 
   token         = state.push('code_block', 'code', 0);
   token.content = state.getLines(startLine, last, 4 + state.blkIndent, true);
   token.map     = [ startLine, state.line ];
+  token.position = pos;
+  token.size = endPos - pos;
 
   return true;
 };

lib/rules_block/fence.js

@@ -4,7 +4,7 @@
 
 
 module.exports = function fence(state, startLine, endLine, silent) {
-  var marker, len, params, nextLine, mem, token, markup,
+  var marker, len, params, nextLine, mem, token, markup, originalPos,
       haveEndMarker = false,
       pos = state.bMarks[startLine] + state.tShift[startLine],
       max = state.eMarks[startLine];
@@ -25,6 +25,7 @@ module.exports = function fence(state, startLine, endLine, silent) {
 
   if (len < 3) { return false; }
 
+  originalPos = mem;
   markup = state.src.slice(mem, pos);
   params = state.src.slice(pos, max);
 
@@ -87,5 +88,8 @@ module.exports = function fence(state, startLine, endLine, silent) {
   token.markup  = markup;
   token.map     = [ startLine, state.line ];
 
+  token.position = originalPos;
+  token.size = pos - originalPos;
+
   return true;
 };

lib/rules_block/heading.js

@@ -6,11 +6,13 @@ var isSpace = require('../common/utils').isSpace;
 
 
 module.exports = function heading(state, startLine, endLine, silent) {
-  var ch, level, tmp, token,
+  var ch, level, tmp, token, originalPos, originalMax,
       pos = state.bMarks[startLine] + state.tShift[startLine],
       max = state.eMarks[startLine];
 
   ch  = state.src.charCodeAt(pos);
+  originalPos = pos;
+  originalMax = max;
 
   if (ch !== 0x23/* # */ || pos >= max) { return false; }
 
@@ -39,14 +41,20 @@ module.exports = function heading(state, startLine, endLine, silent) {
   token        = state.push('heading_open', 'h' + String(level), 1);
   token.markup = '########'.slice(0, level);
   token.map    = [ startLine, state.line ];
+  token.position = originalPos;
+  token.size = pos - originalPos;
 
   token          = state.push('inline', '', 0);
   token.content  = state.src.slice(pos, max).trim();
   token.map      = [ startLine, state.line ];
   token.children = [];
+  token.position = pos;
+  token.size = max - pos;
 
   token        = state.push('heading_close', 'h' + String(level), -1);
   token.markup = '########'.slice(0, level);
+  token.position = max;
+  token.size = originalMax - max;
 
   return true;
 };

lib/rules_block/hr.js

@@ -6,10 +6,11 @@ var isSpace = require('../common/utils').isSpace;
 
 
 module.exports = function hr(state, startLine, endLine, silent) {
-  var marker, cnt, ch, token,
+  var marker, cnt, ch, token, originalPos,
       pos = state.bMarks[startLine] + state.tShift[startLine],
       max = state.eMarks[startLine];
 
+  originalPos = pos;
   marker = state.src.charCodeAt(pos++);
 
   // Check hr marker
@@ -37,6 +38,8 @@ module.exports = function hr(state, startLine, endLine, silent) {
   token        = state.push('hr', 'hr', 0);
   token.map    = [ startLine, state.line ];
   token.markup = Array(cnt + 1).join(String.fromCharCode(marker));
+  token.position = originalPos;
+  token.size = pos - originalPos;
 
   return true;
 };

lib/rules_block/lheading.js

@@ -62,14 +62,20 @@ module.exports = function lheading(state, startLine, endLine/*, silent*/) {
   token          = state.push('heading_open', 'h' + String(level), 1);
   token.markup   = String.fromCharCode(marker);
   token.map      = [ startLine, state.line ];
+  token.position = state.bMarks[startLine];
+  token.size     = 0;
 
   token          = state.push('inline', '', 0);
   token.content  = content;
   token.map      = [ startLine, state.line - 1 ];
   token.children = [];
+  token.position = state.bMarks[startLine];
+  token.size     = content.length;
 
   token          = state.push('heading_close', 'h' + String(level), -1);
   token.markup   = String.fromCharCode(marker);
+  token.position = state.bMarks[state.line - 1];
+  token.size     = state.bMarks[state.line] - state.bMarks[state.line - 1];
 
   return true;
 };

lib/rules_block/paragraph.js

@@ -7,7 +7,8 @@ module.exports = function paragraph(state, startLine/*, endLine*/) {
   var content, terminate, i, l, token,
       nextLine = startLine + 1,
       terminatorRules = state.md.block.ruler.getRules('paragraph'),
-      endLine = state.lineMax;
+      endLine = state.lineMax,
+      pos = state.bMarks[startLine];
 
   // jump line-by-line until empty one or EOF
   for (; nextLine < endLine && !state.isEmpty(nextLine); nextLine++) {
@@ -35,13 +36,19 @@ module.exports = function paragraph(state, startLine/*, endLine*/) {
 
   token          = state.push('paragraph_open', 'p', 1);
   token.map      = [ startLine, state.line ];
+  token.position = pos;
+  token.size     = 0;
 
   token          = state.push('inline', '', 0);
   token.content  = content;
   token.map      = [ startLine, state.line ];
   token.children = [];
+  token.position = pos;
+  token.size     = content.length;
 
   token          = state.push('paragraph_close', 'p', -1);
+  token.size     = 0;
+  token.position = content.length + pos;
 
   return true;
 };

lib/rules_block/table.js

@@ -55,7 +55,7 @@ function escapedSplit(str) {
 
 module.exports = function table(state, startLine, endLine, silent) {
   var ch, lineText, pos, i, nextLine, columns, columnCount, token,
-      aligns, t, tableLines, tbodyLines;
+      aligns, t, tableLines, tbodyLines, columnVIndex;
 
   // should have at least three lines
   if (startLine + 2 > endLine) { return false; }
@@ -110,18 +110,30 @@ module.exports = function table(state, startLine, endLine, silent) {
 
   if (silent) { return true; }
 
-  token     = state.push('table_open', 'table', 1);
-  token.map = tableLines = [ startLine, 0 ];
+  token          = state.push('table_open', 'table', 1);
+  token.map      = tableLines = [ startLine, 0 ];
+  token.size     = 0;
+  token.position = state.bMarks[startLine];
 
-  token     = state.push('thead_open', 'thead', 1);
-  token.map = [ startLine, startLine + 1 ];
 
-  token     = state.push('tr_open', 'tr', 1);
-  token.map = [ startLine, startLine + 1 ];
+  token          = state.push('thead_open', 'thead', 1);
+  token.map      = [ startLine, startLine + 1 ];
+  token.size     = 0;
+  token.position = state.bMarks[startLine];
 
+  token          = state.push('tr_open', 'tr', 1);
+  token.map      = [ startLine, startLine + 1 ];
+  token.size     = 0;
+  token.position = state.bMarks[startLine];
+
+  columnVIndex = state.bMarks[startLine] + state.tShift[startLine];
   for (i = 0; i < columns.length; i++) {
     token          = state.push('th_open', 'th', 1);
     token.map      = [ startLine, startLine + 1 ];
+    token.size     = 1;
+    token.position = columnVIndex;
+    columnVIndex = columnVIndex + 1;
+
     if (aligns[i]) {
       token.attrs  = [ [ 'style', 'text-align:' + aligns[i] ] ];
     }
@@ -130,15 +142,33 @@ module.exports = function table(state, startLine, endLine, silent) {
     token.content  = columns[i].trim();
     token.map      = [ startLine, startLine + 1 ];
     token.children = [];
+    token.position = columnVIndex;
+    token.size     = columns[i].length;
+    columnVIndex = columnVIndex + columns[i].length;
 
     token          = state.push('th_close', 'th', -1);
+    token.position = columnVIndex;
+
+    // Last column?
+    if (i === (columns.length - 1)) {
+      token.size     = 1;
+      columnVIndex = columnVIndex + 1;
+    }
+
   }
 
-  token     = state.push('tr_close', 'tr', -1);
-  token     = state.push('thead_close', 'thead', -1);
+  token          = state.push('tr_close', 'tr', -1);
+  token.size     = 0;
+  token.position = state.eMarks[startLine];
+
+  token          = state.push('thead_close', 'thead', -1);
+  token.size     = state.eMarks[startLine + 1] - state.bMarks[startLine + 1];
+  token.position = state.bMarks[startLine + 1];
 
   token     = state.push('tbody_open', 'tbody', 1);
   token.map = tbodyLines = [ startLine + 2, 0 ];
+  token.size     = 0;
+  token.position = state.bMarks[startLine + 2];
 
   for (nextLine = startLine + 2; nextLine < endLine; nextLine++) {
     if (state.sCount[nextLine] < state.blkIndent) { break; }
@@ -148,8 +178,16 @@ module.exports = function table(state, startLine, endLine, silent) {
     columns = escapedSplit(lineText.replace(/^\||\|$/g, ''));
 
     token = state.push('tr_open', 'tr', 1);
+    token.size     = 0;
+    token.position = state.bMarks[nextLine];
+
+    columnVIndex = state.bMarks[nextLine] + state.tShift[nextLine];
     for (i = 0; i < columnCount; i++) {
       token          = state.push('td_open', 'td', 1);
+      token.size     = 1;
+      token.position = columnVIndex;
+      columnVIndex++;
+
       if (aligns[i]) {
         token.attrs  = [ [ 'style', 'text-align:' + aligns[i] ] ];
       }
@@ -157,13 +195,30 @@ module.exports = function table(state, startLine, endLine, silent) {
       token          = state.push('inline', '', 0);
       token.content  = columns[i] ? columns[i].trim() : '';
       token.children = [];
+      token.size     = (columns[i] || '').length;
+      token.position = columnVIndex;
+      columnVIndex += token.size;
 
       token          = state.push('td_close', 'td', -1);
+      token.position = columnVIndex;
+
+      // Last column?
+      if (i === (columns.length - 1)) {
+        token.size     = 1;
+      }
     }
     token = state.push('tr_close', 'tr', -1);
+    token.size     = 0;
+    token.position = state.eMarks[nextLine];
   }
+
   token = state.push('tbody_close', 'tbody', -1);
+  token.size     = 0;
+  token.position = state.eMarks[nextLine];
+
   token = state.push('table_close', 'table', -1);
+  token.size     = 0;
+  token.position = state.eMarks[nextLine];
 
   tableLines[1] = tbodyLines[1] = nextLine;
   state.line = nextLine;

lib/rules_core/inline.js

@@ -8,6 +8,11 @@ module.exports = function inline(state) {
     tok = tokens[i];
     if (tok.type === 'inline') {
       state.md.inline.parse(tok.content, state.md, state.env, tok.children);
+
+      // Update position of all children to be absolute
+      for (var child = 0; child < tok.children.length; child++) {
+        tok.children[child].position = tok.children[child].position + tok.position;
+      }
     }
   }
 };

lib/rules_inline/backticks.js

@@ -31,6 +31,8 @@ module.exports = function backtick(state, silent) {
         token.content = state.src.slice(pos, matchStart)
                                  .replace(/[ \n]+/g, ' ')
                                  .trim();
+        token.position = start;
+        token.size = matchEnd - token.position;
       }
       state.pos = matchEnd;
       return true;

lib/token.js

@@ -110,6 +110,20 @@ function Token(type, tag, nesting) {
    * to hide paragraphs.
    **/
   this.hidden   = false;
+
+  /**
+   * Token#position -> Number
+   *
+   * Position in the original string
+   **/
+  this.position     = 0;
+
+  /**
+   * Token#size -> Number
+   *
+   * Size of the token
+   **/
+  this.size     = 0;
 }
 
 

test/annotation.js

@@ -0,0 +1,146 @@
+'use strict';
+
+var assert = require('chai').assert;
+
+function assertTokenContent(src, token, content) {
+  assert.strictEqual(src.slice(token.position, token.position + token.size), content);
+}
+
+describe.only('Annotation', function() {
+  var md = require('../')({
+    html: true,
+    langPrefix: '',
+    typographer: false,
+    linkify: false
+  });
+
+  it('should annotate paragraph', function () {
+    var tokens = md.parse('Hello World\n\nThis is great !');
+    assert.strictEqual(tokens.length, 6);
+
+    // First paragraph
+    assert.strictEqual(tokens[0].position, 0);
+    assert.strictEqual(tokens[0].size, 0);
+    assert.strictEqual(tokens[1].position, 0);
+    assert.strictEqual(tokens[1].size, 11);
+    assert.strictEqual(tokens[2].position, 11);
+    assert.strictEqual(tokens[2].size, 0);
+
+    // Second paragraph
+    assert.strictEqual(tokens[3].position, 13);
+    assert.strictEqual(tokens[3].size, 0);
+    assert.strictEqual(tokens[4].position, 13);
+    assert.strictEqual(tokens[4].size, 15);
+    assert.strictEqual(tokens[5].position, 28);
+    assert.strictEqual(tokens[5].size, 0);
+  });
+
+  it('should annotate headings', function () {
+    var tokens = md.parse('# Hello\n\n## World ##\n');
+    assert.strictEqual(tokens.length, 6);
+
+    // First heading
+    assert.strictEqual(tokens[0].position, 0);
+    assert.strictEqual(tokens[0].size, 1);
+    assert.strictEqual(tokens[1].position, 1);
+    assert.strictEqual(tokens[1].size, 6);
+    assert.strictEqual(tokens[2].position, 7);
+    assert.strictEqual(tokens[2].size, 0);
+
+    // Second heading
+    assert.strictEqual(tokens[3].position, 9);
+    assert.strictEqual(tokens[3].size, 2);
+    assert.strictEqual(tokens[4].position, 11);
+    assert.strictEqual(tokens[4].size, 7);
+    assert.strictEqual(tokens[5].position, 18);
+    assert.strictEqual(tokens[5].size, 2);
+  });
+
+  it('should annotate lheadings', function () {
+    var src = 'Hello\n=====\n\nWorld\n=======';
+    var tokens = md.parse(src);
+    assert.strictEqual(tokens.length, 6);
+
+    // First heading
+    assert.strictEqual(tokens[0].position, 0);
+    assertTokenContent(src, tokens[0], '');
+    assertTokenContent(src, tokens[1], 'Hello');
+    assertTokenContent(src, tokens[2], '=====\n');
+
+    // Second heading
+    assert.strictEqual(tokens[3].position, 13);
+    assert.strictEqual(tokens[3].size, 0);
+    assertTokenContent(src, tokens[4], 'World');
+    assertTokenContent(src, tokens[5], '=======');
+  });
+
+  it('should annotate code blocks', function () {
+    var tokens = md.parse('\tHello\n\tWorld\n\nt\n\n\tBlock 2\n');
+    assert.strictEqual(tokens.length, 5);
+
+    assert.strictEqual(tokens[0].position, 0);
+    assert.strictEqual(tokens[0].size, 14);
+
+    assert.strictEqual(tokens[4].position, 18);
+    assert.strictEqual(tokens[4].size, 9);
+  });
+
+  it('should annotate tables', function () {
+    var src = 'Test:\n\n' +
+              ' | Type | Message |\n' +
+              ' | ---- | ------- |\n' +
+              '| Hello | World\n' +
+              ' | Bonjour | Monde |\n';
+    var tokens = md.parse(src);
+    assert.strictEqual(tokens.length, 33);
+
+    // Begin
+    assert.strictEqual(tokens[3].position, 7);
+    assert.strictEqual(tokens[3].size, 0);
+
+    // head (open)
+    assert.strictEqual(tokens[4].position, 7);
+    assert.strictEqual(tokens[4].size, 0);
+
+    // head -> TR (open)
+    assert.strictEqual(tokens[5].position, 7);
+    assert.strictEqual(tokens[5].size, 0);
+
+    // head -> columns
+    assertTokenContent(src, tokens[6], '|');
+    assertTokenContent(src, tokens[7], ' Type ');
+    assertTokenContent(src, tokens[8], '');
+    assertTokenContent(src, tokens[9], '|');
+    assertTokenContent(src, tokens[10], ' Message ');
+    assertTokenContent(src, tokens[11], '|');
+
+    // head -> TR (close)
+    assert.strictEqual(tokens[12].position, 26);
+    assert.strictEqual(tokens[12].size, 0);
+
+    // head (close)
+    assertTokenContent(src, tokens[13], ' | ---- | ------- |');
+
+    // body (open)
+    assert.strictEqual(tokens[14].position, 47);
+    assert.strictEqual(tokens[14].size, 0);
+
+    // body -> rows
+    assertTokenContent(src, tokens[16], '|');
+    assertTokenContent(src, tokens[17], ' Hello ');
+    assertTokenContent(src, tokens[18], '');
+    assertTokenContent(src, tokens[19], '|');
+    assertTokenContent(src, tokens[20], ' World');
+    assertTokenContent(src, tokens[21], '\n');
+
+    assertTokenContent(src, tokens[24], '|');
+    assertTokenContent(src, tokens[25], ' Bonjour ');
+    assertTokenContent(src, tokens[26], '');
+    assertTokenContent(src, tokens[27], '|');
+    assertTokenContent(src, tokens[28], ' Monde ');
+    assertTokenContent(src, tokens[29], '|');
+  });
+
+
+});
+