Added lists and basic support for tables

lib/lexer_block.js

@@ -14,6 +14,7 @@ rules.push(require('./lexer_block/list'));
 rules.push(require('./lexer_block/heading'));
 rules.push(require('./lexer_block/lheading'));
 rules.push(require('./lexer_block/blockquote'));
+rules.push(require('./lexer_block/table'));
 rules.push(require('./lexer_block/paragraph'));
 
 

lib/lexer_block/list.js

@@ -3,78 +3,203 @@
 'use strict';
 
 
-var isEmpty = require('../helpers').isEmpty;
+var isEmpty        = require('../helpers').isEmpty;
+var getLines       = require('../helpers').getLines;
+var skipSpaces     = require('../helpers').skipSpaces;
 var skipEmptyLines = require('../helpers').skipEmptyLines;
 
 
-function bullet_item(state, startLine, endLine, silent) {
+function findEndOfItem(state, startLine, endLine, indent) {
-  var marker, nextLine,
+  var lastNonEmptyLine = startLine,
       rules_named = state.lexerBlock.rules_named,
+      nextLine = startLine + 1;
+
+  // jump line-by-line until empty one or EOF
+  for (; nextLine < endLine; nextLine++) {
+    if (isEmpty(state, nextLine)) {
+      // two successive newlines end the list
+      if (lastNonEmptyLine < nextLine - 1) { break; }
+      continue;
+    }
+
+    // if this line is indented more than with N spaces,
+    // it's the new paragraph of the same list item
+    if (state.tShift[nextLine] >= indent) {
+      lastNonEmptyLine = nextLine;
+      continue;
+    }
+
+    // paragraph after linebreak - not a continuation
+    if (lastNonEmptyLine < nextLine - 1) { break; }
+
+    // Otherwise it's a possible continuation, so we need to check
+    // other tags, same as with blockquote and paragraph.
+
+    if (rules_named.fences(state, nextLine, endLine, true)) { break; }
+    if (rules_named.hr(state, nextLine, endLine, true)) { break; }
+    if (rules_named.heading(state, nextLine, endLine, true)) { break; }
+    if (rules_named.lheading(state, nextLine, endLine, true)) { break; }
+    if (rules_named.blockquote(state, nextLine, endLine, true)) { break; }
+    if (rules_named.list(state, nextLine, endLine, true)) { break; }
+    //if (rules_named.tag(state, nextLine, endLine, true)) { break; }
+    //if (rules_named.def(state, nextLine, endLine, true)) { break; }
+    lastNonEmptyLine = nextLine;
+  }
+  return lastNonEmptyLine;
+}
+
+// skips `[-+*][\n ]`, returns -1 if not found
+function skipBulletListMarker(state, startLine/*, endLine*/) {
+  var marker,
       pos = state.bMarks[startLine] + state.tShift[startLine],
       max = state.eMarks[startLine];
 
-  // TODO: supporting list with only one paragraph for now
+  if (pos > max) { return -1; }
-
-  if (pos > max) { return false; }
 
   marker = state.src.charCodeAt(pos++);
-
   // Check bullet
   if (marker !== 0x2A/* * */ &&
       marker !== 0x2D/* - */ &&
       marker !== 0x2B/* + */) {
-    return false;
+    return -1;
   }
 
-  // Empty list item
+  if (pos < max && state.src.charCodeAt(pos) !== 0x20) {
-  if (pos > max) {
+    // " 1.test " - is not a list item
-    state.tokens.push({ type: 'list_item_open' });
+    return -1;
-    state.tokens.push({ type: 'list_item_close' });
-    return true;
   }
+  return pos;
+}
 
-  if (state.src.charCodeAt(pos++) !== 0x20) { return false; }
+// skips `\d+\.[\n ]`, returns -1 if not found
+function skipOrderedListMarker(state, startLine/*, endLine*/) {
+  var marker,
+      haveMarker = false,
+      pos = state.bMarks[startLine] + state.tShift[startLine],
+      max = state.eMarks[startLine];
 
-  // If we reached this, it's surely a list item
+  if (pos + 1 > max) { return -1; }
-  if (silent) { return true; }
 
-  nextLine = startLine + 1;
+  marker = state.src.charCodeAt(pos++);
+  if (marker < 0x30/* 0 */ || marker > 0x39/* 9 */) { return -1; }
 
-  // jump line-by-line until empty one or EOF
+  while (pos < max) {
-  while (nextLine < endLine && !isEmpty(state, nextLine)) {
+    marker = state.src.charCodeAt(pos++);
-    // Some tags can terminate paragraph without empty line.
+
-    // Try those tags in validation more (without tokens generation)
+    // found valid marker
-    if (rules_named.fences(state, nextLine, endLine, true)) { break; }
+    if (marker === 0x29 || marker === 0x2e) {
-    if (rules_named.hr(state, nextLine, endLine, true)) { break; }
+      haveMarker = true;
-    if (rules_named.heading(state, nextLine, endLine, true)) { break; }
+      break;
-    if (rules_named.lheading(state, nextLine, endLine, true)) { break; }
+    }
-    if (rules_named.blockquote(state, nextLine, endLine, true)) { break; }
+
-    if (bullet_item(state, nextLine, endLine, true)) { break; }
+    // still skipping digits...
-    //if (rules_named.tag(state, nextLine, endLine, true)) { break; }
+    if (marker < 0x30/* 0 */ || marker > 0x39/* 9 */) { return -1; }
-    //if (rules_named.def(state, nextLine, endLine, true)) { break; }
+  }
-    nextLine++;
+
+  if (!haveMarker) {
+    // " 1\n"
+    return -1;
+  }
+  if (pos < max && state.src.charCodeAt(pos) !== 0x20) {
+    // " 1.test " - is not a list item
+    return -1;
+  }
+  return pos;
+}
+
+function bullet_item(state, startLine, endLine, isOrdered) {
+  var indentAfterMarker, indent, start, lastLine, subState, pos,
+      max = state.eMarks[startLine];
+
+  if (isOrdered) {
+    pos = skipOrderedListMarker(state, startLine, endLine);
+  } else {
+    pos = skipBulletListMarker(state, startLine, endLine);
+  }
+
+  if (pos === -1) { return false; }
+
+  start = pos;
+  pos = skipSpaces(state, pos);
+
+  if (pos >= max) {
+    // trimming space in "-    \n  3" case, indent is 1 here
+    indentAfterMarker = 1;
+  } else {
+    indentAfterMarker = pos - start;
   }
 
+  // If we have more than 4 spaces, the indent is 1
+  // (the rest is just indented code block)
+  if (indentAfterMarker > 4) { indentAfterMarker = 1; }
+
+  // If indent is less than 1, assume that it's one, example:
+  //  "-\n  test"
+  if (indentAfterMarker < 1) { indentAfterMarker = 1; }
+
+  // "  -  test"
+  //  ^^^^^ - calculating total length of this thing
+  indent = state.tShift[startLine] + indentAfterMarker + (isOrdered ? 2 : 1);
+
+  lastLine = findEndOfItem(state, startLine, endLine, indent);
+
   state.tokens.push({ type: 'list_item_open' });
-  state.lexerInline.tokenize(
+  /*state.lexerInline.tokenize(
     state,
     state.bMarks[startLine],
-    state.eMarks[nextLine - 1]
+    state.eMarks[lastLine]
-  );
+  );*/
+  subState = state.clone(getLines(state, startLine, lastLine + 1, true)
+              .replace(RegExp('^ {' + indent + '}', 'mg'), '').substr(indent));
+  state.lexerBlock.tokenize(subState, 0, subState.lineMax);
+
   state.tokens.push({ type: 'list_item_close' });
 
-  state.line = skipEmptyLines(state, nextLine);
+  state.line = lastLine + 1;
   return true;
 }
 
 module.exports = function list(state, startLine, endLine, silent) {
-  // TODO: supporting list with only one element for now
+  var line, start, markerInt,
-  if (bullet_item(state, startLine, endLine, true)) {
+      orderedMarker = skipOrderedListMarker(state, startLine, endLine),
-    if (silent) { return true; }
+      bulletMarker = skipBulletListMarker(state, startLine, endLine),
+      isOrdered;
+
+  if (orderedMarker === -1 && bulletMarker === -1) { return false; }
+  if (silent) { return true; }
+
+  isOrdered = orderedMarker !== -1;
+  if (isOrdered) {
+    start = state.bMarks[startLine] + state.tShift[startLine];
+    markerInt = Number(state.src.substr(start, orderedMarker - start));
+    if (markerInt > 1) {
+      state.tokens.push({ type: 'ordered_list_open', start: markerInt });
+    } else {
+      state.tokens.push({ type: 'ordered_list_open' });
+    }
+  } else {
     state.tokens.push({ type: 'bullet_list_open' });
-    bullet_item(state, startLine, endLine);
+  }
+
+  line = startLine;
+  while (line < endLine) {
+    if (bullet_item(state, line, endLine, isOrdered)) {
+      line = state.line;
+
+      // if we have a trailing newline, skip it;
+      // if we have more than one, it should end the list,
+      // so can't use skipEmptyNewlines here
+      if (line < endLine && isEmpty(state, line)) { line++; }
+    } else {
+      break;
+    }
+  }
+
+  if (isOrdered) {
+    state.tokens.push({ type: 'ordered_list_close' });
+  } else {
     state.tokens.push({ type: 'bullet_list_close' });
-    return true;
   }
-  return false;
+  state.line = skipEmptyLines(state, state.line);
+  return true;
 };

lib/lexer_block/table.js

@@ -0,0 +1,82 @@
+// GFM table, non-standard
+
+'use strict';
+
+
+var skipEmptyLines  = require('../helpers').skipEmptyLines;
+
+
+function lineMatch(state, line, reg) {
+  var pos = state.bMarks[line],
+      max = state.eMarks[line];
+
+  return state.src.substr(pos, max - pos).match(reg);
+}
+
+
+module.exports = function table(state, startLine, endLine, silent) {
+  var ch, firstLineMatch, secondLineMatch, i, subState, nextLine, m, rows,
+      aligns, t;
+
+  // should have at least three lines
+  if (startLine + 2 > endLine) { return false; }
+
+  // first character of the second line should be '|' or '-'
+  ch = state.src.charCodeAt(state.bMarks[startLine + 1]
+     + state.tShift[startLine + 1]);
+  if (ch !== 0x7C/* | */ && ch !== 0x2D/* - */) { return false; }
+
+  secondLineMatch = lineMatch(state, startLine + 1,
+    /^ *\|?(( *[:-]-+[:-] *\|)+( *[:-]-+[:-] *))\|? *$/);
+  if (!secondLineMatch) { return false; }
+
+  rows = secondLineMatch[1].split('|');
+  aligns = [];
+  for (i = 0; i < rows.length; i++) {
+    t = rows[i].trim();
+    if (t[t.length - 1] === ':') {
+      aligns[i] = t[0] === ':' ? 'center' : 'right';
+    } else if (t[0] === ':') {
+      aligns[i] = 'left';
+    } else {
+      aligns[i] = '';
+    }
+  }
+
+  firstLineMatch = lineMatch(state, startLine, /^ *\|?(.*?\|.*?)\|? *$/);
+  if (!firstLineMatch) { return false; }
+
+  rows = firstLineMatch[1].split('|');
+  if (aligns.length !== rows.length) { return false; }
+  if (silent) { return true; }
+
+  state.tokens.push({ type: 'table_open' });
+  state.tokens.push({ type: 'tr_open' });
+
+  for (i = 0; i < rows.length; i++) {
+    state.tokens.push({ type: 'th_open', align: aligns[i] });
+    subState = state.clone(rows[i].trim());
+    state.lexerInline.tokenize(subState, 0, subState.eMarks[0]);
+    state.tokens.push({ type: 'th_close' });
+  }
+  state.tokens.push({ type: 'tr_close' });
+
+  for (nextLine = startLine + 2; nextLine < endLine; nextLine++) {
+    m = lineMatch(state, nextLine, /^ *\|?(.*?\|.*?)\|? *$/);
+    if (!m) { break; }
+    rows = m[1].split('|');
+
+    state.tokens.push({ type: 'tr_open' });
+    for (i = 0; i < rows.length; i++) {
+      state.tokens.push({ type: 'td_open', align: aligns[i] });
+      subState = state.clone(rows[i].replace(/^\|? *| *\|?$/g, ''));
+      state.lexerInline.tokenize(subState, 0, subState.eMarks[0]);
+      state.tokens.push({ type: 'td_close' });
+    }
+    state.tokens.push({ type: 'tr_close' });
+  }
+  state.tokens.push({ type: 'table_close' });
+
+  state.line = skipEmptyLines(state, nextLine);
+  return true;
+};

lib/renderer.js

@@ -75,6 +75,16 @@ rules.list_item_close = function (state /*, token*/) {
 };
 
 
+rules.ordered_list_open = function (state, token) {
+  state.result += '<ol'
+    + (token.start ? ' start="' + token.start + '"' : '')
+    + '>\n';
+};
+rules.ordered_list_close = function (state /*, token*/) {
+  state.result += '</ol>\n';
+};
+
+
 rules.paragraph_open = function (state /*, token*/) {
   state.result += '<p>';
 };
@@ -83,6 +93,36 @@ rules.paragraph_close = function (state /*, token*/) {
 };
 
 
+rules.table_open = function (state /*, token*/) {
+  state.result += '<table>\n';
+};
+rules.table_close = function (state /*, token*/) {
+  state.result += '</table>\n';
+};
+rules.tr_open = function (state /*, token*/) {
+  state.result += '<tr>\n';
+};
+rules.tr_close = function (state /*, token*/) {
+  state.result += '</tr>\n';
+};
+rules.th_open = function (state, token) {
+  state.result += '<th'
+    + (token.align ? ' align="' + token.align + '"' : '')
+    + '>';
+};
+rules.th_close = function (state /*, token*/) {
+  state.result += '</th>\n';
+};
+rules.td_open = function (state, token) {
+  state.result += '<td'
+    + (token.align ? ' align="' + token.align + '"' : '')
+    + '>';
+};
+rules.td_close = function (state /*, token*/) {
+  state.result += '</td>\n';
+};
+
+
 rules.text = function (state, token) {
   state.result += escapeHtml(unescapeMd(token.content));
 };