Reimplement lists using indent algorithm

And put blkIndents everywhere appropriate in block rules.

lib/lexer_block.js

@@ -122,6 +122,10 @@ LexerBlock.prototype.tokenize = function (state, startLine, endLine, stopOnTwoNe
     state.line = line = skipEmptyLines(state, line, endLine);
     if (line >= endLine) { break; }
 
+    if (state.tShift[line] < state.blkIndent) { break; }
+
+    state.tight = !hasEmptyLines;
+
     // Try all possible rules.
     // On success, rule should:
     //
@@ -150,8 +154,6 @@ LexerBlock.prototype.tokenize = function (state, startLine, endLine, stopOnTwoNe
       if (line < endLine && stopOnTwoNewlines && isEmpty(state, line)) { break; }
     }
   }
-
-  state.tight = !hasEmptyLines;
 };
 
 

lib/lexer_block/code.js

@@ -10,7 +10,7 @@ var getLines = require('../helpers').getLines;
 module.exports = function code(state, startLine, endLine, silent) {
   var nextLine, last;
 
-  if (state.tShift[startLine] < 4) { return false; }
+  if (state.tShift[startLine] - state.blkIndent < 4) { return false; }
 
   last = nextLine = startLine + 1;
 
@@ -22,7 +22,7 @@ module.exports = function code(state, startLine, endLine, silent) {
       }
       continue;
     }
-    if (state.tShift[nextLine] >= 4) {
+    if (state.tShift[nextLine] - state.blkIndent >= 4) {
       nextLine++;
       last = nextLine;
       continue;
@@ -34,7 +34,8 @@ module.exports = function code(state, startLine, endLine, silent) {
 
   state.tokens.push({
     type: 'code',
-    content: getLines(state, startLine, last, true).replace(/^ {1,4}/gm, '')
+    content: getLines(state, startLine, last, true).replace(
+      new RegExp('^ {1,' + (4 + state.blkIndent) + '}', 'gm'), '')
   });
 
   state.line = nextLine;

lib/lexer_block/fences.js

@@ -10,6 +10,7 @@ var getLines        = require('../helpers').getLines;
 
 module.exports = function fences(state, startLine, endLine, silent) {
   var marker, len, params, nextLine, mem,
+      haveEndMarker = false,
       pos = state.bMarks[startLine] + state.tShift[startLine],
       max = state.eMarks[startLine];
 
@@ -40,7 +41,8 @@ module.exports = function fences(state, startLine, endLine, silent) {
   nextLine = startLine;
 
   for (;;) {
-    if (nextLine + 1 >= endLine) {
+    nextLine++;
+    if (nextLine >= endLine) {
       // unclosed block should be autoclosed by end of document.
       // also block seems to be autoclosed by end of parent
       /*if (state.blkLevel === 0) {
@@ -50,11 +52,16 @@ module.exports = function fences(state, startLine, endLine, silent) {
       break;
     }
 
-    nextLine++;
-
     pos = mem = state.bMarks[nextLine] + state.tShift[nextLine];
     max = state.eMarks[nextLine];
 
+    if (pos < max && state.tShift[nextLine] < state.blkIndent) {
+      // non-empty line with negative indent should stop the list:
+      // - ```
+      //  test
+      break;
+    }
+
     if (state.src.charCodeAt(pos) !== marker) { continue; }
 
     pos = skipChars(state, pos, marker);
@@ -67,11 +74,12 @@ module.exports = function fences(state, startLine, endLine, silent) {
 
     if (pos < max) { continue; }
 
+    haveEndMarker = true;
     // found!
     break;
   }
 
-  // If fense has heading spases, those should be removed from inner block
+  // If a fence has heading spaces, they should be removed from its inner block
   len = state.tShift[startLine];
 
   state.tokens.push({
@@ -84,6 +92,6 @@ module.exports = function fences(state, startLine, endLine, silent) {
                   .replace(RegExp('^ {1,' + len + '}', 'mg'), '')
   });
 
-  state.line = nextLine + 1;
+  state.line = nextLine + (haveEndMarker ? 1 : 0);
   return true;
 };

lib/lexer_block/hr.js

@@ -12,7 +12,7 @@ module.exports = function hr(state, startLine, endLine, silent) {
       max = state.eMarks[startLine];
 
   // should not have > 3 leading spaces
-  if (state.tShift[startLine] > 3) { return false; }
+  if (state.tShift[startLine] - state.blkIndent > 3) { return false; }
 
   pos += state.tShift[startLine];
 

lib/lexer_block/lheading.js

@@ -13,9 +13,10 @@ module.exports = function lheading(state, startLine, endLine, silent) {
       next = startLine + 1;
 
   if (next >= endLine) { return false; }
+  if (state.tShift[next] < state.blkIndent) { return false; }
 
   // Scan next line
-  if (state.tShift[next] > 3) { return false; }
+  if (state.tShift[next] - state.blkIndent > 3) { return false; }
 
   pos = state.bMarks[next] + state.tShift[next];
   max = state.eMarks[next];

lib/lexer_block/list.js

@@ -12,8 +12,7 @@ var skipSpaces = require('../helpers').skipSpaces;
 function skipBulletListMarker(state, startLine) {
   var marker, pos, max;
 
-
+  if (state.tShift[startLine] - state.blkIndent > 3) { return -1; }
-  if (state.tShift[startLine] > 3) { return -1; }
 
   pos = state.bMarks[startLine] + state.tShift[startLine];
   max = state.eMarks[startLine];
@@ -36,7 +35,7 @@ function skipBulletListMarker(state, startLine) {
   return pos;
 }
 
-// Search `\d+\.[\n ]`, returns next pos arter marker on success
+// Search `\d+[.)][\n ]`, returns next pos arter marker on success
 // or -1 on fail.
 function skipOrderedListMarker(state, startLine) {
   var ch,
@@ -79,18 +78,16 @@ function skipOrderedListMarker(state, startLine) {
 module.exports = function list(state, startLine, endLine, silent) {
   var nextLine,
       indent,
+      oldIndent,
+      oldTight,
       start,
       posAfterMarker,
       max,
       indentAfterMarker,
       markerValue,
       isOrdered,
-      lastLine,
-      subState,
-      subString,
       contentStart,
       listTokIdx,
-      lineMax,
       endOfList;
       //rules_named = state.lexerBlock.rules_named;
 
@@ -131,10 +128,10 @@ module.exports = function list(state, startLine, endLine, silent) {
   //
 
   nextLine = startLine;
-  lineMax = state.lineMax;
   endOfList = false;
 
   while (nextLine < endLine && !endOfList) {
+    if (state.tShift[startLine] < state.blkIndent) { return -1; }
     if (isOrdered) {
       posAfterMarker = skipOrderedListMarker(state, nextLine);
       if (posAfterMarker < 0) { break; }
@@ -165,68 +162,39 @@ module.exports = function list(state, startLine, endLine, silent) {
     //  ^^^^^ - calculating total length of this thing
     indent = (posAfterMarker - state.bMarks[nextLine]) + indentAfterMarker;
 
-    //
-    // Scan lines inside list items
-    //
-    lastLine = startLine;
-
     // Run sublexer & write tokens
     state.tokens.push({ type: 'list_item_open' });
 
     nextLine++;
-    for (;;) {
-      // if this line is indented more than with N spaces,
-      // it's the new paragraph of the same list item
-      if (nextLine < lineMax) {
-        if (isEmpty(state, nextLine)) {
-          nextLine++;
-          continue;
-        }
-        if (state.tShift[nextLine] >= indent) {
-          if (nextLine < endLine) { lastLine = nextLine; }
-          nextLine++;
-          continue;
-        }
-      }
 
-      if (lastLine < 0) { break; }
+    oldIndent = state.blkIndent;
+    oldTight = state.tight;
+    state.blkIndent = state.tShift[startLine] = indent;
 
-      subString = state.src.slice(contentStart, state.eMarks[lastLine])
+    state.lexerBlock.tokenize(state, startLine, endLine, true);
-                    .replace(RegExp('^ {' + indent + '}', 'mg'), '');
+
-      if (lastLine < lineMax) {
+    // If any of list item is loose, mark list as loose
-        // TODO: we should slice up to next empty line, not up to the end of the document
+    if (!state.tight || isEmpty(state, state.line - 1)) {
-        // (or even better - up to the next valid token)
+      state.tokens[listTokIdx].tight = false;
-        //
+    }
-        // This has no impact on the algorithm except for performance
-        subString += state.src.slice(state.eMarks[lastLine]);
-      }
 
-      subState = state.clone(subString);
+    state.blkIndent = state.tShift[startLine] = oldIndent;
-      state.lexerBlock.tokenize(subState, 0, lastLine - startLine + 1, true);
+    state.tight = oldTight;
-      nextLine = startLine = subState.line + startLine;
-      lastLine = -1;
-      contentStart = state.eMarks[startLine];
 
-      // TODO: need to detect loose type.
+    state.tokens.push({ type: 'list_item_close' });
-      // Problem: blocks. separated by empty lines can be member of sublists.
 
-      // If any of list item is loose, mark list as loose
+    nextLine = startLine = state.line;
-      if (!subState.tight) {
+    contentStart = state.bMarks[startLine];
-        state.tokens[listTokIdx].tight = false;
-      }
 
-      if (nextLine >= endLine) { break; }
+    if (nextLine >= endLine) { break; }
 
-      if (isEmpty(state, nextLine)) {
+    if (isEmpty(state, nextLine)) {
-        nextLine++;
+      nextLine++;
-        if (nextLine >= endLine || isEmpty(state, nextLine)) {
+      if (nextLine >= endLine || isEmpty(state, nextLine)) {
-          // two newlines end the list
+        // two newlines end the list
-          break;
+        break;
-        }
       }
     }
-
-    state.tokens.push({ type: 'list_item_close' });
   }
 
   // Finilize list

lib/lexer_block/paragraph.js

@@ -32,7 +32,7 @@ module.exports = function paragraph(state, startLine/*, endLine*/) {
   state.tokens.push({ type: 'paragraph_open' });
   state.lexerInline.tokenize(
     state,
-    state.bMarks[startLine],
+    state.bMarks[startLine] + state.tShift[startLine],
     state.eMarks[nextLine - 1]
   );
   state.tokens.push({ type: 'paragraph_close' });