Browse Source

Reimplement lists using indent algorithm

And put blkIndents everywhere appropriate in block rules.
pull/14/head
Alex Kocharin 10 years ago
parent
commit
d6651b5ce6
  1. 6
      lib/lexer_block.js
  2. 7
      lib/lexer_block/code.js
  3. 18
      lib/lexer_block/fences.js
  4. 2
      lib/lexer_block/hr.js
  5. 3
      lib/lexer_block/lheading.js
  6. 68
      lib/lexer_block/list.js
  7. 2
      lib/lexer_block/paragraph.js

6
lib/lexer_block.js

@ -122,6 +122,10 @@ LexerBlock.prototype.tokenize = function (state, startLine, endLine, stopOnTwoNe
state.line = line = skipEmptyLines(state, line, endLine); state.line = line = skipEmptyLines(state, line, endLine);
if (line >= endLine) { break; } if (line >= endLine) { break; }
if (state.tShift[line] < state.blkIndent) { break; }
state.tight = !hasEmptyLines;
// Try all possible rules. // Try all possible rules.
// On success, rule should: // On success, rule should:
// //
@ -150,8 +154,6 @@ LexerBlock.prototype.tokenize = function (state, startLine, endLine, stopOnTwoNe
if (line < endLine && stopOnTwoNewlines && isEmpty(state, line)) { break; } if (line < endLine && stopOnTwoNewlines && isEmpty(state, line)) { break; }
} }
} }
state.tight = !hasEmptyLines;
}; };

7
lib/lexer_block/code.js

@ -10,7 +10,7 @@ var getLines = require('../helpers').getLines;
module.exports = function code(state, startLine, endLine, silent) { module.exports = function code(state, startLine, endLine, silent) {
var nextLine, last; var nextLine, last;
if (state.tShift[startLine] < 4) { return false; } if (state.tShift[startLine] - state.blkIndent < 4) { return false; }
last = nextLine = startLine + 1; last = nextLine = startLine + 1;
@ -22,7 +22,7 @@ module.exports = function code(state, startLine, endLine, silent) {
} }
continue; continue;
} }
if (state.tShift[nextLine] >= 4) { if (state.tShift[nextLine] - state.blkIndent >= 4) {
nextLine++; nextLine++;
last = nextLine; last = nextLine;
continue; continue;
@ -34,7 +34,8 @@ module.exports = function code(state, startLine, endLine, silent) {
state.tokens.push({ state.tokens.push({
type: 'code', type: 'code',
content: getLines(state, startLine, last, true).replace(/^ {1,4}/gm, '') content: getLines(state, startLine, last, true).replace(
new RegExp('^ {1,' + (4 + state.blkIndent) + '}', 'gm'), '')
}); });
state.line = nextLine; state.line = nextLine;

18
lib/lexer_block/fences.js

@ -10,6 +10,7 @@ var getLines = require('../helpers').getLines;
module.exports = function fences(state, startLine, endLine, silent) { module.exports = function fences(state, startLine, endLine, silent) {
var marker, len, params, nextLine, mem, var marker, len, params, nextLine, mem,
haveEndMarker = false,
pos = state.bMarks[startLine] + state.tShift[startLine], pos = state.bMarks[startLine] + state.tShift[startLine],
max = state.eMarks[startLine]; max = state.eMarks[startLine];
@ -40,7 +41,8 @@ module.exports = function fences(state, startLine, endLine, silent) {
nextLine = startLine; nextLine = startLine;
for (;;) { for (;;) {
if (nextLine + 1 >= endLine) { nextLine++;
if (nextLine >= endLine) {
// unclosed block should be autoclosed by end of document. // unclosed block should be autoclosed by end of document.
// also block seems to be autoclosed by end of parent // also block seems to be autoclosed by end of parent
/*if (state.blkLevel === 0) { /*if (state.blkLevel === 0) {
@ -50,11 +52,16 @@ module.exports = function fences(state, startLine, endLine, silent) {
break; break;
} }
nextLine++;
pos = mem = state.bMarks[nextLine] + state.tShift[nextLine]; pos = mem = state.bMarks[nextLine] + state.tShift[nextLine];
max = state.eMarks[nextLine]; max = state.eMarks[nextLine];
if (pos < max && state.tShift[nextLine] < state.blkIndent) {
// non-empty line with negative indent should stop the list:
// - ```
// test
break;
}
if (state.src.charCodeAt(pos) !== marker) { continue; } if (state.src.charCodeAt(pos) !== marker) { continue; }
pos = skipChars(state, pos, marker); pos = skipChars(state, pos, marker);
@ -67,11 +74,12 @@ module.exports = function fences(state, startLine, endLine, silent) {
if (pos < max) { continue; } if (pos < max) { continue; }
haveEndMarker = true;
// found! // found!
break; break;
} }
// If fense has heading spases, those should be removed from inner block // If a fence has heading spaces, they should be removed from its inner block
len = state.tShift[startLine]; len = state.tShift[startLine];
state.tokens.push({ state.tokens.push({
@ -84,6 +92,6 @@ module.exports = function fences(state, startLine, endLine, silent) {
.replace(RegExp('^ {1,' + len + '}', 'mg'), '') .replace(RegExp('^ {1,' + len + '}', 'mg'), '')
}); });
state.line = nextLine + 1; state.line = nextLine + (haveEndMarker ? 1 : 0);
return true; return true;
}; };

2
lib/lexer_block/hr.js

@ -12,7 +12,7 @@ module.exports = function hr(state, startLine, endLine, silent) {
max = state.eMarks[startLine]; max = state.eMarks[startLine];
// should not have > 3 leading spaces // should not have > 3 leading spaces
if (state.tShift[startLine] > 3) { return false; } if (state.tShift[startLine] - state.blkIndent > 3) { return false; }
pos += state.tShift[startLine]; pos += state.tShift[startLine];

3
lib/lexer_block/lheading.js

@ -13,9 +13,10 @@ module.exports = function lheading(state, startLine, endLine, silent) {
next = startLine + 1; next = startLine + 1;
if (next >= endLine) { return false; } if (next >= endLine) { return false; }
if (state.tShift[next] < state.blkIndent) { return false; }
// Scan next line // Scan next line
if (state.tShift[next] > 3) { return false; } if (state.tShift[next] - state.blkIndent > 3) { return false; }
pos = state.bMarks[next] + state.tShift[next]; pos = state.bMarks[next] + state.tShift[next];
max = state.eMarks[next]; max = state.eMarks[next];

68
lib/lexer_block/list.js

@ -12,8 +12,7 @@ var skipSpaces = require('../helpers').skipSpaces;
function skipBulletListMarker(state, startLine) { function skipBulletListMarker(state, startLine) {
var marker, pos, max; var marker, pos, max;
if (state.tShift[startLine] - state.blkIndent > 3) { return -1; }
if (state.tShift[startLine] > 3) { return -1; }
pos = state.bMarks[startLine] + state.tShift[startLine]; pos = state.bMarks[startLine] + state.tShift[startLine];
max = state.eMarks[startLine]; max = state.eMarks[startLine];
@ -36,7 +35,7 @@ function skipBulletListMarker(state, startLine) {
return pos; return pos;
} }
// Search `\d+\.[\n ]`, returns next pos arter marker on success // Search `\d+[.)][\n ]`, returns next pos arter marker on success
// or -1 on fail. // or -1 on fail.
function skipOrderedListMarker(state, startLine) { function skipOrderedListMarker(state, startLine) {
var ch, var ch,
@ -79,18 +78,16 @@ function skipOrderedListMarker(state, startLine) {
module.exports = function list(state, startLine, endLine, silent) { module.exports = function list(state, startLine, endLine, silent) {
var nextLine, var nextLine,
indent, indent,
oldIndent,
oldTight,
start, start,
posAfterMarker, posAfterMarker,
max, max,
indentAfterMarker, indentAfterMarker,
markerValue, markerValue,
isOrdered, isOrdered,
lastLine,
subState,
subString,
contentStart, contentStart,
listTokIdx, listTokIdx,
lineMax,
endOfList; endOfList;
//rules_named = state.lexerBlock.rules_named; //rules_named = state.lexerBlock.rules_named;
@ -131,10 +128,10 @@ module.exports = function list(state, startLine, endLine, silent) {
// //
nextLine = startLine; nextLine = startLine;
lineMax = state.lineMax;
endOfList = false; endOfList = false;
while (nextLine < endLine && !endOfList) { while (nextLine < endLine && !endOfList) {
if (state.tShift[startLine] < state.blkIndent) { return -1; }
if (isOrdered) { if (isOrdered) {
posAfterMarker = skipOrderedListMarker(state, nextLine); posAfterMarker = skipOrderedListMarker(state, nextLine);
if (posAfterMarker < 0) { break; } if (posAfterMarker < 0) { break; }
@ -165,56 +162,30 @@ module.exports = function list(state, startLine, endLine, silent) {
// ^^^^^ - calculating total length of this thing // ^^^^^ - calculating total length of this thing
indent = (posAfterMarker - state.bMarks[nextLine]) + indentAfterMarker; indent = (posAfterMarker - state.bMarks[nextLine]) + indentAfterMarker;
//
// Scan lines inside list items
//
lastLine = startLine;
// Run sublexer & write tokens // Run sublexer & write tokens
state.tokens.push({ type: 'list_item_open' }); state.tokens.push({ type: 'list_item_open' });
nextLine++; nextLine++;
for (;;) {
// if this line is indented more than with N spaces,
// it's the new paragraph of the same list item
if (nextLine < lineMax) {
if (isEmpty(state, nextLine)) {
nextLine++;
continue;
}
if (state.tShift[nextLine] >= indent) {
if (nextLine < endLine) { lastLine = nextLine; }
nextLine++;
continue;
}
}
if (lastLine < 0) { break; }
subString = state.src.slice(contentStart, state.eMarks[lastLine])
.replace(RegExp('^ {' + indent + '}', 'mg'), '');
if (lastLine < lineMax) {
// TODO: we should slice up to next empty line, not up to the end of the document
// (or even better - up to the next valid token)
//
// This has no impact on the algorithm except for performance
subString += state.src.slice(state.eMarks[lastLine]);
}
subState = state.clone(subString); oldIndent = state.blkIndent;
state.lexerBlock.tokenize(subState, 0, lastLine - startLine + 1, true); oldTight = state.tight;
nextLine = startLine = subState.line + startLine; state.blkIndent = state.tShift[startLine] = indent;
lastLine = -1;
contentStart = state.eMarks[startLine];
// TODO: need to detect loose type. state.lexerBlock.tokenize(state, startLine, endLine, true);
// Problem: blocks. separated by empty lines can be member of sublists.
// If any of list item is loose, mark list as loose // If any of list item is loose, mark list as loose
if (!subState.tight) { if (!state.tight || isEmpty(state, state.line - 1)) {
state.tokens[listTokIdx].tight = false; state.tokens[listTokIdx].tight = false;
} }
state.blkIndent = state.tShift[startLine] = oldIndent;
state.tight = oldTight;
state.tokens.push({ type: 'list_item_close' });
nextLine = startLine = state.line;
contentStart = state.bMarks[startLine];
if (nextLine >= endLine) { break; } if (nextLine >= endLine) { break; }
if (isEmpty(state, nextLine)) { if (isEmpty(state, nextLine)) {
@ -226,9 +197,6 @@ module.exports = function list(state, startLine, endLine, silent) {
} }
} }
state.tokens.push({ type: 'list_item_close' });
}
// Finilize list // Finilize list
if (isOrdered) { if (isOrdered) {
state.tokens.push({ type: 'ordered_list_close' }); state.tokens.push({ type: 'ordered_list_close' });

2
lib/lexer_block/paragraph.js

@ -32,7 +32,7 @@ module.exports = function paragraph(state, startLine/*, endLine*/) {
state.tokens.push({ type: 'paragraph_open' }); state.tokens.push({ type: 'paragraph_open' });
state.lexerInline.tokenize( state.lexerInline.tokenize(
state, state,
state.bMarks[startLine], state.bMarks[startLine] + state.tShift[startLine],
state.eMarks[nextLine - 1] state.eMarks[nextLine - 1]
); );
state.tokens.push({ type: 'paragraph_close' }); state.tokens.push({ type: 'paragraph_close' });

Loading…
Cancel
Save