Inline lexer draft

.eslintrc

@@ -17,7 +17,7 @@ rules:
   eqeqeq:               2
   guard-for-in:         2
   handle-callback-err:  2
-  max-depth:            [ 1, 5 ]
+  max-depth:            [ 1, 6 ]
   max-nested-callbacks: [ 1, 4 ]
   # string can exceed 80 chars, but should not overflow github website :)
   max-len:              [ 2, 120, 1000 ]

lib/lexer_block/heading.js

@@ -53,7 +53,10 @@ module.exports = function heading(state, startLine, endLine, silent) {
   state.tokens.push({ type: 'heading_open', level: level });
   // only if header is not empty
   if (pos < max) {
-    state.lexerInline.tokenize(state, state.src.slice(pos, max));
+    state.tokens.push({
+      type: 'inline',
+      content: state.src.slice(pos, max)
+    });
   }
   state.tokens.push({ type: 'heading_close', level: level });
 

lib/lexer_block/lheading.js

@@ -38,7 +38,10 @@ module.exports = function lheading(state, startLine, endLine, silent) {
   max = skipCharsBack(state, state.eMarks[startLine], 0x20/* space */, pos);
 
   state.tokens.push({ type: 'heading_open', level: marker === 0x3D/* = */ ? 1 : 2 });
-  state.lexerInline.tokenize(state, state.src.slice(pos, max));
+  state.tokens.push({
+    type: 'inline',
+    content: state.src.slice(pos, max)
+  });
   state.tokens.push({ type: 'heading_close', level: marker === 0x3D/* = */ ? 1 : 2 });
 
   state.line = next + 1;

lib/lexer_block/paragraph.js

@@ -35,10 +35,10 @@ module.exports = function paragraph(state, startLine/*, endLine*/) {
   }
 
   state.tokens.push({ type: 'paragraph_open' });
-  state.lexerInline.tokenize(
-    state,
-    getLines(state, startLine, nextLine, state.blkIndent, false)
-  );
+  state.tokens.push({
+    type: 'inline',
+    content: getLines(state, startLine, nextLine, state.blkIndent, false)
+  });
   state.tokens.push({ type: 'paragraph_close' });
 
   state.line = nextLine;

lib/lexer_inline.js

@@ -3,6 +3,8 @@
 'use strict';
 
 
+var StateInline = require('./state_inline');
+
 ////////////////////////////////////////////////////////////////////////////////
 // Lexer rules
 
@@ -10,15 +12,13 @@ var rules = [];
 
 
 // Pure text
-rules.push(function text(state, str, begin, end) {
-  state.tokens.push({
-    type: 'text',
-    content: str.slice(begin, end).replace(/^ {1,}/mg, '')
-  });
-
-  state.pos = end;
-  return true;
-});
+rules.push(require('./lexer_inline/text'));
+rules.push(require('./lexer_inline/newline'));
+rules.push(require('./lexer_inline/escape'));
+//
+//
+rules.push(require('./lexer_inline/entity'));
+rules.push(require('./lexer_inline/escape_html_char'));
 
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -47,6 +47,9 @@ function findByName(self, name) {
 function LexerInline() {
   this.rules = [];
 
+  // Rule to skip pure text
+  this.textMatch = /^[^\n\\`*_\[!<&>"]+/;
+
   for (var i = 0; i < rules.length; i++) {
     this.after(null, rules[i]);
   }
@@ -107,14 +110,13 @@ LexerInline.prototype.after = function (name, fn) {
 
 // Generate tokens for input range
 //
-LexerInline.prototype.tokenize = function (state, str) {
+LexerInline.prototype.tokenize = function (state) {
   var ok, i,
       rules = this.rules,
       len = this.rules.length,
-      pos = 0,
-      end = str.length;
+      end = state.src.length;
 
-  while (pos < end) {
+  while (state.pos < end) {
 
     // Try all possible rules.
     // On success, rule should:
@@ -124,17 +126,32 @@ LexerInline.prototype.tokenize = function (state, str) {
     // - return true
 
     for (i = 0; i < len; i++) {
-      ok = rules[i](state, str, pos, end);
+      ok = rules[i](state);
       if (ok) { break; }
     }
 
     if (ok) {
-      pos = state.pos;
+      if (state.pos >= end) { break; }
       continue;
     }
+
+    state.pending += state.src[state.pos++];
   }
 
-  state.pos = end;
+  if (state.pending) {
+    state.pushText();
+  }
+
+  return state.tokens;
+};
+
+// Parse input string.
+//
+LexerInline.prototype.parse = function (str, options) {
+  var state = new StateInline(str, this, options);
+
+  this.tokenize(state);
+  return state.tokens;
 };
 
 

lib/lexer_inline/entity.js

@@ -0,0 +1,43 @@
+// Proceess html entity - &#123, &#xAF, &quot
+
+
+var DIGITAL_RE = /^(?:x[a-f0-9]{1,8}|[0-9]{1,8});/i;
+var NAMED_RE   = /^[a-z][a-z0-9]{1,31};/i;
+
+
+module.exports = function entity(state) {
+  var ch, match, pos = state.pos, max = state.posMax;
+
+
+  if (state.src.charCodeAt(pos) !== 0x26/* & */) { return false; }
+
+  pos++;
+
+  if (pos >= max) {
+    state.pending += '&';
+    state.pos++;
+    return true;
+  }
+
+  ch = state.src.charCodeAt(pos);
+
+  if (ch === 0x23 /* # */) {
+    match = state.src.slice(pos + 1).match(DIGITAL_RE);
+    if (match) {
+      state.pending += '&#' + match[0];
+      state.pos += match[0].length + 2;
+      return true;
+    }
+  } else {
+    match = state.src.slice(pos).match(NAMED_RE);
+    if (match) {
+      state.pending += '&' + match[0];
+      state.pos += match[0].length + 1;
+      return true;
+    }
+  }
+
+  state.pending += '&';
+  state.pos++;
+  return true;
+};

lib/lexer_inline/escape.js

@@ -0,0 +1,46 @@
+// Proceess escaped chars and hardbreaks
+
+var ESCAPED = '\\!"#$%&\'()*+,./:;<=>?@[]^_`{|}~-]'
+                      .split('')
+                      .map(function(ch) { return ch.charCodeAt(0); });
+
+module.exports = function escape(state) {
+  var ch, pos = state.pos, max = state.posMax;
+
+  if (state.src.charCodeAt(pos) !== 0x5C/* \ */) { return false; }
+
+  pos++;
+
+  if (pos < max) {
+    ch = state.src.charCodeAt(pos);
+
+    if (ESCAPED.indexOf(ch) >= 0) {
+      // escape html chars if needed
+      if (ch === 0x26/* & */) {
+        state.pending += '&amp;';
+      } else if (ch === 0x3C/* < */) {
+        state.pending += '&lt;';
+      } else if (ch === 0x3E/* > */) {
+        state.pending += '&gt;';
+      } else if (ch === 0x22/* " */) {
+        state.pending += '&quot;';
+      } else {
+        state.pending += state.src[pos];
+      }
+      state.pos += 2;
+      return true;
+    }
+
+    if (ch === 0x0A) {
+      state.push({
+        type: 'hardbreak'
+      });
+      state.pos += 2;
+      return true;
+    }
+  }
+
+  state.pending += '\\';
+  state.pos++;
+  return true;
+};

lib/lexer_inline/escape_html_char.js

@@ -0,0 +1,18 @@
+// Process < > " (& was processed in markdown escape)
+
+module.exports = function escape_html_char(state) {
+  var ch = state.src.charCodeAt(state.pos);
+
+  if (ch === 0x3C/* < */) {
+    state.pending += '&lt;';
+  } else if (ch === 0x3E/* > */) {
+    state.pending += '&gt;';
+  } else if (ch === 0x22/* " */) {
+    state.pending += '&quot;';
+  } else {
+    return false;
+  }
+
+  state.pos++;
+  return true;
+};

lib/lexer_inline/newline.js

@@ -0,0 +1,32 @@
+// Proceess '\n'
+
+module.exports = function escape(state) {
+  var pmax, pos = state.pos;
+
+  if (state.src.charCodeAt(pos) !== 0x0A/* \n */) { return false; }
+
+  pmax = state.pending.length - 1;
+
+  // '  \n' -> hardbreak
+  if (pmax >= 1 &&
+      state.pending.charCodeAt(pmax) === 0x20 &&
+      state.pending.charCodeAt(pmax - 1) === 0x20) {
+    state.pending = state.pending.slice(0, -2);
+    state.push({
+      type: 'hardbreak'
+    });
+    state.pos++;
+    return true;
+  }
+
+  state.pending = state.pending.trim() + '\n';
+
+  pos++;
+
+  // skip spaces to simplify trim
+  while (state.src.charCodeAt(pos) === 0x20) { pos++; }
+
+  state.pos = pos;
+
+  return true;
+};

lib/lexer_inline/text.js

@@ -0,0 +1,13 @@
+// Skip text characters for text token, place those to pendibg buffer
+// and increment current pos
+
+module.exports = function text(state) {
+  var match = state.src.slice(state.pos).match(state.lexer.textMatch);
+
+  if (!match) { return false; }
+
+  state.pending += match[0];
+  state.pos += match[0].length;
+
+  return true;
+};

lib/parser.js

@@ -32,7 +32,7 @@ Parser.prototype.set = function (options) {
 
 
 Parser.prototype.render = function (src) {
-  var state, lineStart = 0, lastTabPos = 0;
+  var state, lineStart = 0, lastTabPos = 0, tokens, i, l, t;
 
   if (!src) { return ''; }
 
@@ -74,9 +74,20 @@ Parser.prototype.render = function (src) {
     this.options
   );
 
+  // Parse blocks
   state.lexerBlock.tokenize(state, state.line, state.lineMax);
 
-  return this.renderer.render(state);
+  // Parse inlines
+  tokens = state.tokens;
+  for (i = 0, l = tokens.length; i < l; i++) {
+    t = tokens[i];
+    if (t.type === 'inline') {
+      t.children = state.lexerInline.parse(t.content, state.options);
+    }
+  }
+
+  // Render
+  return this.renderer.render(state.tokens, this.options);
 };
 
 

lib/renderer.js

@@ -11,17 +11,11 @@ function escapeHtml(str) {
             .replace(/"/g, '"');
 }
 
-var MD_UNESCAPE_RE = /\\([!"#$%&\'()*+,.\/:;<=>?@[\\\]^_`{|}~-])/g;
-
-function unescapeMd(str) {
-  return str.replace(MD_UNESCAPE_RE, '$1');
-}
-
 
 // check if we need to hide '\n' before next token
-function getBreak(state, idx) {
-  if (++idx < state.tokens.length &&
-      state.tokens[idx].type === 'list_item_close') {
+function getBreak(tokens, idx) {
+  if (++idx < tokens.length &&
+      tokens[idx].type === 'list_item_close') {
     return '';
   }
 
@@ -32,113 +26,122 @@ function getBreak(state, idx) {
 var rules = {};
 
 
-rules.blockquote_open = function (state /*, token, idx*/) {
-  state.result += '<blockquote>\n';
+rules.blockquote_open = function (/*tokens, idx, options*/) {
+  return '<blockquote>\n';
 };
-rules.blockquote_close = function (state, token, idx) {
-  state.result += '</blockquote>' + getBreak(state, idx);
+rules.blockquote_close = function (tokens, idx /*, options*/) {
+  return '</blockquote>' + getBreak(tokens, idx);
 };
 
 
-rules.code = function (state, token, idx) {
-  state.result += '<pre><code>' + escapeHtml(token.content) + '</code></pre>' + getBreak(state, idx);
+rules.code = function (tokens, idx /*, options*/) {
+  return '<pre><code>' + escapeHtml(tokens[idx].content) + '</code></pre>' + getBreak(tokens, idx);
 };
 
 
-rules.fence = function (state, token, idx) {
+rules.fence = function (tokens, idx, options) {
+  var token = tokens[idx];
   var langMark = '';
-  var langPrefix = state.options.codeLangPrefix || '';
+  var langPrefix = options.codeLangPrefix || '';
 
   if (token.params.length) {
     langMark = ' class="' + langPrefix + escapeHtml(token.params[0]) + '"';
   }
 
-  state.result += '<pre><code' + langMark + '>'
-                  + escapeHtml(token.content)
-                  + '</code></pre>' + getBreak(state, idx);
+  return  '<pre><code' + langMark + '>'
+        + escapeHtml(token.content)
+        + '</code></pre>' + getBreak(tokens, idx);
 };
 
 
-rules.heading_open = function (state, token) {
-  state.result += '<h' + token.level + '>';
+rules.heading_open = function (tokens, idx /*, options*/) {
+  return '<h' + tokens[idx].level + '>';
 };
-rules.heading_close = function (state, token) {
-  state.result += '</h' + token.level + '>\n';
+rules.heading_close = function (tokens, idx /*, options*/) {
+  return '</h' + tokens[idx].level + '>\n';
 };
 
 
-rules.hr = function (state, token, idx) {
-  state.result += (state.options.xhtml ? '<hr />' : '<hr>') + getBreak(state, idx);
+rules.hr = function (tokens, idx, options) {
+  return (options.xhtml ? '<hr />' : '<hr>') + getBreak(tokens, idx);
 };
 
 
-rules.bullet_list_open = function (state /*, token, idx*/) {
-  state.result += '<ul>\n';
+rules.bullet_list_open = function (/*tokens, idx, options*/) {
+  return '<ul>\n';
 };
-rules.bullet_list_close = function (state, token, idx) {
-  state.result += '</ul>' + getBreak(state, idx);
+rules.bullet_list_close = function (tokens, idx /*, options*/) {
+  return '</ul>' + getBreak(tokens, idx);
 };
-rules.list_item_open = function (state /*, token, idx*/) {
-  state.result += '<li>';
+rules.list_item_open = function (/*tokens, idx, options*/) {
+  return '<li>';
 };
-rules.list_item_close = function (state /*, token, idx*/) {
-  state.result += '</li>\n';
+rules.list_item_close = function (/*tokens, idx, options*/) {
+  return '</li>\n';
 };
-rules.ordered_list_open = function (state, token /*, idx */) {
-  state.result += '<ol'
+rules.ordered_list_open = function (tokens, idx /*, options*/) {
+  var token = tokens[idx];
+  return '<ol'
     + (token.order > 1 ? ' start="' + token.order + '"' : '')
     + '>\n';
 };
-rules.ordered_list_close = function (state, token, idx) {
-  state.result += '</ol>' + getBreak(state, idx);
+rules.ordered_list_close = function (tokens, idx /*, options*/) {
+  return '</ol>' + getBreak(tokens, idx);
 };
 
 
-rules.paragraph_open = function (state /*, token, idx*/) {
-  state.result += '<p>';
+rules.paragraph_open = function (/*tokens, idx, options*/) {
+  return '<p>';
 };
-rules.paragraph_close = function (state, token, idx) {
-  state.result += '</p>' + getBreak(state, idx);
+rules.paragraph_close = function (tokens, idx /*, options*/) {
+  return '</p>' + getBreak(tokens, idx);
 };
 
 
-rules.table_open = function (state /*, token, idx*/) {
-  state.result += '<table>\n';
+rules.table_open = function (/*tokens, idx, options*/) {
+  return '<table>\n';
 };
-rules.table_close = function (state /*, token, idx*/) {
-  state.result += '</table>\n';
+rules.table_close = function (/*tokens, idx, options*/) {
+  return '</table>\n';
 };
-rules.tr_open = function (state /*, token, idx*/) {
-  state.result += '<tr>\n';
+rules.tr_open = function (/*tokens, idx, options*/) {
+  return '<tr>\n';
 };
-rules.tr_close = function (state /*, token, idx*/) {
-  state.result += '</tr>\n';
+rules.tr_close = function (/*tokens, idx, options*/) {
+  return '</tr>\n';
 };
-rules.th_open = function (state, token) {
-  state.result += '<th'
+rules.th_open = function (tokens, idx /*, options*/) {
+  var token = tokens[idx];
+  return '<th'
     + (token.align ? ' align="' + token.align + '"' : '')
     + '>';
 };
-rules.th_close = function (state /*, token, idx*/) {
-  state.result += '</th>\n';
+rules.th_close = function (/*tokens, idx, options*/) {
+  return '</th>\n';
 };
-rules.td_open = function (state, token) {
-  state.result += '<td'
+rules.td_open = function (tokens, idx /*, options*/) {
+  var token = tokens[idx];
+  return '<td'
     + (token.align ? ' align="' + token.align + '"' : '')
     + '>';
 };
-rules.td_close = function (state /*, token, idx*/) {
-  state.result += '</td>\n';
+rules.td_close = function (/*tokens, idx, options*/) {
+  return '</td>\n';
+};
+
+
+rules.hardbreak = function (tokens, idx, options) {
+  return (options.xhtml ? '<br />' : '<br>') + '\n';
 };
 
 
-rules.text = function (state, token /*, idx*/) {
-  state.result += escapeHtml(unescapeMd(token.content));
+rules.text = function (tokens, idx /*, options*/) {
+  return tokens[idx].content;
 };
 
 
-rules.html = function (state, token /*, idx*/) {
-  state.result += token.content;
+rules.html = function (tokens, idx /*, options*/) {
+  return tokens[idx].content;
 };
 
 
@@ -148,63 +151,67 @@ function Renderer() {
   this.rules = assign({}, rules);
 }
 
-Renderer.prototype.render = function (state) {
+Renderer.prototype.render = function (tokens, options) {
   var i, len, rule, name, next,
-      tokens = state.tokens,
+      result = '',
       rules = this.rules,
       tightStack = [];
 
   // wrap paragraphs on top level by default
-  state.tight = false;
+  var tight = false;
 
   for (i = 0, len = tokens.length; i < len; i++) {
     name = tokens[i].type;
     rule = rules[name];
 
-    // TODO: temporary check
-    if (!rule) {
-      throw new Error('Renderer error: unknown token ' + name);
-    }
-
     // Dirty stack machine to track lists style (loose/tight)
     if (name === 'ordered_list_open' || name === 'bullet_list_open') {
-      tightStack.push(state.tight);
-      state.tight = tokens[i].tight;
+      tightStack.push(tight);
+      tight = tokens[i].tight;
     }
     if (name === 'ordered_list_close' || name === 'bullet_list_close') {
-      state.tight = tightStack.pop();
+      tight = tightStack.pop();
     }
     if (name === 'blockquote_open') {
-      tightStack.push(state.tight);
-      state.tight = false;
+      tightStack.push(tight);
+      tight = false;
     }
     if (name === 'blockquote_close') {
-      state.tight = tightStack.pop();
+      tight = tightStack.pop();
     }
 
 
     // in tight mode just ignore paragraphs for lists
     // TODO - track right nesting to blockquotes
-    if (name === 'paragraph_open' && state.tight) {
+    if (name === 'paragraph_open' && tight) {
       continue;
     }
-    if (name === 'paragraph_close' && state.tight) {
+    if (name === 'paragraph_close' && tight) {
       // Quick hack - texts should have LF if followed by blocks
-      if (i + 1 < state.tokens.length) {
-        next = state.tokens[i + 1].type;
+      if (i + 1 < tokens.length) {
+        next = tokens[i + 1].type;
         if (next === 'bullet_list_open' ||
             next === 'ordered_list_open' ||
             next === 'blockquote_open') {
-          state.result += '\n';
+          result += '\n';
         }
       }
 
       continue;
     }
-    rule(state, tokens[i], i);
+
+    if (tokens[i].type === 'inline') {
+      result += this.render(tokens[i].children, options);
+    } else {
+      // TODO: temporary check
+      if (!rule) {
+        throw new Error('Renderer error: unknown token ' + name);
+      }
+      result += rule(tokens, i, options);
+    }
   }
 
-  return state.result;
+  return result;
 };
 
 module.exports = Renderer;

lib/state_inline.js

@@ -0,0 +1,33 @@
+// Inline parser state
+
+'use strict';
+
+
+function StateInline(src, lexer, options) {
+  this.src = src;
+  this.options = options;
+  this.lexer = lexer;
+  this.tokens = [];
+  this.pos = 0;
+  this.pending = '';
+  this.posMax = this.src.length;
+}
+
+
+StateInline.prototype.pushText = function () {
+  this.tokens.push({
+    type: 'text',
+    content: this.pending.trim()
+  });
+  this.pending = '';
+};
+
+StateInline.prototype.push = function (token) {
+  if (this.pending) {
+    this.pushText();
+  }
+
+  this.tokens.push(token);
+};
+
+module.exports = StateInline;

test/fixtures/stmd/bad.txt

@@ -174,8 +174,8 @@ src line: 1660
 error:
 
 <p>[foo]:
-/url  
-'the title'  </p>
+/url<br />
+'the title'</p>
 <p>[foo]</p>
 
 
@@ -392,23 +392,6 @@ error:
 </blockquote>
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 1952
-
-.
-aaa     
-bbb     
-.
-<p>aaa<br />
-bbb</p>
-.
-
-error:
-
-<p>aaa     
-bbb     </p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 2414
 
@@ -434,13 +417,11 @@ with two lines.</p>
 error:
 
 <ol>
-<li><p>A paragraph
-with two lines.</p>
-<pre><code>indented code
+<li>A paragraph
+with two lines.<pre><code>indented code
 </code></pre>
 <blockquote>
-<pre><code>&gt; A block quote.
-</code></pre>
+<p>A block quote.</p>
 </blockquote></li>
 </ol>
 
@@ -473,13 +454,11 @@ src line: 2574
 error:
 
 <ol>
-<li><p>foo</p>
-<pre><code>bar
+<li>foo<pre><code>bar
 </code></pre>
-<p>baz</p>
+baz
 <blockquote>
-<pre><code>&gt; bam
-</code></pre>
+<p>bam</p>
 </blockquote></li>
 </ol>
 
@@ -509,13 +488,11 @@ with two lines.</p>
 error:
 
 <ol>
-<li><p>A paragraph
-with two lines.</p>
-<pre><code>indented code
+<li>A paragraph
+with two lines.<pre><code>indented code
 </code></pre>
 <blockquote>
-<pre><code> &gt; A block quote.
-</code></pre>
+<p>A block quote.</p>
 </blockquote></li>
 </ol>
 
@@ -545,13 +522,11 @@ with two lines.</p>
 error:
 
 <ol>
-<li><p>A paragraph
-with two lines.</p>
-<pre><code>indented code
+<li>A paragraph
+with two lines.<pre><code>indented code
 </code></pre>
 <blockquote>
-<pre><code>  &gt; A block quote.
-</code></pre>
+<p>A block quote.</p>
 </blockquote></li>
 </ol>
 
@@ -581,13 +556,11 @@ with two lines.</p>
 error:
 
 <ol>
-<li><p>A paragraph
-with two lines.</p>
-<pre><code>indented code
+<li>A paragraph
+with two lines.<pre><code>indented code
 </code></pre>
 <blockquote>
-<pre><code>   &gt; A block quote.
-</code></pre>
+<p>A block quote.</p>
 </blockquote></li>
 </ol>
 
@@ -617,51 +590,15 @@ with two lines.</p>
 error:
 
 <ol>
-<li><p>A paragraph
-with two lines.</p>
-<pre><code>indented code
+<li>A paragraph
+with two lines.<pre><code>indented code
 </code></pre>
 <blockquote>
-<pre><code>  &gt; A block quote.
-</code></pre>
+<p>A block quote.</p>
 </blockquote></li>
 </ol>
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 2866
-
-.
-> 1. > Blockquote
-continued here.
-.
-<blockquote>
-<ol>
-<li><blockquote>
-<p>Blockquote
-continued here.</p>
-</blockquote></li>
-</ol>
-</blockquote>
-.
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 2880
-
-.
-> 1. > Blockquote
-> continued here.
-.
-<blockquote>
-<ol>
-<li><blockquote>
-<p>Blockquote
-continued here.</p>
-</blockquote></li>
-</ol>
-</blockquote>
-.
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3292
 
@@ -727,38 +664,6 @@ error:
 </ul>
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3469
-
-.
-- a
-  - b
-
-    c
-- d
-.
-<ul>
-<li>a
-<ul>
-<li><p>b</p>
-<p>c</p></li>
-</ul></li>
-<li>d</li>
-</ul>
-.
-
-error:
-
-<ul>
-<li>a
-<ul>
-<li><p>b</p>
-<p>c</p></li>
-<li><p>d</p></li>
-</ul></li>
-</ul>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3550
 
@@ -788,48 +693,6 @@ baz</li>
 </ul>
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3565
-
-.
-- a
-  - b
-  - c
-
-- d
-  - e
-  - f
-.
-<ul>
-<li><p>a</p>
-<ul>
-<li>b</li>
-<li>c</li>
-</ul></li>
-<li><p>d</p>
-<ul>
-<li>e</li>
-<li>f</li>
-</ul></li>
-</ul>
-.
-
-error:
-
-<ul>
-<li>a
-<ul>
-<li><p>b</p></li>
-<li><p>c</p></li>
-<li><p>d</p>
-<ul>
-<li>e</li>
-<li>f</li>
-</ul></li>
-</ul></li>
-</ul>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3594
 
@@ -858,23 +721,6 @@ error:
 <p>\*emphasis*</p>
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3655
-
-.
-foo\
-bar
-.
-<p>foo<br />
-bar</p>
-.
-
-error:
-
-<p>foo\
-bar</p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3666
 
@@ -966,62 +812,6 @@ error:
 </code></pre>
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3736
-
-.
-&nbsp; &amp; &copy; &AElig; &Dcaron; &frac34; &HilbertSpace; &DifferentialD; &ClockwiseContourIntegral;
-.
-<p>&nbsp; &amp; &copy; &AElig; &Dcaron; &frac34; &HilbertSpace; &DifferentialD; &ClockwiseContourIntegral;</p>
-.
-
-error:
-
-<p>&amp;nbsp; &amp;amp; &amp;copy; &amp;AElig; &amp;Dcaron; &amp;frac34; &amp;HilbertSpace; &amp;DifferentialD; &amp;ClockwiseContourIntegral;</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3745
-
-.
-&#1; &#35; &#1234; &#992; &#98765432;
-.
-<p>&#1; &#35; &#1234; &#992; &#98765432;</p>
-.
-
-error:
-
-<p>&amp;#1; &amp;#35; &amp;#1234; &amp;#992; &amp;#98765432;</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3755
-
-.
-&#x1; &#X22; &#XD06; &#xcab;
-.
-<p>&#x1; &#X22; &#XD06; &#xcab;</p>
-.
-
-error:
-
-<p>&amp;#x1; &amp;#X22; &amp;#XD06; &amp;#xcab;</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3781
-
-.
-&MadeUpEntity;
-.
-<p>&MadeUpEntity;</p>
-.
-
-error:
-
-<p>&amp;MadeUpEntity;</p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3791
 
@@ -1047,7 +837,7 @@ src line: 3797
 
 error:
 
-<p>[foo](/f&amp;ouml;&amp;ouml; &quot;f&amp;ouml;&amp;ouml;&quot;)</p>
+<p>[foo](/f&ouml;&ouml; &quot;f&ouml;&ouml;&quot;)</p>
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1064,7 +854,7 @@ src line: 3803
 error:
 
 <p>[foo]</p>
-<p>[foo]: /f&amp;ouml;&amp;ouml; &quot;f&amp;ouml;&amp;ouml;&quot;</p>
+<p>[foo]: /f&ouml;&ouml; &quot;f&ouml;&ouml;&quot;</p>
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1096,7 +886,7 @@ src line: 3822
 
 error:
 
-<p>`f&amp;ouml;&amp;ouml;`</p>
+<p>`f&ouml;&ouml;`</p>
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2270,7 +2060,7 @@ src line: 4811
 
 error:
 
-<p>[link](foo%20b&amp;auml;)</p>
+<p>[link](foo%20b&auml;)</p>
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2318,7 +2108,7 @@ src line: 4841
 
 error:
 
-<p>[link](/url &quot;title &quot;&amp;quot;&quot;)</p>
+<p>[link](/url &quot;title &quot;&quot;&quot;)</p>
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3418,91 +3208,6 @@ error:
 <a href="\*">
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 5789
-
-.
-foo  
-baz
-.
-<p>foo<br />
-baz</p>
-.
-
-error:
-
-<p>foo  
-baz</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 5800
-
-.
-foo\
-baz
-.
-<p>foo<br />
-baz</p>
-.
-
-error:
-
-<p>foo\
-baz</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 5810
-
-.
-foo       
-baz
-.
-<p>foo<br />
-baz</p>
-.
-
-error:
-
-<p>foo       
-baz</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 5820
-
-.
-foo  
-     bar
-.
-<p>foo<br />
-bar</p>
-.
-
-error:
-
-<p>foo  
-bar</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 5828
-
-.
-foo\
-     bar
-.
-<p>foo<br />
-bar</p>
-.
-
-error:
-
-<p>foo\
-bar</p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 5839
 
@@ -3516,7 +3221,7 @@ bar</em></p>
 
 error:
 
-<p>*foo  
+<p>*foo<br />
 bar*</p>
 
 
@@ -3533,7 +3238,7 @@ bar</em></p>
 
 error:
 
-<p>*foo\
+<p>*foo<br />
 bar*</p>
 
 
@@ -3549,7 +3254,7 @@ span`
 
 error:
 
-<p>`code  
+<p>`code<br />
 span`</p>
 
 
@@ -3565,7 +3270,7 @@ span`
 
 error:
 
-<p>`code\
+<p>`code<br />
 span`</p>
 
 
@@ -3603,20 +3308,3 @@ error:
 bar">
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 5908
-
-.
-foo 
- baz
-.
-<p>foo
-baz</p>
-.
-
-error:
-
-<p>foo 
-baz</p>
-
-

test/fixtures/stmd/good.txt

@@ -1385,6 +1385,17 @@ bbb
 <p>bbb</p>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 1952
+
+.
+aaa     
+bbb     
+.
+<p>aaa<br />
+bbb</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 1968
 
@@ -2059,6 +2070,40 @@ with two lines.</li>
 </ol>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 2866
+
+.
+> 1. > Blockquote
+continued here.
+.
+<blockquote>
+<ol>
+<li><blockquote>
+<p>Blockquote
+continued here.</p>
+</blockquote></li>
+</ol>
+</blockquote>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 2880
+
+.
+> 1. > Blockquote
+> continued here.
+.
+<blockquote>
+<ol>
+<li><blockquote>
+<p>Blockquote
+continued here.</p>
+</blockquote></li>
+</ol>
+</blockquote>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 2904
 
@@ -2388,6 +2433,26 @@ src line: 3446
 </ul>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3469
+
+.
+- a
+  - b
+
+    c
+- d
+.
+<ul>
+<li>a
+<ul>
+<li><p>b</p>
+<p>c</p></li>
+</ul></li>
+<li>d</li>
+</ul>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3489
 
@@ -2454,6 +2519,32 @@ src line: 3536
 </ul>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3565
+
+.
+- a
+  - b
+  - c
+
+- d
+  - e
+  - f
+.
+<ul>
+<li><p>a</p>
+<ul>
+<li>b</li>
+<li>c</li>
+</ul></li>
+<li><p>d</p>
+<ul>
+<li>e</li>
+<li>f</li>
+</ul></li>
+</ul>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3607
 
@@ -2495,6 +2586,17 @@ src line: 3625
 [foo]: /url &quot;not a reference&quot;</p>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3655
+
+.
+foo\
+bar
+.
+<p>foo<br />
+bar</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3672
 
@@ -2517,6 +2619,33 @@ src line: 3679
 </code></pre>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3736
+
+.
+&nbsp; &amp; &copy; &AElig; &Dcaron; &frac34; &HilbertSpace; &DifferentialD; &ClockwiseContourIntegral;
+.
+<p>&nbsp; &amp; &copy; &AElig; &Dcaron; &frac34; &HilbertSpace; &DifferentialD; &ClockwiseContourIntegral;</p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3745
+
+.
+&#1; &#35; &#1234; &#992; &#98765432;
+.
+<p>&#1; &#35; &#1234; &#992; &#98765432;</p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3755
+
+.
+&#x1; &#X22; &#XD06; &#xcab;
+.
+<p>&#x1; &#X22; &#XD06; &#xcab;</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3763
 
@@ -2535,6 +2664,15 @@ src line: 3772
 <p>&amp;copy</p>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3781
+
+.
+&MadeUpEntity;
+.
+<p>&MadeUpEntity;</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3828
 
@@ -2945,6 +3083,61 @@ src line: 5777
 <p>&lt;a href=&quot;&quot;&quot;&gt;</p>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 5789
+
+.
+foo  
+baz
+.
+<p>foo<br />
+baz</p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 5800
+
+.
+foo\
+baz
+.
+<p>foo<br />
+baz</p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 5810
+
+.
+foo       
+baz
+.
+<p>foo<br />
+baz</p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 5820
+
+.
+foo  
+     bar
+.
+<p>foo<br />
+bar</p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 5828
+
+.
+foo\
+     bar
+.
+<p>foo<br />
+bar</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 5897
 
@@ -2956,6 +3149,17 @@ baz
 baz</p>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 5908
+
+.
+foo 
+ baz
+.
+<p>foo
+baz</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 5927