Reworked whitetspaces cleanup in inlines

lib/lexer_inline.js

@@ -152,9 +152,21 @@ LexerInline.prototype.tokenize = function (state) {
 // Parse input string.
 //
 LexerInline.prototype.parse = function (str, options) {
-  var state = new StateInline(str, this, options);
+  var state = new StateInline(str, this, options),
+      tokens = state.tokens,
+      len, tok;
 
   this.tokenize(state);
+
+  // If last token is text - strip tailing spaces
+  len = tokens.length;
+  if (len) {
+    tok = tokens[len - 1];
+    if (tok.type === 'text' && tok.content.charCodeAt(tok.content.length - 1) === 0x20) {
+      tok.content = tok.content.replace(/ +$/, '');
+    }
+  }
+
   return state.tokens;
 };
 

lib/lexer_inline/escape.js

@@ -35,7 +35,12 @@ module.exports = function escape(state) {
       state.push({
         type: 'hardbreak'
       });
-      state.pos += 2;
+
+      pos++;
+      // skip leading whitespaces from next line
+      while (pos < max && state.src.charCodeAt(pos) === 0x20) { pos++; }
+
+      state.pos = pos;
       return true;
     }
   }

lib/lexer_inline/newline.js

@@ -1,32 +1,40 @@
 // Proceess '\n'
 
 module.exports = function escape(state) {
-  var pmax, pos = state.pos;
+  var pmax, max, pos = state.pos;
 
   if (state.src.charCodeAt(pos) !== 0x0A/* \n */) { return false; }
 
   pmax = state.pending.length - 1;
+  max = state.posMax;
 
   // '  \n' -> hardbreak
   if (pmax >= 1 &&
       state.pending.charCodeAt(pmax) === 0x20 &&
       state.pending.charCodeAt(pmax - 1) === 0x20) {
-    state.pending = state.pending.slice(0, -2);
+    state.pending = state.pending.replace(/ +$/, '');
     state.push({
       type: 'hardbreak'
     });
-    state.pos++;
+
+    pos++;
+    // skip spaces
+    while (pos < max && state.src.charCodeAt(pos) === 0x20) { pos++; }
+
+    state.pos = pos;
     return true;
   }
 
-  state.pending = state.pending.trim() + '\n';
+  if (pmax > 0 && state.pending.charCodeAt(pmax) === 0x20) {
+    state.pending = state.pending.slice(0, -1);
+  }
+
+  state.pending += '\n';
 
   pos++;
-
-  // skip spaces to simplify trim
-  while (state.src.charCodeAt(pos) === 0x20) { pos++; }
+  // skip spaces
+  while (pos < max && state.src.charCodeAt(pos) === 0x20) { pos++; }
 
   state.pos = pos;
-
   return true;
 };

lib/state_inline.js

@@ -15,9 +15,15 @@ function StateInline(src, lexer, options) {
 
 
 StateInline.prototype.pushText = function () {
+  var pending = this.pending;
+  // strip leading spaces from the first token.
+  // others will be stripped by logic in `newline` rule
+  if (this.tokens.length === 0 && pending.charCodeAt(0) === 0x20) {
+    pending = pending.replace(/^ +/, '');
+  }
   this.tokens.push({
     type: 'text',
-    content: this.pending.trim()
+    content: pending
   });
   this.pending = '';
 };

test/fixtures/stmd/bad.txt

@@ -46,38 +46,6 @@ error:
 <h2>Foo *bar*</h2>
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 1237
-
-.
-``` ```
-aaa
-.
-<p><code></code>
-aaa</p>
-.
-
-error:
-
-<p><code></code>aaa</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 1328
-
-.
-``` aa ```
-foo
-.
-<p><code>aa</code>
-foo</p>
-.
-
-error:
-
-<p><code>aa</code>foo</p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 1406
 
@@ -831,20 +799,6 @@ error:
 </code></pre>
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3938
-
-.
-[not a `link](/foo`)
-.
-<p>[not a <code>link](/foo</code>)</p>
-.
-
-error:
-
-<p>[not a<code>link](/foo</code>)</p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3954
 
@@ -1106,7 +1060,7 @@ src line: 4208
 
 error:
 
-<p>*a<code>*</code>*</p>
+<p>*a <code>*</code>*</p>
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1120,7 +1074,7 @@ _a `_`_
 
 error:
 
-<p>_a<code>_</code>_</p>
+<p>_a <code>_</code>_</p>
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

test/fixtures/stmd/good.txt

@@ -942,6 +942,17 @@ aaa
 </code></pre>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 1237
+
+.
+``` ```
+aaa
+.
+<p><code></code>
+aaa</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 1245
 
@@ -1030,6 +1041,17 @@ src line: 1319
 <pre><code class="language-;"></code></pre>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 1328
+
+.
+``` aa ```
+foo
+.
+<p><code>aa</code>
+foo</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 1338
 
@@ -2794,6 +2816,15 @@ src line: 3930
 <p>*foo<code>*</code></p>
 .
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3938
+
+.
+[not a `link](/foo`)
+.
+<p>[not a <code>link](/foo</code>)</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3946