Moved input normalization to core chain

lib/parser_block.js

@@ -100,9 +100,6 @@ ParserBlock.prototype.tokenize = function (state, startLine, endLine) {
   }
 };
 
-var TABS_SCAN_RE = /[\n\t]/g;
-var NEWLINES_RE  = /\r[\n\u0085]|[\u2424\u2028\u0085]/g;
-var NULL_RE      = /\u0000/g;
 
 /**
  * ParserBlock.parse(str, md, env, outTokens)
@@ -110,31 +107,10 @@ var NULL_RE      = /\u0000/g;
  * Process input string and push block tokens into `outTokens`
  **/
 ParserBlock.prototype.parse = function (src, md, env, outTokens) {
-  var state, lineStart = 0, lastTabPos = 0;
+  var state;
 
   if (!src) { return []; }
 
-  // Normalize newlines
-  src = src.replace(NEWLINES_RE, '\n');
-
-  // Strin NULL characters
-  src = src.replace(NULL_RE, '\uFFFD');
-
-  // Replace tabs with proper number of spaces (1..4)
-  if (src.indexOf('\t') >= 0) {
-    src = src.replace(TABS_SCAN_RE, function (match, offset) {
-      var result;
-      if (src.charCodeAt(offset) === 0x0A) {
-        lineStart = offset + 1;
-        lastTabPos = 0;
-        return match;
-      }
-      result = '    '.slice((offset - lineStart - lastTabPos) % 4);
-      lastTabPos = offset - lineStart + 1;
-      return result;
-    });
-  }
-
   state = new this.State(src, md, env, outTokens);
 
   this.tokenize(state, state.line, state.lineMax);

lib/parser_core.js

@@ -11,6 +11,7 @@ var Ruler  = require('./ruler');
 
 
 var _rules = [
+  [ 'normalize',      require('./rules_core/normalize')      ],
   [ 'block',          require('./rules_core/block')          ],
   [ 'inline',         require('./rules_core/inline')         ],
   [ 'replacements',   require('./rules_core/replacements')   ],

lib/presets/commonmark.js

@@ -32,6 +32,7 @@ module.exports = {
 
     core: {
       rules: [
+        'normalize',
         'block',
         'inline'
       ]

lib/presets/zero.js

@@ -33,6 +33,7 @@ module.exports = {
 
     core: {
       rules: [
+        'normalize',
         'block',
         'inline'
       ]

lib/rules_core/block.js

@@ -5,7 +5,7 @@ module.exports = function block(state) {
   if (state.inlineMode) {
     state.tokens.push({
       type: 'inline',
-      content: state.src.replace(/\n/g, ' ').trim(),
+      content: state.src,
       level: 0,
       lines: [ 0, 1 ],
       children: []

lib/rules_core/normalize.js

@@ -0,0 +1,41 @@
+// Normalize input string
+
+'use strict';
+
+
+var TABS_SCAN_RE = /[\n\t]/g;
+var NEWLINES_RE  = /\r[\n\u0085]|[\u2424\u2028\u0085]/g;
+var NULL_RE      = /\u0000/g;
+
+
+module.exports = function inline(state) {
+  var str, lineStart, lastTabPos;
+
+  if (!state.normalizeInput) { return; }
+
+  // Normalize newlines
+  str = state.src.replace(NEWLINES_RE, '\n');
+
+  // Strin NULL characters
+  str = str.replace(NULL_RE, '\uFFFD');
+
+  // Replace tabs with proper number of spaces (1..4)
+  if (str.indexOf('\t') >= 0) {
+    lineStart = 0;
+    lastTabPos = 0;
+
+    str = str.replace(TABS_SCAN_RE, function (match, offset) {
+      var result;
+      if (str.charCodeAt(offset) === 0x0A) {
+        lineStart = offset + 1;
+        lastTabPos = 0;
+        return match;
+      }
+      result = '    '.slice((offset - lineStart - lastTabPos) % 4);
+      lastTabPos = offset - lineStart + 1;
+      return result;
+    });
+  }
+
+  state.src = str;
+};

lib/rules_core/state_core.js

@@ -7,5 +7,6 @@ module.exports = function StateCore(src, md, env) {
   this.env = env;
   this.tokens = [];
   this.inlineMode = false;
+  this.normalizeInput = true;
   this.md = md; // link to parser instance
 };