|
@ -89,6 +89,7 @@ ParserBlock.prototype.tokenize = function (state, startLine, endLine) { |
|
|
var TABS_SCAN_RE = /[\n\t]/g; |
|
|
var TABS_SCAN_RE = /[\n\t]/g; |
|
|
var NEWLINES_RE = /\r[\n\u0085]|[\u2424\u2028\u0085]/g; |
|
|
var NEWLINES_RE = /\r[\n\u0085]|[\u2424\u2028\u0085]/g; |
|
|
var SPACES_RE = /\u00a0/g; |
|
|
var SPACES_RE = /\u00a0/g; |
|
|
|
|
|
var NULL_RE = /\u0000/g; |
|
|
|
|
|
|
|
|
ParserBlock.prototype.parse = function (src, options, env, outTokens) { |
|
|
ParserBlock.prototype.parse = function (src, options, env, outTokens) { |
|
|
var state, lineStart = 0, lastTabPos = 0; |
|
|
var state, lineStart = 0, lastTabPos = 0; |
|
@ -101,6 +102,9 @@ ParserBlock.prototype.parse = function (src, options, env, outTokens) { |
|
|
// Normalize newlines
|
|
|
// Normalize newlines
|
|
|
src = src.replace(NEWLINES_RE, '\n'); |
|
|
src = src.replace(NEWLINES_RE, '\n'); |
|
|
|
|
|
|
|
|
|
|
|
// Strin NULL characters
|
|
|
|
|
|
src = src.replace(NULL_RE, ''); |
|
|
|
|
|
|
|
|
// Replace tabs with proper number of spaces (1..4)
|
|
|
// Replace tabs with proper number of spaces (1..4)
|
|
|
if (src.indexOf('\t') >= 0) { |
|
|
if (src.indexOf('\t') >= 0) { |
|
|
src = src.replace(TABS_SCAN_RE, function (match, offset) { |
|
|
src = src.replace(TABS_SCAN_RE, function (match, offset) { |
|
|