Rename "lexer" to "parser"

and move rules from lexer_* to rules_*

lib/helpers.js

@@ -1,4 +1,4 @@
-// Common functions for lexers
+// Common functions for parsers
 
 'use strict';
 

lib/parser.js

@@ -6,10 +6,10 @@
 var assign = require('object-assign');
 
 
-var Renderer    = require('./renderer');
+var Renderer     = require('./renderer');
-var LexerBlock  = require('./lexer_block');
+var ParserBlock  = require('./parser_block');
-var LexerInline = require('./lexer_inline');
+var ParserInline = require('./parser_inline');
-var defaults    = require('./defaults');
+var defaults     = require('./defaults');
 
 // Main class
 //
@@ -17,8 +17,8 @@ function Parser(options) {
   this.options  = assign({}, defaults);
   this.state    = null;
 
-  this.inline   = new LexerInline();
+  this.inline   = new ParserInline();
-  this.block    = new LexerBlock();
+  this.block    = new ParserBlock();
   this.renderer = new Renderer();
 
   // a bunch of cross-references between parsers

lib/lexer_block.js → lib/parser_block.js

@@ -1,10 +1,10 @@
-// Block lexer
+// Block parser
 
 
 'use strict';
 
 
-var State           = require('./lexer_block/state_block');
+var State           = require('./rules_block/state_block');
 var skipEmptyLines  = require('./helpers').skipEmptyLines;
 var isEmpty         = require('./helpers').isEmpty;
 
@@ -12,16 +12,16 @@ var isEmpty         = require('./helpers').isEmpty;
 var rules = [];
 
 // `list` should be after `hr`, but before `heading`
-rules.push(require('./lexer_block/code'));
+rules.push(require('./rules_block/code'));
-rules.push(require('./lexer_block/fences'));
+rules.push(require('./rules_block/fences'));
-rules.push(require('./lexer_block/blockquote'));
+rules.push(require('./rules_block/blockquote'));
-rules.push(require('./lexer_block/hr'));
+rules.push(require('./rules_block/hr'));
-rules.push(require('./lexer_block/list'));
+rules.push(require('./rules_block/list'));
-rules.push(require('./lexer_block/heading'));
+rules.push(require('./rules_block/heading'));
-rules.push(require('./lexer_block/lheading'));
+rules.push(require('./rules_block/lheading'));
-rules.push(require('./lexer_block/htmlblock'));
+rules.push(require('./rules_block/htmlblock'));
-rules.push(require('./lexer_block/table'));
+rules.push(require('./rules_block/table'));
-rules.push(require('./lexer_block/paragraph'));
+rules.push(require('./rules_block/paragraph'));
 
 
 function functionName(fn) {
@@ -41,9 +41,9 @@ function findByName(self, name) {
 }
 
 
-// Block Lexer class
+// Block Parser class
 //
-function LexerBlock() {
+function ParserBlock() {
   this.rules = [];
   this.rules_named = {};
 
@@ -53,12 +53,12 @@ function LexerBlock() {
 }
 
 
-// Replace/delete lexer function
+// Replace/delete parser function
 //
-LexerBlock.prototype.at = function (name, fn) {
+ParserBlock.prototype.at = function (name, fn) {
   var index = findByName(name);
   if (index === -1) {
-    throw new Error('Lexer rule not found: ' + name);
+    throw new Error('Parser rule not found: ' + name);
   }
 
   if (fn) {
@@ -71,10 +71,10 @@ LexerBlock.prototype.at = function (name, fn) {
 };
 
 
-// Add function to lexer chain before one with given name.
+// Add function to parser chain before one with given name.
 // Or add to start, if name not defined
 //
-LexerBlock.prototype.before = function (name, fn) {
+ParserBlock.prototype.before = function (name, fn) {
   if (!name) {
     this.rules.unshift(fn);
     this.rules_named[functionName(fn)] = fn;
@@ -83,7 +83,7 @@ LexerBlock.prototype.before = function (name, fn) {
 
   var index = findByName(name);
   if (index === -1) {
-    throw new Error('Lexer rule not found: ' + name);
+    throw new Error('Parser rule not found: ' + name);
   }
 
   this.rules.splice(index, 0, fn);
@@ -91,10 +91,10 @@ LexerBlock.prototype.before = function (name, fn) {
 };
 
 
-// Add function to lexer chain after one with given name.
+// Add function to parser chain after one with given name.
 // Or add to end, if name not defined
 //
-LexerBlock.prototype.after = function (name, fn) {
+ParserBlock.prototype.after = function (name, fn) {
   if (!name) {
     this.rules.push(fn);
     this.rules_named[functionName(fn)] = fn;
@@ -103,7 +103,7 @@ LexerBlock.prototype.after = function (name, fn) {
 
   var index = findByName(name);
   if (index === -1) {
-    throw new Error('Lexer rule not found: ' + name);
+    throw new Error('Parser rule not found: ' + name);
   }
 
   this.rules.splice(index + 1, 0, fn);
@@ -113,7 +113,7 @@ LexerBlock.prototype.after = function (name, fn) {
 
 // Generate tokens for input range
 //
-LexerBlock.prototype.tokenize = function (state, startLine, endLine) {
+ParserBlock.prototype.tokenize = function (state, startLine, endLine) {
   var ok, i,
       rules = this.rules,
       len = this.rules.length,
@@ -168,7 +168,7 @@ LexerBlock.prototype.tokenize = function (state, startLine, endLine) {
 };
 
 
-LexerBlock.prototype.parse = function (src, options, env) {
+ParserBlock.prototype.parse = function (src, options, env) {
   var state, lineStart = 0, lastTabPos = 0;
 
   if (!src) { return ''; }
@@ -217,4 +217,4 @@ LexerBlock.prototype.parse = function (src, options, env) {
 };
 
 
-module.exports = LexerBlock;
+module.exports = ParserBlock;

lib/lexer_inline.js → lib/parser_inline.js

@@ -1,31 +1,31 @@
-// Inline lexer
+// Inline parser
 
 'use strict';
 
 
-var StateInline = require('./lexer_inline/state_inline');
+var StateInline = require('./rules_inline/state_inline');
 
 ////////////////////////////////////////////////////////////////////////////////
-// Lexer rules
+// Parser rules
 
 var rules = [];
 
 
 // Pure text
-rules.push(require('./lexer_inline/text'));
+rules.push(require('./rules_inline/text'));
-rules.push(require('./lexer_inline/newline'));
+rules.push(require('./rules_inline/newline'));
-rules.push(require('./lexer_inline/escape'));
+rules.push(require('./rules_inline/escape'));
-rules.push(require('./lexer_inline/backticks'));
+rules.push(require('./rules_inline/backticks'));
-rules.push(require('./lexer_inline/emphasis'));
+rules.push(require('./rules_inline/emphasis'));
-rules.push(require('./lexer_inline/links'));
+rules.push(require('./rules_inline/links'));
-rules.push(require('./lexer_inline/autolink'));
+rules.push(require('./rules_inline/autolink'));
-rules.push(require('./lexer_inline/htmltag'));
+rules.push(require('./rules_inline/htmltag'));
-rules.push(require('./lexer_inline/entity'));
+rules.push(require('./rules_inline/entity'));
-rules.push(require('./lexer_inline/escape_html_char'));
+rules.push(require('./rules_inline/escape_html_char'));
 
 
 ////////////////////////////////////////////////////////////////////////////////
-// Lexer class
+// Parser class
 
 
 function functionName(fn) {
@@ -45,9 +45,9 @@ function findByName(self, name) {
 }
 
 
-// Block Lexer class
+// Block Parser class
 //
-function LexerInline() {
+function ParserInline() {
   this.rules = [];
 
   // Rule to skip pure text
@@ -61,12 +61,12 @@ function LexerInline() {
 }
 
 
-// Replace/delete lexer function
+// Replace/delete parser function
 //
-LexerInline.prototype.at = function (name, fn) {
+ParserInline.prototype.at = function (name, fn) {
   var index = findByName(name);
   if (index === -1) {
-    throw new Error('Lexer rule not found: ' + name);
+    throw new Error('Parser rule not found: ' + name);
   }
 
   if (fn) {
@@ -77,10 +77,10 @@ LexerInline.prototype.at = function (name, fn) {
 };
 
 
-// Add function to lexer chain before one with given name.
+// Add function to parser chain before one with given name.
 // Or add to start, if name not defined
 //
-LexerInline.prototype.before = function (name, fn) {
+ParserInline.prototype.before = function (name, fn) {
   if (!name) {
     this.rules.unshift(fn);
     return;
@@ -88,17 +88,17 @@ LexerInline.prototype.before = function (name, fn) {
 
   var index = findByName(name);
   if (index === -1) {
-    throw new Error('Lexer rule not found: ' + name);
+    throw new Error('Parser rule not found: ' + name);
   }
 
   this.rules.splice(index, 0, fn);
 };
 
 
-// Add function to lexer chain after one with given name.
+// Add function to parser chain after one with given name.
 // Or add to end, if name not defined
 //
-LexerInline.prototype.after = function (name, fn) {
+ParserInline.prototype.after = function (name, fn) {
   if (!name) {
     this.rules.push(fn);
     return;
@@ -106,7 +106,7 @@ LexerInline.prototype.after = function (name, fn) {
 
   var index = findByName(name);
   if (index === -1) {
-    throw new Error('Lexer rule not found: ' + name);
+    throw new Error('Parser rule not found: ' + name);
   }
 
   this.rules.splice(index + 1, 0, fn);
@@ -116,7 +116,7 @@ LexerInline.prototype.after = function (name, fn) {
 // Generate single token;
 // returns `true` if any rule reported success
 //
-LexerInline.prototype.tokenizeSingle = function (state) {
+ParserInline.prototype.tokenizeSingle = function (state) {
   var ok, i,
       rules = this.rules,
       len = this.rules.length;
@@ -132,7 +132,7 @@ LexerInline.prototype.tokenizeSingle = function (state) {
 
 // Generate tokens for input range
 //
-LexerInline.prototype.tokenize = function (state) {
+ParserInline.prototype.tokenize = function (state) {
   var ok, i,
       rules = this.rules,
       len = this.rules.length,
@@ -169,7 +169,7 @@ LexerInline.prototype.tokenize = function (state) {
 
 // Parse input string.
 //
-LexerInline.prototype.parse = function (str, options, env) {
+ParserInline.prototype.parse = function (str, options, env) {
   var state = new StateInline(str, this, options, env);
 
   this.tokenize(state);
@@ -178,4 +178,4 @@ LexerInline.prototype.parse = function (str, options, env) {
 };
 
 
-module.exports = LexerInline;
+module.exports = ParserInline;

lib/parse_ref.js → lib/parser_ref.js

@@ -2,14 +2,14 @@
 'use strict';
 
 
-var StateInline = require('./lexer_inline/state_inline');
+var StateInline = require('./rules_inline/state_inline');
-var links = require('./lexer_inline/links');
+var links       = require('./rules_inline/links');
-var skipSpaces = require('./helpers').skipSpaces;
+var skipSpaces  = require('./helpers').skipSpaces;
 
 
 // Parse link reference definition.
 //
-module.exports = function parse_reference(str, lexer, options, env) {
+module.exports = function parse_reference(str, parser, options, env) {
   var state, labelEnd, pos, max, code, start, href, title, label;
 
   if (str.charCodeAt(0) !== 0x5B/* [ */) { return -1; }
@@ -17,7 +17,7 @@ module.exports = function parse_reference(str, lexer, options, env) {
   // TODO: benchmark this
   if (str.indexOf(']:') === -1) { return -1; }
 
-  state = new StateInline(str, lexer, options, env);
+  state = new StateInline(str, parser, options, env);
   labelEnd = links.parseLinkLabel(state, 0);
 
   if (labelEnd < 0 || str.charCodeAt(labelEnd + 1) !== 0x3A/* : */) { return -1; }

lib/lexer_block/blockquote.js → lib/rules_block/blockquote.js

@@ -8,7 +8,7 @@ var skipSpaces      = require('../helpers').skipSpaces;
 
 module.exports = function blockquote(state, startLine, endLine, silent) {
   var nextLine, lastLineEmpty, oldTShift, oldBMarks, i, oldIndent, oldListMode,
-      rules_named = state.lexer.rules_named,
+      rules_named = state.parser.rules_named,
       pos = state.bMarks[startLine] + state.tShift[startLine],
       max = state.eMarks[startLine];
 
@@ -106,7 +106,7 @@ module.exports = function blockquote(state, startLine, endLine, silent) {
   oldListMode = state.listMode;
   state.listMode = false;
   state.tokens.push({ type: 'blockquote_open', level: state.level++ });
-  state.lexer.tokenize(state, startLine, nextLine);
+  state.parser.tokenize(state, startLine, nextLine);
   state.tokens.push({ type: 'blockquote_close', level: --state.level });
   state.listMode = oldListMode;
 

lib/lexer_block/list.js → lib/rules_block/list.js

@@ -90,7 +90,7 @@ module.exports = function list(state, startLine, endLine, silent) {
       contentStart,
       listTokIdx,
       prevEmptyEnd,
-      rules_named = state.lexer.rules_named;
+      rules_named = state.parser.rules_named;
 
   // Detect list type and position after marker
   if ((posAfterMarker = skipOrderedListMarker(state, startLine)) >= 0) {
@@ -158,7 +158,7 @@ module.exports = function list(state, startLine, endLine, silent) {
     //  ^^^^^ - calculating total length of this thing
     indent = (posAfterMarker - state.bMarks[nextLine]) + indentAfterMarker;
 
-    // Run sublexer & write tokens
+    // Run subparser & write tokens
     state.tokens.push({ type: 'list_item_open', level: state.level++ });
 
     //nextLine++;
@@ -172,7 +172,7 @@ module.exports = function list(state, startLine, endLine, silent) {
     state.tight = true;
     state.listMode = true;
 
-    state.lexer.tokenize(state, startLine, endLine, true);
+    state.parser.tokenize(state, startLine, endLine, true);
 
     // If any of list item is tight, mark list as tight
     if (!state.tight || prevEmptyEnd) {

lib/lexer_block/paragraph.js → lib/rules_block/paragraph.js

@@ -3,15 +3,15 @@
 'use strict';
 
 
-var isEmpty = require('../helpers').isEmpty;
+var isEmpty  = require('../helpers').isEmpty;
 var getLines = require('../helpers').getLines;
-var parseRef = require('../parse_ref');
+var parseRef = require('../parser_ref');
 
 
 module.exports = function paragraph(state, startLine/*, endLine*/) {
   var endLine, content, pos,
       nextLine = startLine + 1,
-      rules_named = state.lexer.rules_named;
+      rules_named = state.parser.rules_named;
 
   endLine = state.lineMax;
 
@@ -38,7 +38,7 @@ module.exports = function paragraph(state, startLine/*, endLine*/) {
   content = getLines(state, startLine, nextLine, state.blkIndent, false).trim();
 
   while (content.length) {
-    pos = parseRef(content, state.lexer.inline, state.options, state.env);
+    pos = parseRef(content, state.parser.inline, state.options, state.env);
     if (pos < 0) { break; }
     content = content.slice(pos).trim();
   }

lib/lexer_block/state_block.js → lib/rules_block/state_block.js

@@ -3,7 +3,7 @@
 'use strict';
 
 
-function State(src, lexer, tokens, options, env) {
+function State(src, parser, tokens, options, env) {
   var ch, s, start, pos, len, indent, indent_found;
 
   // TODO: check if we can move string replaces to parser, to avoid
@@ -18,7 +18,7 @@ function State(src, lexer, tokens, options, env) {
   this.src = src;
 
   // Shortcuts to simplify nested calls
-  this.lexer  = lexer;
+  this.parser = parser;
 
   // TODO: (?) set directly for faster access.
   this.options = options;
@@ -76,10 +76,10 @@ function State(src, lexer, tokens, options, env) {
   this.eMarks.push(s.length);
   this.tShift.push(0);
 
-  // inline lexer variables
+  // inline parser variables
   this.pos        = 0; // char index in src
 
-  // block lexer variables
+  // block parser variables
   this.blkLevel   = 0;
   this.blkIndent  = 0;
   this.line       = 0; // line index in src
@@ -105,7 +105,7 @@ function State(src, lexer, tokens, options, env) {
 State.prototype.clone = function clone(src) {
   return new State(
     src,
-    this.lexer,
+    this.parser,
     this.tokens,
     this.options
   );

lib/lexer_inline/emphasis.js → lib/rules_inline/emphasis.js

@@ -184,7 +184,7 @@ module.exports = function emphasis(state/*, silent*/) {
       }
     }
 
-    ok = state.lexer.tokenizeSingle(state);
+    ok = state.parser.tokenizeSingle(state);
 
     if (ok) {
       haveLiteralAsterisk = false;
@@ -217,7 +217,7 @@ module.exports = function emphasis(state/*, silent*/) {
     state.push({ type: 'em_open', level: state.level++ });
   }
 
-  state.lexer.tokenize(state);
+  state.parser.tokenize(state);
 
   if (startCount === 1 || startCount === 3) {
     state.push({ type: 'em_close', level: --state.level });

lib/lexer_inline/links.js → lib/rules_inline/links.js

@@ -35,7 +35,7 @@ function parseLinkLabel(state, start) {
       }
     }
 
-    ok = state.lexer.tokenizeSingle(state);
+    ok = state.parser.tokenizeSingle(state);
 
     if (!ok) { state.pending += state.src[state.pos++]; }
   }
@@ -296,7 +296,7 @@ function links(state) {
       title: title,
       level: state.level++
     });
-    state.lexer.tokenize(state);
+    state.parser.tokenize(state);
     state.push({ type: 'link_close', level: --state.level });
   }
 

lib/lexer_inline/state_inline.js → lib/rules_inline/state_inline.js

@@ -3,11 +3,11 @@
 'use strict';
 
 
-function StateInline(src, lexer, options, env) {
+function StateInline(src, parser, options, env) {
   this.src = src;
   this.env = env;
   this.options = options;
-  this.lexer = lexer;
+  this.parser = parser;
   this.tokens = [];
   this.pos = 0;
   this.pending = '';

lib/lexer_inline/text.js → lib/rules_inline/text.js

@@ -2,7 +2,7 @@
 // and increment current pos
 
 module.exports = function text(state) {
-  var match = state.src.slice(state.pos).match(state.lexer.textMatch);
+  var match = state.src.slice(state.pos).match(state.parser.textMatch);
 
   if (!match) { return false; }