Docs update

docs/4.0_migration.md

@@ -0,0 +1,19 @@
+Migration to v4
+===============
+
+v4 has the same external API as v3, but significantly changed internals. Plugin
+authors should update their packages.
+
+## Tokens and renderer
+
+- [Tokens](https://github.com/markdown-it/markdown-it/blob/master/lib/token.js)
+  are now classes, and allow arbitrary attributes.
+  - new tokens are created with `token = state.push(type, tag, nesting)`.
+    See [this commit](https://github.com/markdown-it/markdown-it/commit/4aabd5592ea55fb43d6a215b316c89c6f6f1f7db) to understand
+    how to create tokens in new way.
+- [Renderer](https://github.com/markdown-it/markdown-it/blob/master/lib/renderer.js)
+  methods were unified. Number of custom renderer rules were significantly reduced.
+
+## Other changes
+
+- `.validateLink()` moved to root (MarkdownIt) class.

docs/architecture.md

@@ -17,7 +17,7 @@ core
         block.ruleX
 
     core.ruleX1 (intermediate rule that applies on block tokens, nothing yet)
-    ... 
+    ...
     core.ruleXX
 
     inline (applied to each block token with "inline" type)
@@ -49,17 +49,8 @@ The difference is simple:
 - There are special token objects, "inline containers", having nested tokens.
   sequences with inline markup (bold, italic, text, ...).
 
-Each token has some common fields:
-
-- __type__ - token name.
-- __level__ - nesting level, useful to seek closing pair.
-- __lines__ - [begin, end], for block tokens only. Range of input lines,
-  compiled to this token.
-
-Inline tokens (`type === "inline"`) have additional properties:
-
-- __content__ - raw text, unparsed inline content.
-- __children__ - nested token stream.
+See [source](https://github.com/markdown-it/markdown-it/blob/master/lib/token.js)
+for details about each token content.
 
 In total, a token stream is:
 

lib/token.js

@@ -3,50 +3,105 @@
 'use strict';
 
 
-// Create a token
-//
+/**
+ * class Token
+ **/
+
+/**
+ * new Token(type, tag, nesting)
+ *
+ * Create new token and fill passed properties.
+ **/
 function Token(type, tag, nesting) {
-  // type of the token (string, e.g. "paragraph_open")
+  /**
+   * Token#type -> String
+   *
+   * Type of the token (string, e.g. "paragraph_open")
+   **/
   this.type     = type;
 
-  // html tag name, e.g. "p"
+  /**
+   * Token#tag -> String
+   *
+   * html tag name, e.g. "p"
+   **/
   this.tag      = tag;
 
-  // html attributes, format:
-  // [ [ name1, value1 ], [ name2, value2 ] ]
+  /**
+   * Token#attrs -> Array
+   *
+   * Html attributes. Format: `[ [ name1, value1 ], [ name2, value2 ] ]`
+   **/
   this.attrs    = null;
 
-  // source map info, format:
-  // [ line_begin, line_end ]
+  /**
+   * Token#map -> Array
+   *
+   * Source map info. Format: `[ line_begin, line_end ]`
+   **/
   this.map      = null;
 
-  // level change (number in {-1, 0, 1} set), where:
-  //  -  `1` means the tag is opening
-  //  -  `0` means the tag is self-closing
-  //  - `-1` means the tag is closing
+  /**
+   * Token#nesting -> Number
+   *
+   * Level change (number in {-1, 0, 1} set), where:
+   *
+   * -  `1` means the tag is opening
+   * -  `0` means the tag is self-closing
+   * - `-1` means the tag is closing
+   **/
   this.nesting  = nesting;
 
-  // nesting level, same as `state.level`
+  /**
+   * Token#level -> Number
+   *
+   * nesting level, the same as `state.level`
+   **/
   this.level    = 0;
 
-  // an array of child nodes (inline and img tokens)
+  /**
+   * Token#children -> Array
+   *
+   * An array of child nodes (inline and img tokens)
+   **/
   this.children = null;
 
-  // in a case of self-closing tag (code, html, fence, etc.),
-  // it has contents of this tag
+  /**
+   * Token#content -> String
+   *
+   * In a case of self-closing tag (code, html, fence, etc.),
+   * it has contents of this tag.
+   **/
   this.content  = null;
 
-  // '*' or '_' for emphasis, fence string for fence, etc.
+  /**
+   * Token#markup -> String
+   *
+   * '*' or '_' for emphasis, fence string for fence, etc.
+   **/
   this.markup   = '';
 
-  // fence infostring
+  /**
+   * Token#info -> String
+   *
+   * fence infostring
+   **/
   this.info     = null;
 
-  // block or inline-level token,
-  // used in renderer to calculate line breaks
+  /**
+   * Token#block -> Boolean
+   *
+   * True for block-level tokens, false for inline tokens.
+   * Used in renderer to calculate line breaks
+   **/
   this.block    = false;
 
-  // if it's true, ignore this element when rendering
+  /**
+   * Token#hidden -> Boolean
+   *
+   * If it's true, ignore this element when rendering. Used for tight lists
+   * to hide paragraphs.
+   **/
   this.hidden   = false;
 }