Fixed entities in links & fences according to new spec

10 years ago · 6b0c7665b8
4 changed files with 184 additions and 239 deletions
--- a/lib/helpers.js
+++ b/lib/helpers.js
@ -137,6 +137,20 @@ function fromCodePoint(c) {
  return String.fromCharCode(c);
 }

+var NAMED_ENTITY_RE   = /&([a-z][a-z0-9]{1,31});/gi;
+var entities = require('./common/entities');
+
+function replaceEntities(str) {
+  if (str.indexOf('&') < 0) { return str; }
+
+  return str.replace(NAMED_ENTITY_RE, function(match, name) {
+    if (entities.hasOwnProperty(name)) {
+      return entities[name];
+    }
+    return match;
+  });
+}
+
 exports.isWhiteSpace = isWhiteSpace;
 exports.isEmpty = isEmpty;
 exports.skipEmptyLines = skipEmptyLines;
@ -149,3 +163,4 @@ exports.unescapeMd = unescapeMd;
 exports.escapeHtmlKeepEntities = escapeHtmlKeepEntities;
 exports.isValidEntityCode = isValidEntityCode;
 exports.fromCodePoint = fromCodePoint;
+exports.replaceEntities = replaceEntities;
--- a/lib/renderer.js
+++ b/lib/renderer.js
@ -1,10 +1,11 @@
 'use strict';


-var assign = require('object-assign');
-var escapeHtml = require('./helpers').escapeHtml;
-var escapeHtmlKeepEntities = require('./helpers').escapeHtmlKeepEntities;
-var unescapeMd = require('./helpers').unescapeMd;
+var assign          = require('object-assign');
+var escapeHtml      = require('./helpers').escapeHtml;
+var unescapeMd      = require('./helpers').unescapeMd;
+var replaceEntities = require('./helpers').replaceEntities;
+//var escapeHtmlKeepEntities = require('./helpers').escapeHtmlKeepEntities;


 function escapeUrl(str) {
@ -13,6 +14,13 @@ function escapeUrl(str) {
  } catch (__) {}
  return '';
 }
+function unescapeUrl(str) {
+  try {
+    return decodeURI(str);
+  } catch (__) {}
+  return '';
+}
+

 // check if we need to hide '\n' before next token
 function getBreak(tokens, idx) {
@ -52,7 +60,7 @@ rules.fence = function (tokens, idx, options) {

  if (token.params) {
    params = token.params.split(/ +/g);
-    langMark = ' class="' + langPrefix + escapeHtmlKeepEntities(unescapeMd(params[0])) + '"';
+    langMark = ' class="' + langPrefix + escapeHtml(replaceEntities(unescapeMd(params[0]))) + '"';
  }

  return  '<pre><code' + langMark + '>'
@ -106,8 +114,8 @@ rules.paragraph_close = function (tokens, idx /*, options*/) {


 rules.link_open = function (tokens, idx /*, options*/) {
-  var title = tokens[idx].title ? (' title="' + escapeHtmlKeepEntities(tokens[idx].title) + '"') : '';
-  return '<a href="' + escapeHtml(escapeUrl(tokens[idx].href)) + '"' + title + '>';
+  var title = tokens[idx].title ? (' title="' + escapeHtml(replaceEntities(tokens[idx].title)) + '"') : '';
+  return '<a href="' + escapeHtml(escapeUrl(unescapeUrl(replaceEntities(tokens[idx].href)))) + '"' + title + '>';
 };
 rules.link_close = function (/*tokens, idx, options*/) {
  return '</a>';
@ -116,8 +124,8 @@ rules.link_close = function (/*tokens, idx, options*/) {

 rules.image = function (tokens, idx, options) {
  var src = ' src="' + escapeHtml(escapeUrl(tokens[idx].src)) + '"';
-  var title = tokens[idx].title ? (' title="' + escapeHtmlKeepEntities(tokens[idx].title) + '"') : '';
-  var alt = ' alt="' + (tokens[idx].alt ? escapeHtmlKeepEntities(tokens[idx].alt) : '') + '"';
+  var title = tokens[idx].title ? (' title="' + escapeHtml(replaceEntities(tokens[idx].title)) + '"') : '';
+  var alt = ' alt="' + (tokens[idx].alt ? escapeHtml(replaceEntities(tokens[idx].alt)) : '') + '"';
  var suffix = options.xhtml ? ' /' : '';
  return '<img' + src + alt + title + suffix + '>';
 };
--- a/test/fixtures/stmd/bad.txt
+++ b/test/fixtures/stmd/bad.txt
@ -1,59 +1,3 @@
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 1406
-
-.
-<DIV CLASS="foo">
-
-*Markdown*
-
-</DIV>
-.
-<DIV CLASS="foo">
-<p><em>Markdown</em></p>
-</DIV>
-.
-
-error:
-
-<p><DIV CLASS="foo"></p>
-<p><em>Markdown</em></p>
-<p></DIV></p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 1678
-
-.
-[Foo bar]:
-<my url>
-'title'
-
-[Foo bar]
-.
-<p><a href="my%20url" title="title">Foo bar</a></p>
-.
-
-error:
-
-<p><a href="my url" title="title">Foo bar</a></p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 1743
-
-.
-[ΑΓΩ]: /φου
-
-[αγω]
-.
-<p><a href="/%CF%86%CE%BF%CF%85">αγω</a></p>
-.
-
-error:
-
-<p><a href="/φου">αγω</a></p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3550

@ -83,138 +27,6 @@ baz</li>
 </ul>


-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3688
-
-.
-<http://google.com?find=\*>
-.
-<p><a href="http://google.com?find=%5C*">http://google.com?find=\*</a></p>
-.
-
-error:
-
-<p><a href="http://google.com?find=\*">http://google.com?find=\*</a></p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3746
-
-.
-&nbsp; &amp; &copy; &AElig; &Dcaron; &frac34; &HilbertSpace; &DifferentialD; &ClockwiseContourIntegral;
-.
-<p>  &amp; © Æ Ď ¾ ℋ ⅆ ∲</p>
-.
-
-error:
-
-<p>&nbsp; &amp; &copy; &AElig; &Dcaron; &frac34; &HilbertSpace; &DifferentialD; &ClockwiseContourIntegral;</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3757
-
-.
-&#35; &#1234; &#992; &#98765432;
-.
-<p># Ӓ Ϡ �</p>
-.
-
-error:
-
-<p>&#35; &#1234; &#992; &#98765432;</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3767
-
-.
-&#X22; &#XD06; &#xcab;
-.
-<p>&quot; ആ ಫ</p>
-.
-
-error:
-
-<p>&#X22; &#XD06; &#xcab;</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3792
-
-.
-&MadeUpEntity;
-.
-<p>&amp;MadeUpEntity;</p>
-.
-
-error:
-
-<p>&MadeUpEntity;</p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3808
-
-.
-[foo](/f&ouml;&ouml; "f&ouml;&ouml;")
-.
-<p><a href="/f%C3%B6%C3%B6" title="föö">foo</a></p>
-.
-
-error:
-
-<p><a href="/f&ouml;&ouml;" title="f&ouml;&ouml;">foo</a></p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3814
-
-.
-[foo]
-
-[foo]: /f&ouml;&ouml; "f&ouml;&ouml;"
-.
-<p><a href="/f%C3%B6%C3%B6" title="föö">foo</a></p>
-.
-
-error:
-
-<p><a href="/f&ouml;&ouml;" title="f&ouml;&ouml;">foo</a></p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3822
-
-.
-``` f&ouml;&ouml;
-foo
-```
-.
-<pre><code class="language-föö">foo
-</code></pre>
-.
-
-error:
-
-<pre><code class="language-f&ouml;&ouml;">foo
-</code></pre>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 3957
-
-.
-<http://foo.bar.`baz>`
-.
-<p><a href="http://foo.bar.%60baz">http://foo.bar.`baz</a>`</p>
-.
-
-error:
-
-<p><a href="http://foo.bar.`baz">http://foo.bar.`baz</a>`</p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 4336

@ -229,48 +41,6 @@ error:
 <p>*here is a **</p>


-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 4766
-
-.
-[link](</my uri>)
-.
-<p><a href="/my%20uri">link</a></p>
-.
-
-error:
-
-<p><a href="/my uri">link</a></p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 4824
-
-.
-[link](foo%20b&auml;)
-.
-<p><a href="foo%20b%C3%A4">link</a></p>
-.
-
-error:
-
-<p><a href="foo%20b&auml;">link</a></p>
-
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-src line: 4834
-
-.
-[link]("title")
-.
-<p><a href="%22title%22">link</a></p>
-.
-
-error:
-
-<p><a href="&quot;title&quot;">link</a></p>
-
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 4892

--- a/test/fixtures/stmd/good.txt
+++ b/test/fixtures/stmd/good.txt
@ -1133,6 +1133,21 @@ src line: 1394
         <foo><a>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 1406
+
+.
+<DIV CLASS="foo">
+
+*Markdown*
+
+</DIV>
+.
+<DIV CLASS="foo">
+<p><em>Markdown</em></p>
+</DIV>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 1422

@ -1349,6 +1364,19 @@ src line: 1670
 <p><a href="my_(url)" title="title (with parens)">Foo*bar]</a></p>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 1678
+
+.
+[Foo bar]:
+<my url>
+'title'
+
+[Foo bar]
+.
+<p><a href="my%20url" title="title">Foo bar</a></p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 1690

@ -1407,6 +1435,17 @@ src line: 1735
 <p><a href="/url">Foo</a></p>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 1743
+
+.
+[ΑΓΩ]: /φου
+
+[αγω]
+.
+<p><a href="/%CF%86%CE%BF%CF%85">αγω</a></p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 1754

@ -3036,6 +3075,15 @@ src line: 3679
 </code></pre>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3688
+
+.
+<http://google.com?find=\*>
+.
+<p><a href="http://google.com?find=%5C*">http://google.com?find=\*</a></p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3694

@ -3077,6 +3125,33 @@ foo
 </code></pre>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3746
+
+.
+&nbsp; &amp; &copy; &AElig; &Dcaron; &frac34; &HilbertSpace; &DifferentialD; &ClockwiseContourIntegral;
+.
+<p>  &amp; © Æ Ď ¾ ℋ ⅆ ∲</p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3757
+
+.
+&#35; &#1234; &#992; &#98765432;
+.
+<p># Ӓ Ϡ �</p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3767
+
+.
+&#X22; &#XD06; &#xcab;
+.
+<p>&quot; ആ ಫ</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3775

@ -3095,6 +3170,15 @@ src line: 3784
 <p>&amp;copy</p>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3792
+
+.
+&MadeUpEntity;
+.
+<p>&amp;MadeUpEntity;</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3802

@ -3104,6 +3188,38 @@ src line: 3802
 <p><a href="&ouml;&ouml;.html"></p>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3808
+
+.
+[foo](/f&ouml;&ouml; "f&ouml;&ouml;")
+.
+<p><a href="/f%C3%B6%C3%B6" title="föö">foo</a></p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3814
+
+.
+[foo]
+
+[foo]: /f&ouml;&ouml; "f&ouml;&ouml;"
+.
+<p><a href="/f%C3%B6%C3%B6" title="föö">foo</a></p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3822
+
+.
+``` f&ouml;&ouml;
+foo
+```
+.
+<pre><code class="language-föö">foo
+</code></pre>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3833

@ -3207,6 +3323,15 @@ src line: 3949
 <p>[not a <code>link](/foo</code>)</p>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 3957
+
+.
+<http://foo.bar.`baz>`
+.
+<p><a href="http://foo.bar.%60baz">http://foo.bar.`baz</a>`</p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 3965

@ -4022,6 +4147,15 @@ src line: 4760
 <p>[link](/my uri)</p>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 4766
+
+.
+[link](</my uri>)
+.
+<p><a href="/my%20uri">link</a></p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 4774

@ -4078,6 +4212,24 @@ src line: 4814
 <p><a href="foo):">link</a></p>
 .

+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 4824
+
+.
+[link](foo%20b&auml;)
+.
+<p><a href="foo%20b%C3%A4">link</a></p>
+.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+src line: 4834
+
+.
+[link]("title")
+.
+<p><a href="%22title%22">link</a></p>
+.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 src line: 4842