Don't trim non-ASCII whitespaces

lib/common/utils.mjs

@@ -278,6 +278,27 @@ function normalizeReference (str) {
   return str.toLowerCase().toUpperCase()
 }
 
+// Removes space-like characters that are allowed to be removed in CommonMark spec.
+function trimMinimalSpaces (str) {
+  let start = 0
+  for (; start < str.length; start++) {
+    if (!isSpaceLike(str.charCodeAt(start))) {
+      break
+    }
+  }
+  let end = str.length - 1
+  for (; end >= start; end--) {
+    if (!isSpaceLike(str.charCodeAt(end))) {
+      break
+    }
+  }
+  return str.slice(start, end + 1)
+
+  function isSpaceLike (c) {
+    return c === 0x20 || c === 0x09 || c === 0x0a || c === 0x0d
+  }
+}
+
 // Re-export libraries commonly used in both markdown-it and its plugins,
 // so plugins won't have to depend on them explicitly, which reduces their
 // bundled size (e.g. a browser build).
@@ -300,5 +321,6 @@ export {
   isMdAsciiPunct,
   isPunctChar,
   escapeRE,
-  normalizeReference
+  normalizeReference,
+  trimMinimalSpaces
 }

lib/rules_block/heading.mjs

@@ -1,6 +1,6 @@
 // heading (#, ##, ...)
 
-import { isSpace } from '../common/utils.mjs'
+import { isSpace, trimMinimalSpaces } from '../common/utils.mjs'
 
 export default function heading (state, startLine, endLine, silent) {
   let pos = state.bMarks[startLine] + state.tShift[startLine]
@@ -40,7 +40,7 @@ export default function heading (state, startLine, endLine, silent) {
   token_o.map    = [startLine, state.line]
 
   const token_i    = state.push('inline', '', 0)
-  token_i.content  = state.src.slice(pos, max).trim()
+  token_i.content  = trimMinimalSpaces(state.src.slice(pos, max))
   token_i.map      = [startLine, state.line]
   token_i.children = []
 

lib/rules_block/lheading.mjs

@@ -1,5 +1,7 @@
 // lheading (---, ===)
 
+import { trimMinimalSpaces } from '../common/utils.mjs'
+
 export default function lheading (state, startLine, endLine/*, silent */) {
   const terminatorRules = state.md.block.ruler.getRules('paragraph')
 
@@ -60,7 +62,7 @@ export default function lheading (state, startLine, endLine/*, silent */) {
     return false
   }
 
-  const content = state.getLines(startLine, nextLine, state.blkIndent, false).trim()
+  const content = trimMinimalSpaces(state.getLines(startLine, nextLine, state.blkIndent, false))
 
   state.line = nextLine + 1
 

lib/rules_block/paragraph.mjs

@@ -1,5 +1,7 @@
 // Paragraph
 
+import { trimMinimalSpaces } from '../common/utils.mjs'
+
 export default function paragraph (state, startLine, endLine) {
   const terminatorRules = state.md.block.ruler.getRules('paragraph')
   const oldParentType = state.parentType
@@ -26,7 +28,7 @@ export default function paragraph (state, startLine, endLine) {
     if (terminate) { break }
   }
 
-  const content = state.getLines(startLine, nextLine, state.blkIndent, false).trim()
+  const content = trimMinimalSpaces(state.getLines(startLine, nextLine, state.blkIndent, false))
 
   state.line = nextLine
 

test/fixtures/markdown-it/commonmark_extras.txt

@@ -740,3 +740,18 @@ Html in image description
 .
 <p><img src="image.png" alt="text &lt;textarea&gt; text"></p>
 .
+
+
+Issue #1067. Don't trim non-ASCII whitespaces
+.
+# 　U+3000　  
+
+  　U+3000　  
+=
+
+  　U+3000　  
+.
+<h1>　U+3000　</h1>
+<h1>　U+3000　</h1>
+<p>　U+3000　</p>
+.