diff --git a/lib/rules_core/smartquotes.js b/lib/rules_core/smartquotes.js
index a0483b9..e599990 100644
--- a/lib/rules_core/smartquotes.js
+++ b/lib/rules_core/smartquotes.js
@@ -50,9 +50,37 @@ function process_inlines(tokens, state) {
       pos = t.index + 1;
       isSingle = (t[0] === "'");
 
-      // treat begin/end of the line as a whitespace
-      lastChar = t.index - 1 >= 0 ? text.charCodeAt(t.index - 1) : 0x20;
-      nextChar = pos < max ? text.charCodeAt(pos) : 0x20;
+      // Find previous character,
+      // default to space if it's the beginning of the line
+      //
+      lastChar = 0x20;
+
+      if (t.index - 1 >= 0) {
+        lastChar = text.charCodeAt(t.index - 1);
+      } else {
+        for (j = i - 1; j >= 0; j--) {
+          if (tokens[j].type !== 'text') { continue; }
+
+          lastChar = tokens[j].content.charCodeAt(tokens[j].content.length - 1);
+          break;
+        }
+      }
+
+      // Find next character,
+      // default to space if it's the end of the line
+      //
+      nextChar = 0x20;
+
+      if (pos < max) {
+        nextChar = text.charCodeAt(pos);
+      } else {
+        for (j = i + 1; j < tokens.length; j++) {
+          if (tokens[j].type !== 'text') { continue; }
+
+          nextChar = tokens[j].content.charCodeAt(0);
+          break;
+        }
+      }
 
       isLastPunctChar = isMdAsciiPunct(lastChar) || isPunctChar(String.fromCharCode(lastChar));
       isNextPunctChar = isMdAsciiPunct(nextChar) || isPunctChar(String.fromCharCode(nextChar));
diff --git a/test/fixtures/markdown-it/smartquotes.txt b/test/fixtures/markdown-it/smartquotes.txt
index aaea704..7b62307 100644
--- a/test/fixtures/markdown-it/smartquotes.txt
+++ b/test/fixtures/markdown-it/smartquotes.txt
@@ -96,3 +96,16 @@ Quotes between punctuation chars:
 .
 <p>“(hai)”.</p>
 .
+
+Quotes at the start/end of the tokens:
+.
+"*foo* bar"
+
+"foo *bar*"
+
+"*foo bar*"
+.
+<p>“<em>foo</em> bar”</p>
+<p>“foo <em>bar</em>”</p>
+<p>“<em>foo bar</em>”</p>
+.