fix/handle text_special and text_join rule causing incorrect position and size with escaped links

dist/index.cjs.js

@@ -1866,10 +1866,17 @@ function text_join(state) {
         tokens[curr].type = 'text';
       }
     }
-    for (curr = last = 0; curr < max; curr++) {
+    for (curr = last = 0; curr < tokens.length; curr++) {
       if (tokens[curr].type === 'text' && curr + 1 < max && tokens[curr + 1].type === 'text') {
         // collapse two adjacent text nodes
         tokens[curr + 1].content = tokens[curr].content + tokens[curr + 1].content;
+
+        // only move forward position when left token has content
+        if (tokens[curr].content) {
+          tokens[curr + 1].position = tokens[curr].position;
+        }
+        // add up size
+        tokens[curr + 1].size = (tokens[curr].size || 0) + (tokens[curr + 1].size || 0);
       } else {
         if (curr !== last) {
           tokens[last] = tokens[curr];
@@ -2138,6 +2145,11 @@ StateBlock.prototype.getLines = function getLines(begin, end, indent, keepLastLF
     if (replaceIndentSpaceWithZWSP) {
       queue[i] = new Array(first - lineStart + 1).join(ZWSP) + queue[i];
     }
+
+    // Ensure a trailing LF when requested even if source lacks one
+    if (keepLastLF && !queue[i].endsWith('\n')) {
+      queue[i] += '\n';
+    }
   }
   return queue.join('');
 };
@@ -4076,6 +4088,8 @@ function escape(state, silent) {
     }
     token.markup = origStr;
     token.info = 'escape';
+    // size should reflect original source span (including backslash)
+    token.size = origStr.length;
   }
   state.pos = pos + 1;
   return true;

dist/markdown-it.js

@@ -1,4 +1,4 @@
-/*! @hackmd/markdown-it 14.1.0 https://github.com/hackmdio/markdown-it @license MIT */
+/*! @hackmd/markdown-it 14.1.1 https://github.com/hackmdio/markdown-it @license MIT */
 (function(global, factory) {
   typeof exports === "object" && typeof module !== "undefined" ? module.exports = factory() : typeof define === "function" && define.amd ? define(factory) : (global = typeof globalThis !== "undefined" ? globalThis : global || self, 
   global.markdownit = factory());
@@ -2628,10 +2628,16 @@
           tokens[curr].type = "text";
         }
       }
-      for (curr = last = 0; curr < max; curr++) {
+      for (curr = last = 0; curr < tokens.length; curr++) {
         if (tokens[curr].type === "text" && curr + 1 < max && tokens[curr + 1].type === "text") {
           // collapse two adjacent text nodes
           tokens[curr + 1].content = tokens[curr].content + tokens[curr + 1].content;
+          // only move forward position when left token has content
+                    if (tokens[curr].content) {
+            tokens[curr + 1].position = tokens[curr].position;
+          }
+          // add up size
+                    tokens[curr + 1].size = (tokens[curr].size || 0) + (tokens[curr + 1].size || 0);
         } else {
           if (curr !== last) {
             tokens[last] = tokens[curr];
@@ -2889,6 +2895,10 @@
       if (replaceIndentSpaceWithZWSP) {
         queue[i] = new Array(first - lineStart + 1).join(ZWSP) + queue[i];
       }
+      // Ensure a trailing LF when requested even if source lacks one
+            if (keepLastLF && !queue[i].endsWith("\n")) {
+        queue[i] += "\n";
+      }
     }
     return queue.join("");
   };
@@ -4672,6 +4682,8 @@
       }
       token.markup = origStr;
       token.info = "escape";
+      // size should reflect original source span (including backslash)
+            token.size = origStr.length;
     }
     state.pos = pos + 1;
     return true;

lib/rules_block/state_block.mjs

@@ -224,6 +224,11 @@ StateBlock.prototype.getLines = function getLines (begin, end, indent, keepLastL
     if (replaceIndentSpaceWithZWSP) {
       queue[i] = new Array(first - lineStart + 1).join(ZWSP) + queue[i]
     }
+
+    // Ensure a trailing LF when requested even if source lacks one
+    if (keepLastLF && !queue[i].endsWith('\n')) {
+      queue[i] += '\n'
+    }
   }
 
   return queue.join('')

lib/rules_core/text_join.mjs

@@ -23,12 +23,19 @@ export default function text_join (state) {
       }
     }
 
-    for (curr = last = 0; curr < max; curr++) {
+    for (curr = last = 0; curr < tokens.length; curr++) {
       if (tokens[curr].type === 'text' &&
           curr + 1 < max &&
           tokens[curr + 1].type === 'text') {
         // collapse two adjacent text nodes
         tokens[curr + 1].content = tokens[curr].content + tokens[curr + 1].content
+
+        // only move forward position when left token has content
+        if (tokens[curr].content) {
+          tokens[curr + 1].position = tokens[curr].position
+        }
+        // add up size
+        tokens[curr + 1].size = (tokens[curr].size || 0) + (tokens[curr + 1].size || 0)
       } else {
         if (curr !== last) { tokens[last] = tokens[curr] }
 

lib/rules_inline/escape.mjs

@@ -62,6 +62,8 @@ export default function escape (state, silent) {
 
     token.markup = origStr
     token.info   = 'escape'
+    // size should reflect original source span (including backslash)
+    token.size = origStr.length
   }
 
   state.pos = pos + 1

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackmd/markdown-it",
-  "version": "14.1.0",
+  "version": "14.1.1",
   "description": "Markdown-it - modern pluggable markdown parser.",
   "keywords": [
     "markdown",

test/commonmark.mjs

@@ -3,6 +3,7 @@ import { relative } from 'node:path'
 import { load } from 'markdown-it-testgen'
 import markdownit from '../index.mjs'
 import { assert } from 'chai'
+import { stripZeroWidthSpaces } from './patch.mjs'
 
 function normalize (text) {
   return text.replace(/<blockquote>\n<\/blockquote>/g, '<blockquote></blockquote>')
@@ -26,6 +27,7 @@ function generate (path, md) {
 
 describe('CommonMark', function () {
   const md = markdownit('commonmark')
+  md.use(stripZeroWidthSpaces)
 
   generate(fileURLToPath(new URL('fixtures/commonmark/good.txt', import.meta.url)), md)
 })

test/markdown-it.mjs

@@ -1,6 +1,7 @@
 import { fileURLToPath } from 'node:url'
 import generate from 'markdown-it-testgen'
 import markdownit from '../index.mjs'
+import { stripZeroWidthSpaces } from './patch.mjs'
 
 describe('markdown-it', function () {
   const md = markdownit({
@@ -9,6 +10,7 @@ describe('markdown-it', function () {
     typographer: true,
     linkify: true
   })
+  md.use(stripZeroWidthSpaces)
 
   generate(fileURLToPath(new URL('fixtures/markdown-it', import.meta.url)), md)
 })

test/misc.mjs

@@ -1,6 +1,7 @@
 import { assert } from 'chai'
 import markdownit from '../index.mjs'
 import forInline from 'markdown-it-for-inline'
+import { stripZeroWidthSpaces } from './patch.mjs'
 
 describe('API', function () {
   it('constructor', function () {
@@ -209,6 +210,7 @@ describe('Misc', function () {
 
   it('Should correctly parse strings without tailing \\n', function () {
     const md = markdownit()
+    md.use(stripZeroWidthSpaces)
 
     assert.strictEqual(md.render('123'), '<p>123</p>\n')
     assert.strictEqual(md.render('123\n'), '<p>123</p>\n')

test/patch.mjs

@@ -0,0 +1,8 @@
+export function stripZeroWidthSpaces (md) {
+  const originalRender = md.render.bind(md)
+
+  md.render = (src, env) => {
+    const result = originalRender(src, env)
+    return result.replace(/\u200b/g, '')
+  }
+}