fix(chat): preserve backticks inside citation JSON

Author: waleedlatif1

apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts

@@ -2,10 +2,39 @@
  * @vitest-environment node
  */
 import { describe, expect, it } from 'vitest'
+import { sanitizeChatDisplayContent } from '@/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize'
 import { scalingRatioOver4x } from '@/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers'
-import { sanitizeChatDisplayContent } from './chat-sanitize'
 
 describe('sanitizeChatDisplayContent', () => {
+  it.each(['source', 'workspace_resource'])('preserves backticks inside %s JSON', (name) => {
+    const payload = JSON.stringify({
+      url: 'https://example.com',
+      type: 'file',
+      path: 'notes`draft`.md',
+      title: 'Use `config` and "quoted" text',
+    })
+    const tag = `<${name}>${payload}</${name}>`
+
+    expect(sanitizeChatDisplayContent(`\`${tag}\``)).toBe(tag)
+    expect(sanitizeChatDisplayContent(`\`See ${tag}.\``)).toBe(`See ${tag}.`)
+    expect(sanitizeChatDisplayContent(`\`${tag} done`)).toBe(`${tag} done`)
+    expect(sanitizeChatDisplayContent(`${tag}\` done`)).toBe(`${tag} done`)
+    expect(sanitizeChatDisplayContent(`\`a\`${tag}\`b\``)).toBe(`\`a\`${tag}\`b\``)
+    expect(sanitizeChatDisplayContent(`\`\`\`json\n${tag}\n\`\`\``)).toBe(
+      `\`\`\`json\n${tag}\n\`\`\``
+    )
+  })
+
+  it.each(['source', 'workspace_resource'])(
+    'unwraps %s JSON that mentions the other chip tag',
+    (name) => {
+      const otherTag = name === 'source' ? 'workspace_resource' : 'source'
+      const tag = `<${name}>${JSON.stringify({ title: `Use <${otherTag}>` })}</${name}>`
+
+      expect(sanitizeChatDisplayContent(`\`${tag}\``)).toBe(tag)
+    }
+  )
+
   it('unwraps workspace resource tags from inline code spans', () => {
     const content =
       '`I updated <workspace_resource>{"type":"workflow","id":"wf-1","title":"Workflow"}</workspace_resource>.`'
@@ -120,4 +149,15 @@ describe('sanitizeChatDisplayContent', () => {
       '<workspace_resource>{"type":"file","path":"a.md","title":"a"}</workspace_resource> done'
     )
   })
+
+  it.each(['source', 'workspace_resource'])(
+    'stays linear on repeated %s tags with unterminated JSON strings',
+    (name) => {
+      expect(
+        scalingRatioOver4x(sanitizeChatDisplayContent, (times) =>
+          `<${name}>${String.fromCharCode(92, 34)}`.repeat(times)
+        )
+      ).toBeLessThan(8)
+    }
+  )
 })

apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts

@@ -1,65 +1,65 @@
 const HIDDEN_INLINE_REFERENCE_PATTERN =
   /`[^`\n]*(?:internal\/tool-results\/|internal\/blocktips\/|components\/integrations\/[^`\n]*README)[^`\n]*`/g
 
-/**
- * A complete inline-chip tag — `<workspace_resource>` or `<source>` — as
- * opener, payload, closer. Both are JSON-bodied tags the model places inside a
- * sentence, so both attract the same stray backticks.
- *
- * Two constraints on the payload, both load-bearing:
- *
- * - **No backtick.** A payload is JSON and carries none, so this is what tells a
- *   real tag from prose MENTIONING the tag name — a message explaining the
- *   syntax writes the opener and the closer as two separately backticked spans.
- * - **No nested opener**, via the negative lookahead. A cost bound rather than a
- *   correctness rule: a lazy scan allowed to cross an opener restarts from every
- *   opener, so a message repeating the tag name is quadratic — on the main
- *   thread, for every streamed chunk.
- *
- * Accepted trade: a resource whose title or path itself contains a backtick is
- * not matched, so it renders as text rather than a chip. That costs one chip and
- * is rare; the failure it replaces corrupts a whole message and is common.
- */
-const COMPLETE_TAG_SOURCE =
-  '<(?<chipTag>workspace_resource|source)>(?:(?!<\\k<chipTag>>)[^`])*?<\\/\\k<chipTag>>'
-
-/** Non-global so {@link RegExp.test} has no `lastIndex` to carry between calls. */
-const COMPLETE_INLINE_CHIP_TAG = new RegExp(COMPLETE_TAG_SOURCE)
+/** A matching nested opener bounds every payload scan, including quoted strings. */
+function chipTagPattern(tag: 'workspace_resource' | 'source'): string {
+  const noNestedTag = `(?!<${tag}>)`
+  /** JSON strings own their backticks; valid escapes cannot consume a nested tag's opener. */
+  const jsonString = `"(?:${noNestedTag}(?:\\\\(?:["\\\\/bfnrt]|u[0-9a-fA-F]{4})|[^"\\\\]))*"`
+  return `<${tag}>(?:${noNestedTag}(?:${jsonString}|[^"\`]))*?</${tag}>`
+}
 
-/**
- * One left-to-right pass over the two things that can own a backtick: an inline
- * code span, and a tag with a stray backtick pressed against it.
- *
- * ONE pass is the design. Two separate passes each have to guess which backticks
- * belong together, and every previous arrangement of this file got a different
- * case wrong — a span two words away, a code fence, then a span sitting flush
- * against the tag. Here a span consumes its own delimiters as the scan reaches
- * them, so `` `config.json`<tag> `` keeps its pair without a special case.
- *
- * The trailing backtick is only taken when no further backtick follows on the
- * line; otherwise it is not a stray at all but the opener of the next span, and
- * `` <tag>`config.json` `` would lose that span's delimiter. A LEADING backtick
- * needs no such guard, because a backtick that closes a span is consumed as part
- * of that span. Of the two, only the trailing lookahead is pinned by a test —
- * swapping the alternatives changes behaviour only for a span that both opens
- * flush against a tag and closes elsewhere, which no fixture covers.
- */
-const CODE_SPAN_OR_FLANKED_TAG = new RegExp(
-  `\`[^\`\\n]*\`|\`?(${COMPLETE_TAG_SOURCE})(?:\`(?![^\`\\n]*\`))?`,
+const INLINE_CHIP_OR_DELIMITER = new RegExp(
+  `${chipTagPattern('workspace_resource')}|${chipTagPattern('source')}|\`|\\n`,
   'g'
 )
 
+interface OpenCodeSpan {
+  index: number
+  containsChip: boolean
+  touchesChip: boolean
+}
+
+/** Unwraps chip-bearing spans without borrowing delimiters from neighboring code or fences. */
 export function sanitizeChatDisplayContent(content: string): string {
-  return content
-    .replace(CODE_SPAN_OR_FLANKED_TAG, (match, tag?: string) => {
-      // A tag with stray backticks against it: keep the tag, drop the strays.
-      if (tag !== undefined) return tag
+  const removedDelimiters: number[] = []
+  let openSpan: OpenCodeSpan | null = null
+  let previousChipEnd = -1
+
+  const finishLine = () => {
+    if (openSpan?.touchesChip) removedDelimiters.push(openSpan.index)
+    openSpan = null
+  }
+
+  for (const token of content.matchAll(INLINE_CHIP_OR_DELIMITER)) {
+    const [value] = token
+    const index = token.index
+    if (value === '\n') {
+      finishLine()
+      previousChipEnd = -1
+    } else if (value === '`') {
+      if (openSpan) {
+        if (openSpan.containsChip) removedDelimiters.push(openSpan.index, index)
+        openSpan = null
+      } else {
+        openSpan = { index, containsChip: false, touchesChip: previousChipEnd === index }
+      }
+    } else {
+      if (openSpan) {
+        openSpan.containsChip = true
+        openSpan.touchesChip ||= index === openSpan.index + 1
+      }
+      previousChipEnd = index + value.length
+    }
+  }
+  finishLine()
 
-      // A code span. Unwrap it only when it genuinely holds a tag — the parser
-      // lifts the tag out either way, so leaving the delimiters would strand a
-      // pair of backticks around a hole. Anything else is someone else's span.
-      const inner = match.slice(1, -1)
-      return COMPLETE_INLINE_CHIP_TAG.test(inner) ? inner : match
-    })
-    .replace(HIDDEN_INLINE_REFERENCE_PATTERN, '')
+  const parts: string[] = []
+  let cursor = 0
+  for (const index of removedDelimiters) {
+    parts.push(content.slice(cursor, index))
+    cursor = index + 1
+  }
+  parts.push(content.slice(cursor))
+  return parts.join('').replace(HIDDEN_INLINE_REFERENCE_PATTERN, '')
 }

apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/scaling-test-helpers.ts

@@ -27,13 +27,16 @@ function fastest(run: (content: string) => void, content: string): number {
  * through at the single size it happens to sample. Quadratic costs ~16x for 4x
  * the input; linear costs ~4x.
  */
-export function scalingRatioOver4x(run: (content: string) => void): number {
+export function scalingRatioOver4x(
+  run: (content: string) => void,
+  buildContent: (times: number) => string = buildRepeatedTagMentions
+): number {
   // Warm up first — the JIT would otherwise charge the whole compile to the
   // small sample and flatter the ratio.
-  fastest(run, buildRepeatedTagMentions(2_000))
+  fastest(run, buildContent(2_000))
 
-  const small = fastest(run, buildRepeatedTagMentions(2_000))
-  const large = fastest(run, buildRepeatedTagMentions(8_000))
+  const small = fastest(run, buildContent(2_000))
+  const large = fastest(run, buildContent(8_000))
 
   return large / small
 }