fix(search): safe result links, script-aware term matching, and honest header stripping

A result links only to an http(s) URL; term matching judges word edges
by the surrounding characters instead of ASCII \b and strips quotes
from a phrase, and the same matcher bolds the passage; a chunk that is
nothing but fields keeps its content; the agent leaves unknown optional
citation fields out and keeps the tool's published result count; route
errors log the wrapped cause and Postgres code.

Author: waleedlatif1

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx

@@ -14,7 +14,10 @@ import {
   SOURCE_ROW_MARK_CLASSES,
   SourceCard,
 } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card'
-import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags'
+import {
+  isHttpUrl,
+  type SourceTagData,
+} from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags'
 import { isIndexing } from '@/app/workspace/[workspaceId]/home/components/search-sources'
 import {
   resourceUrlKeys,
@@ -76,11 +79,12 @@ export function indexingSourceNames(
 
 /**
  * A result as the source card renders it: the row's second line names the
- * source app, or the knowledge base for an upload. A document without a
- * source URL cannot be opened.
+ * source app, or the knowledge base for an upload. A document without an
+ * http(s) source URL cannot be opened, and a connector-supplied value of any
+ * other scheme is never handed to the browser as a link.
  */
 function toSource(result: WorkspaceKnowledgeSearchResult, query: string): SourceTagData | null {
-  if (!result.sourceUrl) return null
+  if (!isHttpUrl(result.sourceUrl)) return null
   return {
     url: result.sourceUrl,
     title: result.documentName ?? undefined,

apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx

@@ -7,6 +7,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { formatDate } from '@sim/utils/formatting'
 import { faviconUrl } from '@/lib/core/utils/favicon'
+import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet'
 import {
   externalLinkHostname,
   handleExternalLinkClick,
@@ -21,8 +22,6 @@ import { BrandIcon } from '@/blocks/brand-icon'
 
 const logger = createLogger('SourceCard')
 
-/** Query terms shorter than this are too common to bold. */
-const MIN_HIGHLIGHT_TERM_LENGTH = 3
 /** How long the copied state shows on the copy-link action. */
 const COPIED_FEEDBACK_MS = 1_500
 
@@ -37,35 +36,27 @@ export const SOURCE_ROW_CLASSES =
 /** The 16px mark slot, nudged to centre on the title's first line. */
 export const SOURCE_ROW_MARK_CLASSES = cn(chipIconSlotClass, 'mt-[3px]')
 
-function escapeRegExp(value: string): string {
-  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-}
-
 /**
  * The snippet with every query term in bold, so the reader sees why the
- * document matched. Terms are matched as whole words, case-insensitively.
+ * document matched. Terms are matched as whole words in any script,
+ * case-insensitively, by the same rule the snippet was centred with.
  */
 export function highlightTerms(text: string, query: string | undefined): ReactNode {
-  const terms = [
-    ...new Set(
-      (query ?? '')
-        .split(/\s+/)
-        .map((term) => term.trim())
-        .filter((term) => term.length >= MIN_HIGHLIGHT_TERM_LENGTH)
-    ),
-  ]
-  if (terms.length === 0) return text
-  const pattern = new RegExp(`\\b(${terms.map(escapeRegExp).join('|')})\\b`, 'gi')
-  const parts = text.split(pattern)
-  return parts.map((part, index) =>
-    index % 2 === 1 ? (
-      <strong key={index} className='font-medium text-[var(--text-primary)]'>
-        {part}
+  const matches = findTermMatches(text, queryTerms(query))
+  if (matches.length === 0) return text
+  const parts: ReactNode[] = []
+  let cursor = 0
+  for (const match of matches) {
+    if (match.index > cursor) parts.push(text.slice(cursor, match.index))
+    parts.push(
+      <strong key={match.index} className='font-medium text-[var(--text-primary)]'>
+        {text.slice(match.index, match.index + match.length)}
       </strong>
-    ) : (
-      part
     )
-  )
+    cursor = match.index + match.length
+  }
+  if (cursor < text.length) parts.push(text.slice(cursor))
+  return parts
 }
 
 function parseUpdatedAt(value: string | undefined): Date | null {

apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts

@@ -25,6 +25,7 @@ export {
   CredentialDisplay,
   credentialTagHasVisibleCard,
   formatCredentialSubmissionMessage,
+  isHttpUrl,
   PendingTagIndicator,
   parseCredentialSubmissionMessage,
   parseCredentialSubmissionProgress,

apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx

@@ -567,7 +567,7 @@ function isMothershipErrorTagData(value: unknown): value is MothershipErrorTagDa
  * a source. Parsed rather than pattern-matched so a malformed value such as
  * `https://?` — which a prefix check would accept — never becomes a dead link.
  */
-function isHttpUrl(value: unknown): value is string {
+export function isHttpUrl(value: unknown): value is string {
   if (typeof value !== 'string' || /\s/.test(value)) return false
   try {
     const url = new URL(value)

apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts

@@ -401,7 +401,7 @@ describe('manage_knowledge_base trusted application delegation', () => {
       workspaceId: 'workspace-paid',
       knowledgeBaseIds: [KNOWLEDGE_BASE.id],
       query: '{{KB_QUERY}}',
-      topK: 10,
+      topK: 5,
       resultSecretRegistry: registry,
     })
     expect(mockReadKnowledgeBase).not.toHaveBeenCalled()

apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts

@@ -59,14 +59,14 @@ import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-sec
 const logger = createLogger('KnowledgeBaseServerTool')
 
 /** Results a query returns unless the caller asks for a number. */
-const DEFAULT_QUERY_TOP_K = 10
+const DEFAULT_QUERY_TOP_K = 5
 /**
  * How the model cites a knowledge result in its reply. The `<source>` tag is
  * what the chat renders as a link back to the document, so a result without
  * a source URL is quoted by name instead.
  */
 const KNOWLEDGE_CITATION_INSTRUCTION =
-  'Cite each result you use inline, right after the sentence it supports, as <source>{"url":<sourceUrl>,"title":<documentName>,"siteName":<knowledgeBaseName>,"connectorType":<connectorType>,"snippet":<the sentence or two of content you relied on>,"updatedAt":<sourceModifiedAt>,"author":<author>}</source>; omit the tag for a result whose sourceUrl is null and name the document instead.'
+  'Cite each result you use inline, right after the sentence it supports, as <source>{"url":<sourceUrl>,"title":<documentName>,"siteName":<knowledgeBaseName>,"connectorType":<connectorType>,"snippet":<the sentence or two of content you relied on>,"updatedAt":<sourceModifiedAt>,"author":<author>}</source>; leave out any optional field whose value is null or unknown, and omit the tag for a result whose sourceUrl is null and name the document instead.'
 
 /**
  * Resolves an environment-variable reference passed as a connector API key.

apps/sim/lib/core/utils/with-route-handler.ts

@@ -1,5 +1,5 @@
 import { createLogger, runWithRequestContext } from '@sim/logger'
-import { getErrorMessage } from '@sim/utils/errors'
+import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors'
 import type { NextRequest } from 'next/server'
 import { NextResponse } from 'next/server'
 import { getRateLimitHeaders } from '@/lib/api/server/rate-limit-context'
@@ -87,6 +87,20 @@ function traceIdFromTraceparent(header: string | null | undefined): string | und
   return match[1]
 }
 
+/**
+ * What a wrapped error hides: a query failure from the database client carries
+ * the driver's reason and the Postgres code on its cause, and only the outer
+ * message names the query.
+ */
+function errorDetail(error: unknown): { cause?: string; code?: string } {
+  const cause = error instanceof Error && error.cause !== undefined ? error.cause : undefined
+  const code = getPostgresErrorCode(error)
+  return {
+    ...(cause !== undefined ? { cause: getErrorMessage(cause) } : {}),
+    ...(code ? { code } : {}),
+  }
+}
+
 /**
  * Wraps a Next.js API route handler with centralized error reporting.
  *
@@ -119,6 +133,7 @@ export function withRouteHandler<T>(
       } catch (error) {
         const duration = Date.now() - startTime
         const message = getErrorMessage(error, 'Unknown error')
+        const detail = errorDetail(error)
         if (request.signal.aborted) {
           logger.info('Client closed request', { duration, status: 499 })
           response = options.clientAbortResponse
@@ -132,7 +147,12 @@ export function withRouteHandler<T>(
         if (typedError) {
           const typedStatus = typedError.statusCode
           if (typedStatus >= 500) {
-            logger.error('Unhandled route error', { duration, status: typedStatus, error: message })
+            logger.error('Unhandled route error', {
+              duration,
+              status: typedStatus,
+              error: message,
+              ...detail,
+            })
           } else {
             logger.warn('Typed route error', { duration, status: typedStatus, error: message })
           }
@@ -144,13 +164,13 @@ export function withRouteHandler<T>(
         }
 
         if (options.unhandledErrorResponse) {
-          logger.error('Unhandled route error', { duration, error: message })
+          logger.error('Unhandled route error', { duration, error: message, ...detail })
           response = options.unhandledErrorResponse({ error, requestId })
           applyResponseHeaders(response, request, requestId)
           return response
         }
 
-        logger.error('Unhandled route error', { duration, error: message })
+        logger.error('Unhandled route error', { duration, error: message, ...detail })
         response = NextResponse.json({ error: 'Internal server error', requestId }, { status: 500 })
         applyResponseHeaders(response, request, requestId)
         return response

apps/sim/lib/knowledge/search/snippet.test.ts

@@ -3,9 +3,10 @@
  */
 import { describe, expect, it } from 'vitest'
 import {
+  findTermMatches,
   matchSnippet,
+  queryTerms,
   SNIPPET_LENGTH,
-  snippetTerms,
   stripLeadingHeaders,
 } from '@/lib/knowledge/search/snippet'
 
@@ -18,6 +19,10 @@ const EMAIL = [
   `${'Thanks for your patience. '.repeat(12)}The Volvo order shipped on Monday and the tracking number follows. ${'More text here. '.repeat(20)}`,
 ].join('\n')
 
+const EVENT = ['Title: Weekly sync', 'Organizer: Ada', 'When: Monday 9am', 'Where: Room 4'].join(
+  '\n'
+)
+
 describe('stripLeadingHeaders', () => {
   it('drops the header block a connector writes above an email body', () => {
     expect(stripLeadingHeaders(EMAIL).startsWith('\nThanks for your patience.')).toBe(true)
@@ -28,12 +33,35 @@ describe('stripLeadingHeaders', () => {
       'Plain prose: with a colon inside.'
     )
   })
+
+  it('keeps a chunk that is nothing but fields, such as a calendar event', () => {
+    expect(stripLeadingHeaders(EVENT)).toBe(EVENT)
+    expect(stripLeadingHeaders(`${EVENT}\n\n`)).toBe(`${EVENT}\n\n`)
+  })
 })
 
-describe('snippetTerms', () => {
+describe('queryTerms', () => {
   it('keeps distinct terms of three or more characters, longest first', () => {
-    expect(snippetTerms('the Volvo invoice is volvo')).toEqual(['invoice', 'Volvo', 'volvo', 'the'])
-    expect(snippetTerms(undefined)).toEqual([])
+    expect(queryTerms('the Volvo invoice is volvo')).toEqual(['invoice', 'Volvo', 'volvo', 'the'])
+    expect(queryTerms(undefined)).toEqual([])
+  })
+
+  it('strips the quotes and punctuation around a term', () => {
+    expect(queryTerms('"foo bar" (baz),')).toEqual(['foo', 'bar', 'baz'])
+  })
+})
+
+describe('findTermMatches', () => {
+  it('matches whole words in any script', () => {
+    expect(findTermMatches('Der Bericht über Zürich.', ['Zürich'])).toEqual([
+      { index: 17, length: 6 },
+    ])
+    expect(findTermMatches('Reports on Zürichsee.', ['Zürich'])).toEqual([])
+    expect(findTermMatches('東京の天気', ['天気'])).toEqual([{ index: 3, length: 2 }])
+  })
+
+  it('skips a hit glued to another word character', () => {
+    expect(findTermMatches('subvolvo volvo_x volvo', ['volvo'])).toEqual([{ index: 17, length: 5 }])
   })
 })
 
@@ -51,6 +79,12 @@ describe('matchSnippet', () => {
     expect(snippet.length).toBeLessThanOrEqual(SNIPPET_LENGTH + 2)
   })
 
+  it('centres on a quoted phrase and on a non-ASCII term', () => {
+    expect(matchSnippet(EMAIL, '"Volvo order"')).toContain('The Volvo order shipped')
+    const german = `${'Einleitung. '.repeat(30)}Die Lieferung nach Zürich ist unterwegs. ${'Mehr. '.repeat(30)}`
+    expect(matchSnippet(german, 'Zürich')).toContain('nach Zürich')
+  })
+
   it('falls back to the opening when no term appears in the chunk', () => {
     const snippet = matchSnippet(EMAIL, 'unrelated')
     expect(snippet.startsWith('Thanks for your patience.')).toBe(true)

apps/sim/lib/knowledge/search/snippet.ts

@@ -6,6 +6,14 @@ const LEAD_LENGTH = 90
 const MIN_TERM_LENGTH = 3
 /** `Key: value` lines a connector writes above an email or ticket body. */
 const HEADER_LINE = /^[A-Z][A-Za-z-]{1,15}: .*$/
+/**
+ * A character that continues a word, so a term touching one on either side is
+ * part of a longer word rather than a hit. Scripts written without spaces
+ * (Han, kana, Hangul, Thai) have no such edges, so their letters never
+ * disqualify a neighbouring match.
+ */
+const WORD_CHARACTER =
+  /(?![\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}\p{sc=Thai}])[\p{L}\p{N}_]/u
 
 function escapeRegExp(value: string): string {
   return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
@@ -15,28 +23,60 @@ function escapeRegExp(value: string): string {
  * The document text without the header block some connectors prefix (the
  * `Subject:` / `From:` / `To:` lines of an email): the title already says
  * what the subject is, and a snippet spent on the header never shows why the
- * document matched.
+ * document matched. Only a block the connector closed with a blank line
+ * counts, and only when a body follows it: a chunk that is nothing but
+ * `Key: value` fields, such as a calendar event, is the document.
  */
 export function stripLeadingHeaders(content: string): string {
   const lines = content.split('\n')
   let index = 0
   while (index < lines.length && HEADER_LINE.test(lines[index].trim())) index += 1
-  if (index === 0) return content
-  return lines.slice(index).join('\n')
+  if (index === 0 || index >= lines.length || lines[index].trim() !== '') return content
+  const body = lines.slice(index).join('\n')
+  return body.trim() ? body : content
 }
 
-/** The query's terms worth anchoring on, longest first so the most specific one wins. */
-export function snippetTerms(query: string | undefined): string[] {
+/**
+ * The query's terms worth matching, longest first so the most specific one
+ * wins: quotes and other search syntax around a term are not part of it.
+ */
+export function queryTerms(query: string | undefined): string[] {
   return [
     ...new Set(
       (query ?? '')
         .split(/\s+/)
-        .map((term) => term.trim())
+        .map((term) => term.replace(/^["'“”‘’(]+|["'“”‘’),.;:!?]+$/g, '').trim())
         .filter((term) => term.length >= MIN_TERM_LENGTH)
     ),
   ].sort((a, b) => b.length - a.length)
 }
 
+export interface TermMatch {
+  index: number
+  length: number
+}
+
+/**
+ * Where the query terms occur in the text as whole words, in order and without
+ * overlap. Word edges are judged by the characters around a hit rather than
+ * by `\b`, which knows only ASCII letters, so a term in any script still
+ * matches; a hit glued to another word character on either side is not a
+ * word and is skipped.
+ */
+export function findTermMatches(text: string, terms: readonly string[]): TermMatch[] {
+  if (terms.length === 0) return []
+  const pattern = new RegExp(terms.map(escapeRegExp).join('|'), 'giu')
+  const matches: TermMatch[] = []
+  for (const match of text.matchAll(pattern)) {
+    const before = text[match.index - 1]
+    const after = text[match.index + match[0].length]
+    if (before !== undefined && WORD_CHARACTER.test(before)) continue
+    if (after !== undefined && WORD_CHARACTER.test(after)) continue
+    matches.push({ index: match.index, length: match[0].length })
+  }
+  return matches
+}
+
 /**
  * The passage of a document a search result shows: a window around the first
  * query term found, the way a search page shows why a document matched, and
@@ -48,13 +88,8 @@ export function matchSnippet(content: string, query?: string): string {
   const flat = stripLeadingHeaders(content).replace(/\s+/g, ' ').trim()
   if (flat.length <= SNIPPET_LENGTH) return flat
 
-  let start = 0
-  for (const term of snippetTerms(query)) {
-    const match = new RegExp(`\\b${escapeRegExp(term)}\\b`, 'i').exec(flat)
-    if (!match) continue
-    start = Math.max(0, match.index - LEAD_LENGTH)
-    break
-  }
+  const first = findTermMatches(flat, queryTerms(query))[0]
+  let start = first ? Math.max(0, first.index - LEAD_LENGTH) : 0
   if (start > 0) {
     const boundary = flat.indexOf(' ', start)
     if (boundary !== -1 && boundary - start < LEAD_LENGTH) start = boundary + 1