FEAT: Support nested and multiple conversion markers

doc/gui/0_gui.md

@@ -131,14 +131,65 @@ you can edit it directly before applying it.
 
 To convert only part of a text value, select it and click **Convert selection only**.
 This wraps the selection in `⟪` and `⟫`. The next converter transforms only the marked
-regions and removes their markers, preserving everything outside them. Marked regions
-have a colored highlight while their markers stay visible. Later stages convert the
-whole result unless you select another region. Multiple and multiline
-regions are supported; unmatched and nested regions are rejected. Empty regions
-pass an empty string to the converter. Partial
+regions and removes their markers, preserving everything outside them. Multiple,
+multiline, empty, and nested regions are supported. Each stage transforms **all
+innermost regions** and removes only their marker pairs. Outer markers remain for
+later stages. Marked regions have a colored highlight while every marker stays visible.
+You can select text inside a marked region or select one or more complete regions
+to add an outer pair. A selection that crosses only one boundary of an existing pair
+is rejected. Unmatched markers must be corrected before adding a region or converting.
+Empty regions pass an empty string to the converter. Partial
 conversion requires text input and text output. Without markers, converters retain
 their normal whole-value behavior, including media conversions.
 
+For a **Translate to French -> Base64 -> ROT13** pipeline, wrap each region three times:
+
+```text
+Decode this recursively: ⟪⟪⟪Hello⟫⟫⟫ and ⟪⟪⟪Goodbye⟫⟫⟫
+```
+
+If translation returns `Bonjour` and `Au revoir`, the regions change as follows:
+
+| Stage | First region | Second region |
+|---|---|---|
+| Translate to French | `⟪⟪Bonjour⟫⟫` | `⟪⟪Au revoir⟫⟫` |
+| Base64 | `⟪Qm9uam91cg==⟫` | `⟪QXUgcmV2b2ly⟫` |
+| ROT13 | `Dz9hnz91pt==` | `DKHtpzI2o2yl` |
+
+`Decode this recursively:` and ` and ` stay unchanged through these three stages.
+Regions can have different depths. A region with no markers left stays unchanged
+while other marked regions are selected. When no markers remain anywhere, any later
+converter transforms the whole value. If the pipeline ends with outer markers still
+present, those markers remain in the final value.
+
+For a `SelectiveTextConverter` with `TokenSelectionStrategy`, setting
+`preserve_tokens=True` keeps each converted region's marker pair for the next stage.
+That stage does not consume a marker layer. The default, `preserve_tokens=False`,
+consumes the innermost pairs as described above.
+Python callers can request the same behavior on ordinary converters with
+`convert_tokens_async(..., keep_tokens=True)`. Without markers, this wraps the
+whole text result; it does not wrap non-text outputs.
+Native token-selection wrappers nested with the same markers share one selection:
+if either preserves tokens, they retain one pair instead of adding duplicate pairs.
+Explicit nested marker pairs in the input remain intact.
+
+API clients can set non-empty `start_token` and `end_token` strings on converter
+preview and message requests, including queued sends. The same settings control
+request and response converter pipelines. For example, use
+`start_token="<|pyrit_start_8f3a|>"` and `end_token="<|pyrit_end_8f3a|>"`
+to select ASCII-marked regions:
+
+```text
+<|pyrit_start_8f3a|>hello<|pyrit_end_8f3a|>
+```
+
+Use the same settings on token-based `SelectiveTextConverter` instances in the
+pipeline. Other marker characters stay literal. Omitting these fields retains the
+Unicode defaults; the GUI selection button still inserts those defaults.
+Choose markers that are unlikely to appear in prompts or replies. If response
+converters are configured, an unmatched marker in a reply raises before that reply
+is stored. Longer markers reduce accidental matches but do not eliminate them.
+
 Click **Add converted value** to apply the final result, then **Send**. The exact
 applied value is sent and stored alongside the unchanged original; the backend does
 not rerun the pipeline. The exact ordered list of applied converters is retained

frontend/src/components/Chat/ConverterPanel.test.tsx

@@ -1012,4 +1012,38 @@ describe('ConverterPanel', () => {
     expect(screen.getByRole('textbox', { name: 'Stage 1 output - Text' })).toHaveValue('first')
     expect(screen.getByRole('textbox', { name: 'Stage 2 output - Text' })).toHaveValue('last')
   })
+
+  it('keeps nested multiple-region markers when continuing from an edited stage', async () => {
+    const original = 'keep \u27ea\u27ea\u27eaone\u27eb\u27eb\u27eb and \u27ea\u27ea\u27eatwo\u27eb\u27eb\u27eb'
+    const first = 'keep \u27ea\u27eab25l\u27eb\u27eb and \u27ea\u27eadHdv\u27eb\u27eb'
+    const final = 'keep \u27eaYjI1bA==\u27eb and \u27eaZEhkdg==\u27eb'
+    mockedConvertersApi.previewConversion
+      .mockResolvedValueOnce(makePreviewResponse(['base64-default', 'base64-default'], [first, final], original))
+      .mockResolvedValueOnce(makePreviewResponse(['base64-default'], [`${final} edited`], `${first} edited`))
+    const user = userEvent.setup()
+    renderPanel({ previewText: original })
+    await screen.findByTestId('converter-panel-list')
+    await selectConverter('base64-default')
+    await selectConverter('base64-default')
+    await user.click(screen.getByRole('button', { name: 'Convert', exact: true }))
+
+    expect(mockedConvertersApi.previewConversion).toHaveBeenLastCalledWith({
+      original_value: original, original_value_data_type: 'text',
+      converter_ids: ['base64-default', 'base64-default'],
+    })
+    const intermediate = screen.getByRole('textbox', { name: 'Stage 1 output - Text' })
+    expect(intermediate).toHaveValue(first)
+    await user.type(intermediate, ' edited')
+    expect(screen.queryByRole('textbox', { name: 'Stage 2 output - Text' })).not.toBeInTheDocument()
+    await user.click(screen.getByRole('button', { name: 'Convert Text from stage 2 to end' }))
+
+    expect(mockedConvertersApi.previewConversion).toHaveBeenLastCalledWith({
+      original_value: `${first} edited`, original_value_data_type: 'text',
+      converter_ids: ['base64-default'],
+    })
+    expect(screen.getByRole('textbox', { name: 'Stage 2 output - Text' })).toHaveValue(`${final} edited`)
+    await user.click(screen.getByRole('button', { name: 'Add converted value' }))
+    expect(JSON.parse(screen.getByTestId('applied-conversions').textContent ?? '{}').text.convertedValue)
+      .toBe(`${final} edited`)
+  })
 })

frontend/src/components/Chat/ConverterPanel/ConversionTextEditor.test.tsx

@@ -55,7 +55,35 @@ describe('ConversionTextEditor', () => {
     expect(screen.getAllByTestId('conversion-marked-region')[0].textContent).toBe('\u27eaone\ntwo\u27eb')
   })
 
-  it.each([[1, 4], [0, 5], [3, 8]])('rejects a selection overlapping markers (%s, %s)', async (start: number, end: number) => {
+  it.each([
+    ['\u27eaone\u27eb rest', 1, 4, '\u27ea\u27eaone\u27eb\u27eb rest'],
+    ['\u27eaone\u27eb rest', 0, 5, '\u27ea\u27eaone\u27eb\u27eb rest'],
+    ['\u27eaone\u27eb rest', 2, 3, '\u27eao\u27ean\u27ebe\u27eb rest'],
+    ['\u27ea\u27eaone\u27eb\u27eb rest', 1, 6, '\u27ea\u27ea\u27eaone\u27eb\u27eb\u27eb rest'],
+    ['\u27ea\u27eaone\u27eb\u27eb rest', 2, 5, '\u27ea\u27ea\u27eaone\u27eb\u27eb\u27eb rest'],
+    ['\u27eaone\u27eb and \u27eatwo\u27eb', 0, 15, '\u27ea\u27eaone\u27eb and \u27eatwo\u27eb\u27eb'],
+    ['\u27eaone\u27eb\u27eatwo\u27eb', 0, 10, '\u27ea\u27eaone\u27eb\u27eatwo\u27eb\u27eb'],
+    ['\u27eaone\r\ntwo\u27eb rest', 1, 8, '\u27ea\u27eaone\r\ntwo\u27eb\u27eb rest'],
+  ])('allows a nested selection in %j (%s, %s)', async (
+    value: string, start: number, end: number, expected: string,
+  ) => {
+    const user = userEvent.setup()
+    render(<TestWrapper initial={value} />)
+    const editor = screen.getByRole('textbox', { name: 'Working input' })
+    if (!(editor instanceof HTMLTextAreaElement)) throw new Error('Expected textarea')
+    await user.click(editor)
+    editor.setSelectionRange(start, end)
+    fireEvent.select(editor)
+    await user.click(screen.getByRole('button', { name: 'Convert selection only in Working input' }))
+
+    expect(editor).toHaveValue(expected.replace(/\r\n?/g, '\n'))
+    expect(screen.queryByRole('alert')).not.toBeInTheDocument()
+    expect(editor).toHaveFocus()
+  })
+
+  it.each([[0, 4], [1, 5], [3, 8]])('rejects a selection crossing one marker boundary (%s, %s)', async (
+    start: number, end: number,
+  ) => {
     const user = userEvent.setup()
     render(<TestWrapper initial={'\u27eaone\u27eb rest'} />)
     const editor = screen.getByRole('textbox', { name: 'Working input' })
@@ -65,15 +93,58 @@ describe('ConversionTextEditor', () => {
       { keys: '[/MouseLeft]' },
     ])
     await user.click(screen.getByRole('button', { name: 'Convert selection only in Working input' }))
-    expect(screen.getByText('Select text outside an existing marked region.')).toBeInTheDocument()
+    expect(screen.getByText('Select text inside a marked region or include the complete region.')).toBeInTheDocument()
     expect(editor).toHaveValue('\u27eaone\u27eb rest')
   })
 
-  it('does not highlight an unmatched marker', () => {
+  it('rejects a selection crossing a nested boundary', async () => {
+    const user = userEvent.setup()
+    const value = '\u27eaouter \u27eainner\u27eb tail\u27eb'
+    render(<TestWrapper initial={value} />)
+    const editor = screen.getByRole('textbox', { name: 'Working input' })
+    if (!(editor instanceof HTMLTextAreaElement)) throw new Error('Expected textarea')
+    await user.click(editor)
+    editor.setSelectionRange(1, 13)
+    fireEvent.select(editor)
+    await user.click(screen.getByRole('button', { name: 'Convert selection only in Working input' }))
+
+    expect(screen.getByText('Select text inside a marked region or include the complete region.')).toBeInTheDocument()
+    expect(editor).toHaveValue(value)
+  })
+
+  it('does not add a region to unmatched markers', async () => {
+    const user = userEvent.setup()
     render(<TestWrapper initial={'plain \u27eaunfinished'} />)
+    const editor = screen.getByRole('textbox', { name: 'Working input' })
+    await user.pointer([
+      { target: editor, offset: 0, keys: '[MouseLeft>]' },
+      { target: editor, offset: 5 },
+      { keys: '[/MouseLeft]' },
+    ])
+    await user.click(screen.getByRole('button', { name: 'Convert selection only in Working input' }))
+
+    expect(screen.getByText('Match all start and end markers before adding a marked region.')).toBeInTheDocument()
+    expect(editor).toHaveValue('plain \u27eaunfinished')
+  })
+
+  it.each([
+    'plain \u27eaunfinished',
+    '\u27ebreversed\u27ea',
+    '\u27eaouter \u27eainner\u27eb',
+    '\u27eavalid\u27eb then \u27eb',
+  ])('does not highlight malformed regions in %j', (value: string) => {
+    render(<TestWrapper initial={value} />)
 
     expect(screen.queryByTestId('conversion-marked-region')).not.toBeInTheDocument()
-    expect(screen.getByTestId('conversion-highlight-layer')).toHaveTextContent('plain \u27eaunfinished')
+    expect(screen.getByTestId('conversion-highlight-layer').textContent).toBe(value)
+  })
+
+  it('highlights complete outer regions and keeps every nested marker visible', () => {
+    const regions = ['\u27eaouter \u27eainner\u27eb tail\u27eb', '\u27ea\u27ea\u27eatwo\u27eb\u27eb\u27eb']
+    render(<TestWrapper initial={`keep ${regions[0]} and ${regions[1]} after`} />)
+
+    expect(screen.getAllByTestId('conversion-marked-region').map((element) => element.textContent)).toEqual(regions)
+    expect(screen.getByTestId('conversion-highlight-layer').textContent).toBe(`keep ${regions[0]} and ${regions[1]} after`)
   })
 
   it.each([

frontend/src/components/Chat/ConverterPanel/ConversionTextEditor.tsx

@@ -8,6 +8,27 @@ import { useConverterPanelStyles } from './ConverterPanel.styles'
 const START_MARKER = '\u27ea'
 const END_MARKER = '\u27eb'
 
+interface MarkerRegion {
+  readonly start: number
+  readonly end: number
+  readonly depth: number
+}
+
+function parseMarkedRegions(value: string): MarkerRegion[] | null {
+  const starts: number[] = []
+  const regions: MarkerRegion[] = []
+  for (let index = 0; index < value.length; index++) {
+    if (value[index] === START_MARKER) {
+      starts.push(index)
+    } else if (value[index] === END_MARKER) {
+      const start = starts.pop()
+      if (start === undefined) return null
+      regions.push({ start, end: index + END_MARKER.length, depth: starts.length })
+    }
+  }
+  return starts.length ? null : regions
+}
+
 function sourceOffset(value: string, textareaOffset: number): number {
   let offset = 0
   for (let index = 0; index < textareaOffset && offset < value.length; index++) {
@@ -18,30 +39,31 @@ function sourceOffset(value: string, textareaOffset: number): number {
 }
 
 function highlightMarkedRegions(value: string): ReactNode[] {
+  const regions = parseMarkedRegions(value)
+  if (!regions) return [value]
   const parts: ReactNode[] = []
   let cursor = 0
-  while (cursor < value.length) {
-    const start = value.indexOf(START_MARKER, cursor)
-    if (start < 0) {
-      parts.push(value.slice(cursor))
-      break
-    }
-    const end = value.indexOf(END_MARKER, start + START_MARKER.length)
-    if (end < 0) {
-      parts.push(value.slice(cursor))
-      break
-    }
+  for (const { start, end, depth } of regions) {
+    if (depth !== 0) continue
     parts.push(value.slice(cursor, start))
     parts.push(
       <mark key={`${start}-${end}`} data-testid="conversion-marked-region">
-        {value.slice(start, end + END_MARKER.length)}
+        {value.slice(start, end)}
       </mark>,
     )
-    cursor = end + END_MARKER.length
+    cursor = end
   }
+  parts.push(value.slice(cursor))
   return parts
 }
 
+function selectionCrossesRegion(start: number, end: number, region: MarkerRegion): boolean {
+  const outside = end <= region.start || start >= region.end
+  const inside = start >= region.start + START_MARKER.length && end <= region.end - END_MARKER.length
+  const enclosing = start <= region.start && end >= region.end
+  return !outside && !inside && !enclosing
+}
+
 interface ConversionTextEditorProps {
   readonly value: string
   readonly label: string
@@ -78,11 +100,13 @@ export default function ConversionTextEditor({
     const selected = value.slice(start, end)
     const before = value.slice(0, start)
     if (!selected || selection.value !== value) return
-    if (
-      selected.includes(START_MARKER) || selected.includes(END_MARKER)
-      || before.lastIndexOf(START_MARKER) > before.lastIndexOf(END_MARKER)
-    ) {
-      setError('Select text outside an existing marked region.')
+    const regions = parseMarkedRegions(value)
+    if (!regions) {
+      setError('Match all start and end markers before adding a marked region.')
+      return
+    }
+    if (regions.some((region) => selectionCrossesRegion(start, end, region))) {
+      setError('Select text inside a marked region or include the complete region.')
       return
     }
     setError(null)
@@ -132,7 +156,7 @@ export default function ConversionTextEditor({
           className={styles.selectionButton}
           disabled={selection.start === selection.end || selection.value !== value}
           aria-label={`Convert selection only in ${label}`}
-          title="Convert only the marked text in the next stage. Later stages use the whole result."
+          title="Each stage converts innermost marked regions and removes one layer. With no markers, it converts the whole result."
           onClick={markSelection}
         >
           Convert selection only

frontend/src/types/index.ts

@@ -394,6 +394,8 @@ export interface ConverterPreviewRequest {
   original_value: string
   converter_ids: string[]
   original_value_data_type?: string
+  start_token?: string
+  end_token?: string
 }
 
 /** One converter stage of a `/converters/preview` pipeline run. */
@@ -668,6 +670,8 @@ export interface AddMessageRequest extends MessageRequest {
   converter_ids?: string[]
   request_converter_configurations?: ConverterConfigurationRequest[]
   response_converter_configurations?: ConverterConfigurationRequest[]
+  start_token?: string
+  end_token?: string
   target_conversation_id: string
 }
 

pyrit/backend/models/attacks.py

@@ -700,6 +700,12 @@ class AddMessageRequest(MessageRequest):
         max_length=MAX_ITEMS,
         description="Ordered registry-backed converter pipelines to apply to the response.",
     )
+    start_token: str = Field(
+        default="⟪", min_length=1, description="Opening marker for request and response converter pipelines"
+    )
+    end_token: str = Field(
+        default="⟫", min_length=1, description="Closing marker for request and response converter pipelines"
+    )
     target_conversation_id: IdentifierStr = Field(
         ...,
         description="The conversation_id to store and send messages under. "

pyrit/backend/models/converters.py

@@ -119,6 +119,8 @@ class ConverterPreviewRequest(BaseModel):
     original_value: str = Field(..., description="Text to convert")
     original_value_data_type: PromptDataType = Field(default="text", description="Data type of original value")
     converter_ids: list[IdentifierStr] = Field(..., max_length=MAX_ITEMS, description="Converter instance IDs to apply")
+    start_token: str = Field(default="⟪", min_length=1, description="Opening marker for selected text regions")
+    end_token: str = Field(default="⟫", min_length=1, description="Closing marker for selected text regions")
 
 
 class ConverterPreviewResponse(BaseModel):

pyrit/backend/services/converter_service.py

@@ -216,7 +216,7 @@ async def preview_conversion_async(self, *, request: ConverterPreviewRequest) ->
 
         For non-text data types (image_path, audio_path, etc.), persists base64 data
         to a temporary file so converters can operate on file paths. Marked text
-        regions are converted by the next converter, which consumes their delimiters.
+        regions use the request's delimiter settings for every stage.
 
         Returns:
             ConverterPreviewResponse with step-by-step conversion results.
@@ -240,7 +240,11 @@ async def preview_conversion_async(self, *, request: ConverterPreviewRequest) ->
 
         converters = self._gather_converters(converter_ids=request.converter_ids)
         steps, final_value, final_type = await self._apply_converters_async(
-            converters=converters, initial_value=original_value, initial_type=data_type
+            converters=converters,
+            initial_value=original_value,
+            initial_type=data_type,
+            start_token=request.start_token,
+            end_token=request.end_token,
         )
 
         return ConverterPreviewResponse(
@@ -423,6 +427,8 @@ async def _apply_converters_async(
         converters: list[tuple[str, str, Any]],
         initial_value: str,
         initial_type: PromptDataType,
+        start_token: str = "⟪",
+        end_token: str = "⟫",
     ) -> tuple[list[PreviewStep], str, PromptDataType]:
         """
         Collect preview steps using the normalizer's conversion-only path.
@@ -440,7 +446,7 @@ async def _apply_converters_async(
             not_in_memory=True,
         )
         message = piece.to_message()
-        normalizer = PromptNormalizer()
+        normalizer = PromptNormalizer(start_token=start_token, end_token=end_token)
         steps: list[PreviewStep] = []
 
         for conv_id, conv_type, conv_obj in converters: