feat(webhooks): dedup and custom ack configuration (#3525)

* feat(webhooks): dedup and custom ack configuration

* address review comments

* reject object typed idempotency key

Author: icecrasher321

apps/sim/blocks/blocks/generic_webhook.ts

@@ -18,6 +18,7 @@ export const GenericWebhookBlock: BlockConfig = {
   bestPractices: `
   - You can test the webhook by sending a request to the webhook URL. E.g. depending on authorization:  curl -X POST http://localhost:3000/api/webhooks/trigger/d8abcf0d-1ee5-4b77-bb07-b1e8142ea4e9 -H "Content-Type: application/json" -H "X-Sim-Secret: 1234" -d '{"message": "Test webhook trigger", "data": {"key": "v"}}'
   - Continuing example above, the body can be accessed in downstream block using dot notation. E.g. <webhook1.message> and <webhook1.data.key>
+  - To deduplicate incoming events, set the Deduplication Field to a dot-notation path of a unique field in the payload (e.g. "event.id"). Duplicate values within 7 days will be skipped.
   - Only use when there's no existing integration for the service with triggerAllowed flag set to true.
   `,
   subBlocks: [...getTrigger('generic_webhook').subBlocks],

apps/sim/executor/handlers/trigger/trigger-handler.ts

@@ -22,7 +22,7 @@ export class TriggerBlockHandler implements BlockHandler {
     }
 
     const existingState = ctx.blockStates.get(block.id)
-    if (existingState?.output && Object.keys(existingState.output).length > 0) {
+    if (existingState?.output) {
       return existingState.output
     }
 

apps/sim/lib/core/idempotency/service.ts

@@ -413,6 +413,7 @@ export class IdempotencyService {
       : undefined
 
     const webhookIdHeader =
+      normalizedHeaders?.['x-sim-idempotency-key'] ||
       normalizedHeaders?.['webhook-id'] ||
       normalizedHeaders?.['x-webhook-id'] ||
       normalizedHeaders?.['x-shopify-webhook-id'] ||

apps/sim/lib/webhooks/processor.ts

@@ -1049,7 +1049,7 @@ export async function queueWebhookExecution(
       }
     }
 
-    const headers = Object.fromEntries(request.headers.entries())
+    const { 'x-sim-idempotency-key': _, ...headers } = Object.fromEntries(request.headers.entries())
 
     // For Microsoft Teams Graph notifications, extract unique identifiers for idempotency
     if (
@@ -1067,9 +1067,20 @@ export async function queueWebhookExecution(
       }
     }
 
-    // Extract credentialId from webhook config
-    // Note: Each webhook now has its own credentialId (credential sets are fanned out at save time)
     const providerConfig = (foundWebhook.providerConfig as Record<string, any>) || {}
+
+    if (foundWebhook.provider === 'generic') {
+      const idempotencyField = providerConfig.idempotencyField as string | undefined
+      if (idempotencyField && body) {
+        const value = idempotencyField
+          .split('.')
+          .reduce((acc: any, key: string) => acc?.[key], body)
+        if (value !== undefined && value !== null && typeof value !== 'object') {
+          headers['x-sim-idempotency-key'] = String(value)
+        }
+      }
+    }
+
     const credentialId = providerConfig.credentialId as string | undefined
 
     // credentialSetId is a direct field on webhook table, not in providerConfig
@@ -1193,6 +1204,26 @@ export async function queueWebhookExecution(
       })
     }
 
+    if (foundWebhook.provider === 'generic' && providerConfig.responseMode === 'custom') {
+      const rawCode = Number(providerConfig.responseStatusCode) || 200
+      const statusCode = rawCode >= 100 && rawCode <= 599 ? rawCode : 200
+      const responseBody = (providerConfig.responseBody as string | undefined)?.trim()
+
+      if (!responseBody) {
+        return new NextResponse(null, { status: statusCode })
+      }
+
+      try {
+        const parsed = JSON.parse(responseBody)
+        return NextResponse.json(parsed, { status: statusCode })
+      } catch {
+        return new NextResponse(responseBody, {
+          status: statusCode,
+          headers: { 'Content-Type': 'text/plain' },
+        })
+      }
+    }
+
     return NextResponse.json({ message: 'Webhook processed' })
   } catch (error: any) {
     logger.error(`[${options.requestId}] Failed to queue webhook execution:`, error)

apps/sim/lib/workflows/comparison/compare.test.ts

@@ -433,7 +433,7 @@ describe('hasWorkflowChanged', () => {
       expect(hasWorkflowChanged(state1, state2)).toBe(true)
     })
 
-    it.concurrent('should detect subBlock type changes', () => {
+    it.concurrent('should ignore subBlock type changes', () => {
       const state1 = createWorkflowState({
         blocks: {
           block1: createBlock('block1', {
@@ -448,7 +448,7 @@ describe('hasWorkflowChanged', () => {
           }),
         },
       })
-      expect(hasWorkflowChanged(state1, state2)).toBe(true)
+      expect(hasWorkflowChanged(state1, state2)).toBe(false)
     })
 
     it.concurrent('should handle null/undefined subBlock values consistently', () => {

apps/sim/lib/workflows/comparison/normalize.ts

@@ -496,7 +496,14 @@ export function normalizeSubBlockValue(subBlockId: string, value: unknown): unkn
  * @returns SubBlock fields excluding value and is_diff
  */
 export function extractSubBlockRest(subBlock: Record<string, unknown>): Record<string, unknown> {
-  const { value: _v, is_diff: _sd, ...rest } = subBlock as SubBlockWithDiffMarker
+  const {
+    value: _v,
+    is_diff: _sd,
+    type: _type,
+    ...rest
+  } = subBlock as SubBlockWithDiffMarker & {
+    type?: unknown
+  }
   return rest
 }
 

apps/sim/triggers/generic/webhook.ts

@@ -49,6 +49,49 @@ export const genericWebhookTrigger: TriggerConfig = {
       required: false,
       mode: 'trigger',
     },
+    {
+      id: 'idempotencyField',
+      title: 'Deduplication Field (Optional)',
+      type: 'short-input',
+      placeholder: 'e.g. event.id',
+      description:
+        'Dot-notation path to a unique field in the payload for deduplication. If the same value is seen within 7 days, the duplicate webhook will be skipped.',
+      required: false,
+      mode: 'trigger',
+    },
+    {
+      id: 'responseMode',
+      title: 'Acknowledgement',
+      type: 'dropdown',
+      options: [
+        { label: 'Default', id: 'default' },
+        { label: 'Custom', id: 'custom' },
+      ],
+      defaultValue: 'default',
+      mode: 'trigger',
+    },
+    {
+      id: 'responseStatusCode',
+      title: 'Response Status Code',
+      type: 'short-input',
+      placeholder: '200 (default)',
+      description:
+        'HTTP status code (100–599) to return to the webhook caller. Defaults to 200 if empty or invalid.',
+      required: false,
+      mode: 'trigger',
+      condition: { field: 'responseMode', value: 'custom' },
+    },
+    {
+      id: 'responseBody',
+      title: 'Response Body',
+      type: 'code',
+      language: 'json',
+      placeholder: '{"ok": true}',
+      description: 'JSON body to return to the webhook caller. Leave empty for no body.',
+      required: false,
+      mode: 'trigger',
+      condition: { field: 'responseMode', value: 'custom' },
+    },
     {
       id: 'inputFormat',
       title: 'Input Format',
@@ -76,7 +119,7 @@ export const genericWebhookTrigger: TriggerConfig = {
         'The webhook will receive any HTTP method (GET, POST, PUT, DELETE, etc.).',
         'All request data (headers, body, query parameters) will be available in your workflow.',
         'If authentication is enabled, include the token in requests using either the custom header or "Authorization: Bearer TOKEN".',
-        'Common fields like "event", "id", and "data" will be automatically extracted from the payload when available.',
+        'To deduplicate incoming events, set the Deduplication Field to the dot-notation path of a unique identifier in the payload (e.g. "event.id"). Duplicate values within 7 days will be skipped.',
       ]
         .map(
           (instruction, index) =>