From d706ea256737b331a10386be430ac9bfc021a4fd Mon Sep 17 00:00:00 2001
From: Uzair Khan <uzairkhanuk3215@gmail.com>
Date: Mon, 4 Nov 2024 21:04:47 +0530
Subject: [PATCH] feat(config): update vector config (#6365)

---
 config/vector.yaml | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/config/vector.yaml b/config/vector.yaml
index 3f0709ae03c..4b801935ae5 100644
--- a/config/vector.yaml
+++ b/config/vector.yaml
@@ -1,5 +1,16 @@
 acknowledgements:
   enabled: true
+enrichment_tables:
+  sdk_map:
+    type: file
+    file:
+      path: /etc/vector/config/sdk_map.csv
+      encoding:
+        type: csv
+    schema:
+      publishable_key: string
+      merchant_id: string
+
 
 api:
   enabled: true
@@ -87,6 +98,21 @@ transforms:
     key_field: "{{ .payment_id }}{{ .merchant_id }}"
     threshold: 1000
     window_secs: 60
+  
+  amend_sdk_logs:
+    type: remap
+    inputs:
+      - sdk_transformed
+    source: |
+      .before_transform = now()
+
+      merchant_id = .merchant_id
+      row = get_enrichment_table_record!("sdk_map", { "publishable_key" : merchant_id }, case_sensitive: true) 
+      .merchant_id = row.merchant_id
+
+      .after_transform = now()
+
+  
 
 sinks:
   opensearch_events_1:
@@ -110,6 +136,9 @@ sinks:
         - offset
         - partition
         - topic
+        - clickhouse_database
+        - last_synced
+        - sign_flag
     bulk:
       index: "vector-{{ .topic }}"
 
@@ -134,6 +163,9 @@ sinks:
         - offset
         - partition
         - topic
+        - clickhouse_database
+        - last_synced
+        - sign_flag
     bulk:
       # Add a date suffixed index for better grouping
       index: "vector-{{ .topic }}-%Y-%m-%d"
@@ -224,7 +256,7 @@ sinks:
         - "path"
         - "source_type"
     inputs:
-      - "sdk_transformed"
+      - "amend_sdk_logs"
     bootstrap_servers: kafka0:29092
     topic: hyper-sdk-logs
     key_field: ".merchant_id"