adds new column addition strategy

nucleuscloud · Oct 31, 2024 · acfee14 · acfee14
1 parent 1745f83
commit acfee14
Show file tree

Hide file tree

Showing 9 changed files with 2,063 additions and 1,723 deletions.
diff --git a/backend/gen/go/protos/mgmt/v1alpha1/job.pb.go b/backend/gen/go/protos/mgmt/v1alpha1/job.pb.go
diff --git a/backend/protos/mgmt/v1alpha1/job.proto b/backend/protos/mgmt/v1alpha1/job.proto
@@ -130,10 +130,22 @@ message DynamoDBSourceTableOption {
 }
 
 message PostgresSourceConnectionOptions {
+  // @deprecated - Use new_column_addition_strategy instead
   bool halt_on_new_column_addition = 1;
   repeated PostgresSourceSchemaOption schemas = 2;
   string connection_id = 3 [(buf.validate.field).string.uuid = true];
   bool subset_by_foreign_key_constraints = 4;
+  NewColumnAdditionStrategy new_column_addition_strategy = 5;
+
+  message NewColumnAdditionStrategy {
+    oneof strategy {
+      // halt job if a new column is detected. This is equiavlent to the old halt_on_new_column_addition
+      bool halt_job = 1;
+      // neosync will automatically anonymize unmapped columns. View the docs page for details on this strategy
+      bool use_auto_map = 2;
+      // space for more discrete strategies in the future
+    }
+  }
 }
 
 message PostgresSourceSchemaOption {

diff --git a/docs/openapi/mgmt/v1alpha1/job.openapi.yaml b/docs/openapi/mgmt/v1alpha1/job.openapi.yaml
@@ -3767,6 +3767,7 @@ components:
         haltOnNewColumnAddition:
           type: boolean
           title: halt_on_new_column_addition
+          description: '@deprecated - Use new_column_addition_strategy instead'
         schemas:
           type: array
           items:
@@ -3779,8 +3780,48 @@ components:
         subsetByForeignKeyConstraints:
           type: boolean
           title: subset_by_foreign_key_constraints
+        newColumnAdditionStrategy:
+          allOf:
+            - title: new_column_addition_strategy
+            - $ref: '#/components/schemas/mgmt.v1alpha1.PostgresSourceConnectionOptions.NewColumnAdditionStrategy'
       title: PostgresSourceConnectionOptions
       additionalProperties: false
+    mgmt.v1alpha1.PostgresSourceConnectionOptions.NewColumnAdditionStrategy:
+      type: object
+      allOf:
+        - anyOf:
+            - required:
+                - haltJob
+            - required:
+                - useAutoMap
+            - not:
+                anyOf:
+                  - required:
+                      - haltJob
+                  - required:
+                      - useAutoMap
+      anyOf:
+        - required:
+            - haltJob
+        - required:
+            - useAutoMap
+        - not:
+            anyOf:
+              - required:
+                  - haltJob
+              - required:
+                  - useAutoMap
+      properties:
+        haltJob:
+          type: boolean
+          title: halt_job
+          description: halt job if a new column is detected. This is equiavlent to the old halt_on_new_column_addition
+        useAutoMap:
+          type: boolean
+          title: use_auto_map
+          description: neosync will automatically anonymize unmapped columns. View the docs page for details on this strategy space for more discrete strategies in the future
+      title: NewColumnAdditionStrategy
+      additionalProperties: false
     mgmt.v1alpha1.PostgresSourceSchemaOption:
       type: object
       properties:

diff --git a/docs/openapi/neosync.mgmt.v1alpha1.yaml b/docs/openapi/neosync.mgmt.v1alpha1.yaml
@@ -10344,6 +10344,7 @@ components:
         haltOnNewColumnAddition:
           type: boolean
           title: halt_on_new_column_addition
+          description: '@deprecated - Use new_column_addition_strategy instead'
         schemas:
           type: array
           items:
@@ -10356,8 +10357,54 @@ components:
         subsetByForeignKeyConstraints:
           type: boolean
           title: subset_by_foreign_key_constraints
+        newColumnAdditionStrategy:
+          allOf:
+            - title: new_column_addition_strategy
+            - $ref: >-
+                #/components/schemas/mgmt.v1alpha1.PostgresSourceConnectionOptions.NewColumnAdditionStrategy
       title: PostgresSourceConnectionOptions
       additionalProperties: false
+    mgmt.v1alpha1.PostgresSourceConnectionOptions.NewColumnAdditionStrategy:
+      type: object
+      allOf:
+        - anyOf:
+            - required:
+                - haltJob
+            - required:
+                - useAutoMap
+            - not:
+                anyOf:
+                  - required:
+                      - haltJob
+                  - required:
+                      - useAutoMap
+      anyOf:
+        - required:
+            - haltJob
+        - required:
+            - useAutoMap
+        - not:
+            anyOf:
+              - required:
+                  - haltJob
+              - required:
+                  - useAutoMap
+      properties:
+        haltJob:
+          type: boolean
+          title: halt_job
+          description: >-
+            halt job if a new column is detected. This is equiavlent to the old
+            halt_on_new_column_addition
+        useAutoMap:
+          type: boolean
+          title: use_auto_map
+          description: >-
+            neosync will automatically anonymize unmapped columns. View the docs
+            page for details on this strategy space for more discrete strategies
+            in the future
+      title: NewColumnAdditionStrategy
+      additionalProperties: false
     mgmt.v1alpha1.PostgresSourceSchemaOption:
       type: object
       properties: