Skip to content

Commit

Permalink
adds support for configuring batch, maxinflight for sql dest connections
Browse files Browse the repository at this point in the history
  • Loading branch information
@nickzelei
nickzelei committed Oct 31, 2024
1 parent 27edd03 commit 451a0c5
Show file tree
Hide file tree
Showing 13 changed files with 1,842 additions and 1,349 deletions.
2,307 changes: 1,195 additions & 1,112 deletions backend/gen/go/protos/mgmt/v1alpha1/job.pb.go
View file

Large diffs are not rendered by default.

14 changes: 13 additions & 1 deletion backend/protos/mgmt/v1alpha1/job.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,10 @@ message PostgresDestinationConnectionOptions {
PostgresOnConflictConfig on_conflict = 3;
// Insert all valid records, skipping any that violate foreign key constraints.
bool skip_foreign_key_violations = 4;
// Configure batching options to handle how much data is sent to your database at once.
BatchConfig batch = 5;
// Determines the maximum number of parallel batched inserts.
optional uint32 max_in_flight = 6 [(buf.validate.field).uint32 = {gte: 1}];
}

message PostgresOnConflictConfig {
Expand All @@ -239,6 +243,10 @@ message MysqlDestinationConnectionOptions {
MysqlOnConflictConfig on_conflict = 3;
// Insert all valid records, skipping any that violate foreign key constraints.
bool skip_foreign_key_violations = 4;
// Configure batching options to handle how much data is sent to your database at once.
BatchConfig batch = 5;
// Determines the maximum number of parallel batched inserts.
optional uint32 max_in_flight = 6 [(buf.validate.field).uint32 = {gte: 1}];
}
message MysqlTruncateTableConfig {
bool truncate_before_insert = 1;
Expand All @@ -255,6 +263,10 @@ message MssqlDestinationConnectionOptions {
MssqlOnConflictConfig on_conflict = 3;
// Insert all valid records, skipping any that violate foreign key constraints.
bool skip_foreign_key_violations = 4;
// Configure batching options to handle how much data is sent to your database at once.
BatchConfig batch = 5;
// Determines the maximum number of parallel batched inserts.
optional uint32 max_in_flight = 6 [(buf.validate.field).uint32 = {gte: 1}];
}
message MssqlTruncateTableConfig {
bool truncate_before_insert = 1;
Expand All @@ -267,7 +279,7 @@ message AwsS3DestinationConnectionOptions {
// The storage class that will be used when objects are written to S3
StorageClass storage_class = 1;
// The maximum number of batched messages to have in flight at a given time. Increase this to improve throughput.
optional uint32 max_in_flight = 2;
optional uint32 max_in_flight = 2 [(buf.validate.field).uint32 = {gte: 1}];
// The maximum period (duration string) to wait on an upload before abandoning it and reattempting.
optional string timeout = 3;
// Configure batching options to more efficiently store records in S3
Expand Down
76 changes: 76 additions & 0 deletions docs/openapi/mgmt/v1alpha1/job.openapi.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1403,6 +1403,7 @@ components:
maxInFlight:
type: integer
title: max_in_flight
minimum: 1
description: The maximum number of batched messages to have in flight at a given time. Increase this to improve throughput.
timeout:
type: string
Expand Down Expand Up @@ -3196,6 +3197,21 @@ components:
description: MongoDB connection options for a job source
mgmt.v1alpha1.MssqlDestinationConnectionOptions:
type: object
allOf:
- anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
properties:
truncateTable:
allOf:
Expand All @@ -3214,6 +3230,16 @@ components:
type: boolean
title: skip_foreign_key_violations
description: Insert all valid records, skipping any that violate foreign key constraints.
batch:
allOf:
- title: batch
description: Configure batching options to handle how much data is sent to your database at once.
- $ref: '#/components/schemas/mgmt.v1alpha1.BatchConfig'
maxInFlight:
type: integer
title: max_in_flight
minimum: 1
description: Determines the maximum number of parallel batched inserts.
title: MssqlDestinationConnectionOptions
additionalProperties: false
mgmt.v1alpha1.MssqlOnConflictConfig:
Expand Down Expand Up @@ -3303,6 +3329,21 @@ components:
additionalProperties: false
mgmt.v1alpha1.MysqlDestinationConnectionOptions:
type: object
allOf:
- anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
properties:
truncateTable:
allOf:
Expand All @@ -3319,6 +3360,16 @@ components:
type: boolean
title: skip_foreign_key_violations
description: Insert all valid records, skipping any that violate foreign key constraints.
batch:
allOf:
- title: batch
description: Configure batching options to handle how much data is sent to your database at once.
- $ref: '#/components/schemas/mgmt.v1alpha1.BatchConfig'
maxInFlight:
type: integer
title: max_in_flight
minimum: 1
description: Determines the maximum number of parallel batched inserts.
title: MysqlDestinationConnectionOptions
additionalProperties: false
mgmt.v1alpha1.MysqlOnConflictConfig:
Expand Down Expand Up @@ -3659,6 +3710,21 @@ components:
additionalProperties: false
mgmt.v1alpha1.PostgresDestinationConnectionOptions:
type: object
allOf:
- anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
properties:
truncateTable:
allOf:
Expand All @@ -3675,6 +3741,16 @@ components:
type: boolean
title: skip_foreign_key_violations
description: Insert all valid records, skipping any that violate foreign key constraints.
batch:
allOf:
- title: batch
description: Configure batching options to handle how much data is sent to your database at once.
- $ref: '#/components/schemas/mgmt.v1alpha1.BatchConfig'
maxInFlight:
type: integer
title: max_in_flight
minimum: 1
description: Determines the maximum number of parallel batched inserts.
title: PostgresDestinationConnectionOptions
additionalProperties: false
mgmt.v1alpha1.PostgresOnConflictConfig:
Expand Down
94 changes: 94 additions & 0 deletions docs/openapi/neosync.mgmt.v1alpha1.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8515,6 +8515,7 @@ components:
maxInFlight:
type: integer
title: max_in_flight
minimum: 1
description: >-
The maximum number of batched messages to have in flight at a given
time. Increase this to improve throughput.
Expand Down Expand Up @@ -9940,6 +9941,21 @@ components:
description: MongoDB connection options for a job source
mgmt.v1alpha1.MssqlDestinationConnectionOptions:
type: object
allOf:
- anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
properties:
truncateTable:
allOf:
Expand All @@ -9962,6 +9978,18 @@ components:
description: >-
Insert all valid records, skipping any that violate foreign key
constraints.
batch:
allOf:
- title: batch
description: >-
Configure batching options to handle how much data is sent to
your database at once.
- $ref: '#/components/schemas/mgmt.v1alpha1.BatchConfig'
maxInFlight:
type: integer
title: max_in_flight
minimum: 1
description: Determines the maximum number of parallel batched inserts.
title: MssqlDestinationConnectionOptions
additionalProperties: false
mgmt.v1alpha1.MssqlOnConflictConfig:
Expand Down Expand Up @@ -10051,6 +10079,21 @@ components:
additionalProperties: false
mgmt.v1alpha1.MysqlDestinationConnectionOptions:
type: object
allOf:
- anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
properties:
truncateTable:
allOf:
Expand All @@ -10069,6 +10112,18 @@ components:
description: >-
Insert all valid records, skipping any that violate foreign key
constraints.
batch:
allOf:
- title: batch
description: >-
Configure batching options to handle how much data is sent to
your database at once.
- $ref: '#/components/schemas/mgmt.v1alpha1.BatchConfig'
maxInFlight:
type: integer
title: max_in_flight
minimum: 1
description: Determines the maximum number of parallel batched inserts.
title: MysqlDestinationConnectionOptions
additionalProperties: false
mgmt.v1alpha1.MysqlOnConflictConfig:
Expand Down Expand Up @@ -10228,6 +10283,21 @@ components:
additionalProperties: false
mgmt.v1alpha1.PostgresDestinationConnectionOptions:
type: object
allOf:
- anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
anyOf:
- required:
- maxInFlight
- not:
anyOf:
- required:
- maxInFlight
properties:
truncateTable:
allOf:
Expand All @@ -10246,6 +10316,18 @@ components:
description: >-
Insert all valid records, skipping any that violate foreign key
constraints.
batch:
allOf:
- title: batch
description: >-
Configure batching options to handle how much data is sent to
your database at once.
- $ref: '#/components/schemas/mgmt.v1alpha1.BatchConfig'
maxInFlight:
type: integer
title: max_in_flight
minimum: 1
description: Determines the maximum number of parallel batched inserts.
title: PostgresDestinationConnectionOptions
additionalProperties: false
mgmt.v1alpha1.PostgresOnConflictConfig:
Expand Down Expand Up @@ -11834,6 +11916,13 @@ components:
anyOf:
- required:
- reason
- anyOf:
- required:
- trialExpiresAt
- not:
anyOf:
- required:
- trialExpiresAt
properties:
isValid:
type: boolean
Expand Down Expand Up @@ -11871,6 +11960,11 @@ components:
- title: account_status
description: The current status of the account. Default is valid.
- $ref: '#/components/schemas/mgmt.v1alpha1.AccountStatus'
trialExpiresAt:
allOf:
- title: trial_expires_at
description: The time when the trial expires
- $ref: '#/components/schemas/google.protobuf.Timestamp'
title: IsAccountStatusValidResponse
additionalProperties: false
mgmt.v1alpha1.IsUserInAccountRequest:
Expand Down
Loading

0 comments on commit 451a0c5

Please sign in to comment.